Mizu fetch url (#721)

* Fix mizu fetch url

* fix env files

Co-authored-by: RoyUP9 <87927115+RoyUP9@users.noreply.github.com>

.dockerignore

@@ -2,7 +2,6 @@
 .dockerignore
 .editorconfig
 .gitignore
-**/.env*
 Dockerfile
 Makefile
 LICENSE

.github/workflows/pr_validation.yml

@@ -49,10 +49,10 @@ jobs:
     name: Build UI
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Node 14
+      - name: Set up Node 16
         uses: actions/setup-node@v2
         with:
-          node-version: '14'
+          node-version: '16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2

.github/workflows/publish.yml

@@ -1,90 +0,0 @@
-name: publish
-
-on:
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-
-concurrency:
-  group: mizu-publish-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
-        with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
-      - uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
-      - name: Auto Increment Semver Action
-        uses: MCKanpolat/auto-semver-action@1.0.5
-        id: versioning
-        with:
-          releaseType: ${{ steps.condval.outputs.value }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Get version parameters
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-      - name: Docker meta
-        id: meta
-        uses: crazy-max/ghaction-docker-meta@v2
-        with:
-          images: ${{ steps.base_image_step.outputs.image }}
-          tags: |
-            type=sha
-            type=raw,${{ github.sha }}
-            type=raw,${{ steps.versioning.outputs.version }}
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            SEM_VER=${{ steps.versioning.outputs.version }}
-            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
-            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
-            COMMIT_HASH=${{ github.sha }}
-      - name: Build and Push CLI
-        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
-      - shell: bash
-        run: |
-          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
-      - name: publish
-        uses: ncipollo/release-action@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          artifacts: "cli/bin/*"
-          commit: ${{ steps.version_parameters.outputs.branch }}
-          tag: ${{ steps.versioning.outputs.version }}
-          prerelease: ${{ github.ref != 'refs/heads/main' }}
-          bodyFile: 'cli/bin/README.md'

.github/workflows/release.yml

@@ -0,0 +1,278 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-publish-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker-registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        target:
+          - amd64
+          - arm64v8
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            up9inc/mizu
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=-${{ matrix.target }},onlatest=true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            TARGETARCH=${{ matrix.target }}
+            SEM_VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+
+  gcp-registry:
+    name: Push Docker image to GCR
+    runs-on: ubuntu-latest
+
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        target:
+          - amd64
+          - arm64v8
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ${{ steps.base_image_step.outputs.image }}
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=-${{ matrix.target }},onlatest=true
+
+      - name: Login to GCR
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            TARGETARCH=${{ matrix.target }}
+            SEM_VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+
+  docker-manifest:
+    name: Create and Push a Docker Manifest
+    runs-on: ubuntu-latest
+    needs: [docker-registry]
+    steps:
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            up9inc/mizu
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Create manifest
+        run: |
+          while IFS= read -r line; do
+            docker manifest create $line --amend $line-amd64 --amend $line-arm64v8
+          done <<< "${{ steps.meta.outputs.tags }}"
+
+      - name: Push manifest
+        run: |
+          while IFS= read -r line; do
+            docker manifest push $line
+          done <<< "${{ steps.meta.outputs.tags }}"
+
+  cli:
+    name: Build the CLI and publish
+    runs-on: ubuntu-latest
+    needs: [docker-manifest, gcp-registry]
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
+
+      - uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto Increment Semver Action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Build and Push CLI
+        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+
+      - name: Log the version into a .txt file
+        shell: bash
+        run: |
+          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
+
+      - name: Release
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          artifacts: "cli/bin/*"
+          commit: ${{ steps.version_parameters.outputs.branch }}
+          tag: ${{ steps.versioning.outputs.version }}
+          prerelease: ${{ github.ref != 'refs/heads/main' }}
+          bodyFile: 'cli/bin/README.md'

Dockerfile

@@ -1,4 +1,8 @@
-FROM node:14-slim AS site-build
+ARG BUILDARCH=amd64
+ARG TARGETARCH=amd64
+
+### Front-end
+FROM node:16 AS front-end
 
 WORKDIR /app/ui-build
 
@@ -7,57 +11,95 @@ COPY ui/package-lock.json .
 RUN npm i
 COPY ui .
 RUN npm run build
+RUN npm run build-ent
+
+### Base builder image for native builds architecture
+FROM golang:1.16-alpine AS builder-native-base
+ENV CGO_ENABLED=1 GOOS=linux
+RUN apk add libpcap-dev g++ perl-utils
 
 
-FROM golang:1.16-alpine AS builder
-# Set necessary environment variables needed for our image.
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
+### Intermediate builder image for x86-64 to x86-64 native builds
+FROM builder-native-base AS builder-from-amd64-to-amd64
+ENV GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash perl-utils
 
-# Move to agent working directory (/agent-build).
+### Intermediate builder image for AArch64 to AArch64 native builds
+FROM builder-native-base AS builder-from-arm64v8-to-arm64v8
+ENV GOARCH=arm64
+
+
+### Builder image for x86-64 to AArch64 cross-compilation
+FROM up9inc/linux-arm64-musl-go-libpcap AS builder-from-amd64-to-arm64v8
+ENV CGO_ENABLED=1 GOOS=linux
+ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
+
+
+### Final builder image where the build happens
+# Possible build strategies:
+# BUILDARCH=amd64 TARGETARCH=amd64
+# BUILDARCH=arm64v8 TARGETARCH=arm64v8
+# BUILDARCH=amd64 TARGETARCH=arm64v8
+ARG BUILDARCH=amd64
+ARG TARGETARCH=amd64
+FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
+
+# Move to agent working directory (/agent-build)
 WORKDIR /app/agent-build
 
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.* ../tap/api/
+COPY tap/api/go.mod ../tap/api/
+COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
+COPY tap/extensions/http/go.mod ../tap/extensions/http/
+COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
+COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
 RUN go mod download
-# cheap trick to make the build faster (As long as go.mod wasn't changes)
+# cheap trick to make the build faster (as long as go.mod did not change)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
+# Copy and build agent code
+COPY shared ../shared
+COPY tap ../tap
+COPY agent .
+
 ARG COMMIT_HASH
 ARG GIT_BRANCH
 ARG BUILD_TIMESTAMP
 ARG SEM_VER=0.0.0
 
-# Copy and build agent code
-COPY shared ../shared
-COPY tap ../tap
-COPY agent .
-RUN go build -ldflags="-s -w \
-     -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-     -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
-     -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-     -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
+WORKDIR /app/agent-build
 
-COPY devops/build_extensions.sh ..
-RUN cd .. && /bin/bash build_extensions.sh
+RUN go build -ldflags="-extldflags=-static -s -w \
+    -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+    -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
+    -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+    -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
-FROM alpine:3.15
+# Download Basenine executable, verify the sha1sum
+ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256
+RUN chmod +x ./basenine_linux_${GOARCH}
+RUN mv ./basenine_linux_${GOARCH} ./basenine
 
-RUN apk add bash libpcap-dev
 
-WORKDIR /app
-
-# Copy binary and config files from /build to root folder of scratch container.
-COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent/build/extensions", "extensions"]
-COPY --from=site-build ["/app/ui-build/build", "site"]
-RUN mkdir /app/data/
+### The shipped image
+ARG TARGETARCH=amd64
+FROM ${TARGETARCH}/busybox:latest
 
 # gin-gonic runs in debug mode without this
 ENV GIN_MODE=release
 
+WORKDIR /app/data/
+WORKDIR /app
+
+# Copy binary and config files from /build to root folder of scratch container.
+COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
+COPY --from=front-end ["/app/ui-build/build", "site"]
+COPY --from=front-end ["/app/ui-build/build-ent", "site-standalone"]
+
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
-ENTRYPOINT "/app/mizuagent"
+ENTRYPOINT ["/app/mizuagent"]

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
+.PHONY: help ui agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -37,13 +37,11 @@ build-cli-ci: ## Build CLI for CI.
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
-	${MAKE} extensions
 	@ls -l agent/build
 
 agent-debug: ## Build agent for debug.
 	@(echo "building mizu agent for debug.." )
 	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
-	${MAKE} extensions-debug
 	@ls -l agent/build
 
 docker: ## Build and publish agent docker image.
@@ -55,10 +53,6 @@ push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	devops/build-push-featurebranch.sh
 
-push-docker-debug:
-	@echo "publishing debug Docker image .. "
-	devops/build-push-featurebranch-debug.sh
-
 build-docker-ci: ## Build agent docker image for CI.
 	@echo "building docker image for ci"
 	devops/build-agent-ci.sh
@@ -71,7 +65,7 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
+clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
@@ -82,18 +76,9 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-extensions:  ## Clean extensions
-	@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
-
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-extensions-debug:
-	devops/build_extensions_debug.sh
-
-extensions:
-	devops/build_extensions.sh
-
 test-cli:
 	@echo "running cli tests"; cd cli && $(MAKE) test
 

README.md

@@ -24,7 +24,7 @@ Think TCPDump and Wireshark re-invented for Kubernetes.
 
 - Simple and powerful CLI
 - Monitoring network traffic in real-time. Supported protocols:
-  - [HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc2616) (REST, etc.)
+  - [HTTP/1.x](https://datatracker.ietf.org/doc/html/rfc2616) (REST, GraphQL, SOAP, etc.)
   - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
   - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
   - [Apache Kafka](https://kafka.apache.org/protocol)

acceptanceTests/cypress.json

@@ -4,17 +4,22 @@
   "viewportHeight": 1080,
   "video": false,
   "screenshotOnRunFailure": false,
-
-  "testFiles":
-  ["tests/GuiPort.js",
-  "tests/MultipleNamespaces.js",
-  "tests/Redact.js",
+  "defaultCommandTimeout": 5000,
+  "testFiles": [
+    "tests/GuiPort.js",
+    "tests/MultipleNamespaces.js",
+    "tests/Redact.js",
     "tests/NoRedact.js",
-  "tests/Regex.js"],
+    "tests/Regex.js",
+    "tests/RegexMasking.js",
+    "tests/IgnoredUserAgents.js",
+    "tests/UiTest.js"
+  ],
 
   "env": {
     "testUrl": "http://localhost:8899/",
     "redactHeaderContent": "User-Header[REDACTED]",
-    "redactBodyContent": "{ \"User\": \"[REDACTED]\" }"
+    "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
+    "regexMaskingBodyContent": "[REDACTED]"
   }
 }

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -7,3 +7,12 @@ export function isValueExistsInElement(shouldInclude, content, domPathToContaine
         });
     });
 }
+
+export function resizeToHugeMizu() {
+    cy.viewport(1920, 3500);
+}
+
+export function resizeToNormalMizu() {
+    cy.viewport(1920, 1080);
+}
+

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -0,0 +1,34 @@
+import {isValueExistsInElement, resizeToHugeMizu} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+it('going through each entry', function () {
+    resizeToHugeMizu();
+    cy.get('#total-entries').then(number => {
+        const getNum = () => {
+            const numOfEntries = number.text();
+            return parseInt(numOfEntries);
+        };
+        cy.wrap({ there: getNum }).invoke('there').should('be.gte', 25);
+
+        checkThatAllEntriesShown();
+
+        const entriesNum = getNum();
+        [...Array(entriesNum).keys()].map(checkEntry);
+    });
+});
+
+function checkThatAllEntriesShown() {
+    cy.get('#entries-length').then(number => {
+        if (number.text() === '1')
+            cy.get('[title="Fetch old records"]').click();
+    });
+}
+
+function checkEntry(entryIndex) {
+    cy.get(`#entry-${entryIndex}`).click();
+    cy.get('#tbody-Headers').should('be.visible');
+    isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+}

acceptanceTests/cypress/integration/tests/MultipleNamespaces.js

@@ -15,4 +15,3 @@ function doItFunc(number) {
         findLineAndCheck(getExpectedDetailsDict(podName, namespace));
     });
 }
-

acceptanceTests/cypress/integration/tests/RegexMasking.js

@@ -0,0 +1,7 @@
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+})
+
+isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -0,0 +1,218 @@
+import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
+import {resizeToHugeMizu, resizeToNormalMizu} from "../testHelpers/TrafficHelper";
+const greenFilterColor = 'rgb(210, 250, 210)';
+const redFilterColor = 'rgb(250, 214, 220)';
+const refreshWaitTimeout = 10000;
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+it('top bar check', function () {
+    const podName1 = 'httpbin', namespace1 = 'mizu-tests';
+    const podName2 = 'httpbin2', namespace2 = 'mizu-tests';
+
+    cy.get('.podsCount').trigger('mouseover');
+    findLineAndCheck(getExpectedDetailsDict(podName1, namespace1));
+    findLineAndCheck(getExpectedDetailsDict(podName2, namespace2));
+    cy.reload();
+});
+
+it('filtering guide check', function () {
+    cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
+    cy.get('#modal-modal-title').should('be.visible');
+    cy.get('[lang="en"]').click(0, 0);
+    cy.get('#modal-modal-title').should('not.exist');
+});
+
+checkIllegalFilter('invalid filter');
+
+checkFilter({
+    name: 'http',
+    leftSidePath: '> :nth-child(1) > :nth-child(1)',
+    leftSideExpectedText: 'HTTP',
+    rightSidePath: '[title=HTTP]',
+    rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
+    applyByEnter: true
+});
+
+checkFilter({
+    name: 'response.status == 200',
+    leftSidePath: '[title="Status Code"]',
+    leftSideExpectedText: '200',
+    rightSidePath: '> :nth-child(2) [title="Status Code"]',
+    rightSideExpectedText: '200',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'src.name == ""',
+    leftSidePath: '[title="Source Name"]',
+    leftSideExpectedText: '[Unresolved]',
+    rightSidePath: '> :nth-child(2) [title="Source Name"]',
+    rightSideExpectedText: '[Unresolved]',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'method == "GET"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    leftSideExpectedText: 'GET',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    rightSideExpectedText: 'GET',
+    applyByEnter: true
+});
+
+checkFilter({
+    name: 'summary == "/get"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    leftSideExpectedText: '/get',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    rightSideExpectedText: '/get',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'dst.name == "httpbin.mizu-tests"',
+    leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+    leftSideExpectedText: 'httpbin.mizu-tests',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+    rightSideExpectedText: 'httpbin.mizu-tests',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'src.ip == "127.0.0.1"',
+    leftSidePath: '[title="Source IP"]',
+    leftSideExpectedText: '127.0.0.1',
+    rightSidePath: '> :nth-child(2) [title="Source IP"]',
+    rightSideExpectedText: '127.0.0.1',
+    applyByEnter: false
+});
+
+checkFilterNoResults('method == "POST"');
+
+function checkFilterNoResults(filterName) {
+    it(`checking the filter: ${filterName}. Expecting no results`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            // applying the filter
+            cy.get('.w-tc-editor-text').type(filterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
+            cy.get('[type="submit"]').click();
+
+            // waiting for the entries number to load
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+
+            // the DOM should show 0 entries
+            cy.get('#entries-length').should('have.text', '0');
+
+            // going through every potential entry and verifies that it doesn't exist
+            [...Array(parseInt(totalEntries)).keys()].map(shouldNotExist);
+
+            cy.get('[title="Fetch old records"]').click();
+            cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
+            cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function shouldNotExist(entryNum) {
+    cy.get(`entry-${entryNum}`).should('not.exist');
+}
+
+function checkIllegalFilter(illegalFilterName) {
+    it(`should show red search bar with the input: ${illegalFilterName}`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            cy.get('.w-tc-editor-text').type(illegalFilterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', redFilterColor);
+            cy.get('[type="submit"]').click();
+
+            cy.get('[role="alert"]').should('be.visible');
+            cy.get('.w-tc-editor-text').clear();
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+function checkFilter(filterDetails){
+    const {name, leftSidePath, rightSidePath, rightSideExpectedText, leftSideExpectedText, applyByEnter} = filterDetails;
+    const entriesForDeeperCheck = 5;
+
+    it(`checking the filter: ${name}`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            // checks the hover on the last entry (the only one in DOM at the beginning)
+            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
+
+            // applying the filter with alt+enter or with the button
+            cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
+            if (!applyByEnter)
+                cy.get('[type="submit"]').click();
+
+            // only one entry in DOM after filtering, checking all four checks on it
+            leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
+            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, name);
+
+            cy.get('[title="Fetch old records"]').click();
+            resizeToHugeMizu();
+
+            // waiting for the entries number to load
+            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+
+            // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
+            [...Array(parseInt(totalEntries)).keys()].forEach(entryNum => {
+                leftTextCheck(entryNum, leftSidePath, leftSideExpectedText);
+            });
+
+            // making the other 3 checks on the first X entries (longer time for each check)
+            deeperChcek(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
+
+            // reloading then waiting for the entries number to load
+            resizeToNormalMizu();
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
+    [...Array(entriesNumToCheck).keys()].forEach(entryNum => {
+        leftOnHoverCheck(entryNum, leftSidePath, filterName);
+
+        cy.get(`#list #entry-${entryNum}`).click();
+        rightTextCheck(rightSidePath, rightSideExpectedText);
+        rightOnHoverCheck(rightSidePath, filterName);
+    });
+}
+
+function leftTextCheck(entryNum, path, expectedText) {
+    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
+}
+
+function leftOnHoverCheck(entryNum, path, filterName) {
+    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
+    cy.get(`#list #entry-${entryNum} .Queryable-Tooltip`).should('have.text', filterName);
+}
+
+function rightTextCheck(path, expectedText) {
+    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).should('have.text', expectedText);
+}
+
+function rightOnHoverCheck(path, expectedText) {
+    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).trigger('mouseover');
+    cy.get(`.TrafficPage-Container > :nth-child(2) .Queryable-Tooltip`).should('have.text', expectedText);
+}

acceptanceTests/tap_test.go

@@ -62,35 +62,7 @@ func TestTap(t *testing.T) {
 				}
 			}
 
-			entriesCheckFunc := func() error {
-				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
-				if err != nil {
-					return err
-				}
-				err = checkEntriesAtLeast(entries, 1)
-				if err != nil {
-					return err
-				}
-				entry := entries[0]
-
-				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
-				requestResult, requestErr := executeHttpGetRequest(entryUrl)
-				if requestErr != nil {
-					return fmt.Errorf("failed to get entry, err: %v", requestErr)
-				}
-
-				if requestResult == nil {
-					return fmt.Errorf("unexpected nil entry result")
-				}
-
-				return nil
-			}
-			if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
-				t.Errorf("%v", err)
-				return
-			}
+			runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/UiTest.js\"")
 		})
 	}
 }
@@ -377,7 +349,7 @@ func TestTapRedact(t *testing.T) {
 		}
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Redact.js\""))
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redact.js\"")
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -480,41 +452,8 @@ func TestTapRegexMasking(t *testing.T) {
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+	runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\"")
 
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-		firstEntry := entries[0]
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr := executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		request := entry["request"].(map[string]interface{})
-
-		postData := request["postData"].(map[string]interface{})
-		textData := postData["text"].(string)
-
-		if textData != "[REDACTED]" {
-			return fmt.Errorf("unexpected result - body is not redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
 }
 
 func TestTapIgnoredUserAgents(t *testing.T) {
@@ -575,45 +514,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		}
 	}
 
-	ignoredUserAgentsCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-
-		for _, entryInterface := range entries {
-			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"])
-			requestResult, requestErr := executeHttpGetRequest(entryUrl)
-			if requestErr != nil {
-				return fmt.Errorf("failed to get entry, err: %v", requestErr)
-			}
-
-			entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-			request := entry["request"].(map[string]interface{})
-
-			headers := request["_headers"].([]interface{})
-			for _, headerInterface := range headers {
-				header := headerInterface.(map[string]interface{})
-				if header["name"].(string) != ignoredUserAgentCustomHeader {
-					continue
-				}
-
-				return fmt.Errorf("unexpected result - user agent is not ignored")
-			}
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, ignoredUserAgentsCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\"")
 }
 
 func TestTapDumpLogs(t *testing.T) {

agent/README.md

@@ -1,20 +0,0 @@
-# mizu agent
-Agent for MIZU (API server and tapper)
-Basic APIs:
-* /stats - retrieve statistics of collected data
-* /viewer - web ui
-
-## Remote Debugging
-### Setup remote debugging
-1. Run `go get github.com/go-delve/delve/cmd/dlv`
-2. Create a "Go Remote" run/debug configuration in Intellij, set to localhost:2345
-3. Build and push a debug image using
-   `docker build . -t  gcr.io/up9-docker-hub/mizu/debug:latest -f debug.Dockerfile && docker push gcr.io/up9-docker-hub/mizu/debug:latest`
-
-### Connecting
-1. Start mizu using the cli with the debug
-   image `mizu tap --set agent-image=gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
-2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
-3. Run the run/debug configuration you've created earlier in Intellij.
-
-<small>Do note that dlv won't start the api until a debugger connects to it.</small>

agent/go.mod

@@ -6,25 +6,30 @@ require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
 	github.com/chanced/openapi v0.0.6
 	github.com/djherbis/atime v1.0.0
+	github.com/elastic/go-elasticsearch/v7 v7.16.0
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-playground/locales v0.13.0
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
-	github.com/google/martian v2.1.0+incompatible
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
+	github.com/ory/keto-client-go v0.7.0-alpha.1
 	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d
+	github.com/stretchr/testify v1.7.0
+	github.com/up9inc/basenine/client/go v0.0.0-20220125035724-573fff0d5075
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
+	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
+	github.com/up9inc/mizu/tap/extensions/http v0.0.0
+	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
+	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	golang.org/x/text v0.3.5 // indirect
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
 	k8s.io/client-go v0.21.2
@@ -35,3 +40,11 @@ replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
+
+replace github.com/up9inc/mizu/tap/extensions/amqp v0.0.0 => ../tap/extensions/amqp
+
+replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/http
+
+replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
+
+replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis

agent/go.sum

@@ -54,7 +54,9 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YH
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -70,6 +72,13 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
+github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
+github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
+github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -115,6 +124,11 @@ github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/elastic/go-elasticsearch/v7 v7.16.0 h1:GHsxDFXIAlhSleXun4kwA89P7kQFADRChqvgOPeYP5A=
+github.com/elastic/go-elasticsearch/v7 v7.16.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -128,10 +142,13 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
+github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
@@ -164,10 +181,22 @@ github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70t
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
+github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
 github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
+github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ=
+github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk=
+github.com/go-openapi/analysis v0.20.0 h1:UN09o0kNhleunxW7LR+KnltD0YrJ8FF03pSqvAN3Vro=
+github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og=
 github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.20.1 h1:j23mMDtRxMwIobkpId7sWh7Ddcx4ivaoqUbfXx5P+a8=
+github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
@@ -178,32 +207,73 @@ github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3Hfo
 github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
 github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
+github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI=
 github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
+github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY=
+github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
+github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
+github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4=
+github.com/go-openapi/loads v0.20.2 h1:z5p5Xf5wujMxS1y8aP+vxwW5qYT2zdJBbXKmQUG3lcc=
+github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o=
 github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
 github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
 github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
+github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
+github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
+github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
+github.com/go-openapi/runtime v0.20.0 h1:DEV4oYH28MqakaabtbxH0cjvlzFegi/15kfUVCfiZW0=
+github.com/go-openapi/runtime v0.20.0/go.mod h1:2WnLRxMiOUWNN0UZskSkxW0+WXdfB1KmqRKCFH+ZWYk=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
+github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
+github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ=
+github.com/go-openapi/spec v0.20.3 h1:uH9RQ6vdyPSs2pSy9fL8QPspDF2AMIMPtmK5coSSjtQ=
+github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg=
 github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
+github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
 github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
+github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
 github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
+github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
+github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
+github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk=
+github.com/go-openapi/strfmt v0.20.3 h1:YVG4ZgPZ00km/lRHrIf7c6cKL5/4FAUtG2T9RxWAgDY=
+github.com/go-openapi/strfmt v0.20.3/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
+github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
+github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M=
+github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
+github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo=
 github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8=
+github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
+github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI=
+github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
+github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw=
+github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
@@ -214,8 +284,34 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -250,6 +346,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -327,7 +425,12 @@ github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
@@ -336,13 +439,23 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.14.1 h1:hLQYb23E8/fO+1u53d02A97a8UnsddcvYzq4ERRU4ds=
+github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -358,9 +471,13 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
@@ -378,6 +495,12 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -387,12 +510,16 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/ohler55/ojg v1.12.12 h1:hepbQFn7GHAecTPmwS3j5dCiOLsOpzPLvhiqnlAVAoE=
+github.com/ohler55/ojg v1.12.12/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+nyJU88=
+github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -403,8 +530,12 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
+github.com/ory/keto-client-go v0.7.0-alpha.1 h1:8iZz37j7YXFvFBxO/lAqFkGyQK7KysToQhAE4aNFzqE=
+github.com/ory/keto-client-go v0.7.0-alpha.1/go.mod h1:xtlyH3xz9lgzvYnuzFXeGwaui0Swc/hxnmqzY2g8AYs=
 github.com/ory/kratos-client-go v0.8.2-alpha.1 h1:YlKhGOSZjounlB9iY4xSWlqHbyLYkeLzlLk8ZL7/nEM=
 github.com/ory/kratos-client-go v0.8.2-alpha.1/go.mod h1:dOQIsar76K07wMPJD/6aMhrWyY+sFGEagLDLso1CpsA=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -412,7 +543,11 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
+github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -439,6 +574,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -446,10 +583,14 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
 github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/segmentio/kafka-go v0.4.27 h1:sIhEozeL/TLN2mZ5dkG462vcGEWYKS+u31sXPjKhAM4=
+github.com/segmentio/kafka-go v0.4.27/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -458,6 +599,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -494,18 +636,25 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d h1:WTz53dcfqCIWZpZLQoHbIcNc21s0ZHEZH7EqMPp99qQ=
-github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220125035724-573fff0d5075 h1:Tp0yckZkvb8BNC+bB4B+fBuQdomTner6bOte1S8SYCo=
+github.com/up9inc/basenine/client/go v0.0.0-20220125035724-573fff0d5075/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
 github.com/wI2L/jsondiff v0.1.1/go.mod h1:bAbJSAJXZtfOCZ5y3v7Mfb6UQa3DGdGFjQj1cNv8EcM=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -514,6 +663,13 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI=
+go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -527,11 +683,15 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
@@ -600,11 +760,15 @@ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -618,6 +782,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -633,10 +798,13 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -689,6 +857,7 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiT
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -696,9 +865,13 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -843,6 +1016,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

agent/keto/Dockerfile

@@ -0,0 +1,13 @@
+FROM oryd/keto:v0.7.0-alpha.1-sqlite
+
+USER root
+
+RUN apk add sqlite
+
+RUN mkdir -p /etc/config/keto
+
+COPY ./keto.yml /etc/config/keto/keto.yml
+COPY ./start.sh /opt/start.sh
+RUN chmod +x /opt/start.sh
+
+ENTRYPOINT ["/opt/start.sh"]

agent/keto/build-push-featurebranch.sh

@@ -3,7 +3,7 @@ set -e
 
 GCP_PROJECT=up9-docker-hub
 REPOSITORY=gcr.io/$GCP_PROJECT
-SERVER_NAME=mizu
+SERVER_NAME=mizu-keto
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
 
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
@@ -19,7 +19,7 @@ fi
 
 echo "building ${DOCKER_TAGGED_BUILDS[@]}"
 DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build -f debug.Dockerfile $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do

agent/keto/keto.yml

@@ -0,0 +1,22 @@
+version: v0.7.0-alpha.1
+
+dsn: sqlite:///app/data/kratos.sqlite?_fk=true
+
+log:
+  level: info
+  format: text
+  leak_sensitive_values: false
+
+serve:
+  read:
+    host: 0.0.0.0
+    port: 4466
+  write:
+    host: 0.0.0.0
+    port: 4467
+
+namespaces:
+  - id: 0
+    name: system
+  - id: 1
+    name: workspaces

agent/keto/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+keto migrate up -c /etc/config/keto/keto.yml --yes # this initializes the db
+keto serve -c /etc/config/keto/keto.yml # start keto

agent/kratos/kratos.yml

@@ -57,7 +57,7 @@ selfservice:
 log:
   level: info
   format: text
-  leak_sensitive_values: true
+  leak_sensitive_values: false
 
 secrets:
   cookie:

agent/main.go

@@ -9,18 +9,17 @@ import (
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/elastic"
 	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/oas"
 	"mizuserver/pkg/routes"
+	"mizuserver/pkg/servicemap"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
 	"os/signal"
-	"path"
-	"path/filepath"
-	"plugin"
 	"sort"
 	"strconv"
 	"strings"
@@ -39,6 +38,11 @@ import (
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
+
+	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
+	httpExt "github.com/up9inc/mizu/tap/extensions/http"
+	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
+	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -153,6 +157,10 @@ func enableExpFeatureIfNeeded() {
 	if config.Config.OAS {
 		oas.GetOasGeneratorInstance().Start()
 	}
+	if config.Config.ServiceMap {
+		servicemap.GetInstance().SetConfig(config.Config)
+	}
+	elastic.GetInstance().Configure(config.Config.Elastic)
 }
 
 func configureBasenineServer(host string, port string) {
@@ -161,7 +169,7 @@ func configureBasenineServer(host string, port string) {
 		wait.WithWait(200*time.Millisecond),
 		wait.WithBreak(50*time.Millisecond),
 		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(true),
+		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
 	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
 		logger.Log.Panicf("Basenine is not available!")
 	}
@@ -185,36 +193,36 @@ func configureBasenineServer(host string, port string) {
 }
 
 func loadExtensions() {
-	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
-	extensionsDir := path.Join(dir, "./extensions/")
-
-	files, err := ioutil.ReadDir(extensionsDir)
-	if err != nil {
-		logger.Log.Fatal(err)
-	}
-	extensions = make([]*tapApi.Extension, len(files))
+	extensions = make([]*tapApi.Extension, 4)
 	extensionsMap = make(map[string]*tapApi.Extension)
-	for i, file := range files {
-		filename := file.Name()
-		logger.Log.Infof("Loading extension: %s", filename)
-		extension := &tapApi.Extension{
-			Path: path.Join(extensionsDir, filename),
-		}
-		plug, _ := plugin.Open(extension.Path)
-		extension.Plug = plug
-		symDissector, err := plug.Lookup("Dissector")
 
-		var dissector tapApi.Dissector
-		var ok bool
-		dissector, ok = symDissector.(tapApi.Dissector)
-		if err != nil || !ok {
-			panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
-		}
-		dissector.Register(extension)
-		extension.Dissector = dissector
-		extensions[i] = extension
-		extensionsMap[extension.Protocol.Name] = extension
-	}
+	extensionAmqp := &tapApi.Extension{}
+	dissectorAmqp := amqpExt.NewDissector()
+	dissectorAmqp.Register(extensionAmqp)
+	extensionAmqp.Dissector = dissectorAmqp
+	extensions[0] = extensionAmqp
+	extensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+
+	extensionHttp := &tapApi.Extension{}
+	dissectorHttp := httpExt.NewDissector()
+	dissectorHttp.Register(extensionHttp)
+	extensionHttp.Dissector = dissectorHttp
+	extensions[1] = extensionHttp
+	extensionsMap[extensionHttp.Protocol.Name] = extensionHttp
+
+	extensionKafka := &tapApi.Extension{}
+	dissectorKafka := kafkaExt.NewDissector()
+	dissectorKafka.Register(extensionKafka)
+	extensionKafka.Dissector = dissectorKafka
+	extensions[2] = extensionKafka
+	extensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+
+	extensionRedis := &tapApi.Extension{}
+	dissectorRedis := redisExt.NewDissector()
+	dissectorRedis.Register(extensionRedis)
+	extensionRedis.Dissector = dissectorRedis
+	extensions[3] = extensionRedis
+	extensionsMap[extensionRedis.Protocol.Name] = extensionRedis
 
 	sort.Slice(extensions, func(i, j int) bool {
 		return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority
@@ -243,7 +251,12 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	if err := setUIFlags(); err != nil {
 		logger.Log.Errorf("Error setting ui mode, err: %v", err)
 	}
-	app.Use(static.ServeRoot("/", "./site"))
+
+	if config.Config.StandaloneMode {
+		app.Use(static.ServeRoot("/", "./site-standalone"))
+	} else {
+		app.Use(static.ServeRoot("/", "./site"))
+	}
 
 	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
@@ -254,10 +267,12 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 		routes.UserRoutes(app)
 		routes.InstallRoutes(app)
 	}
-
 	if config.Config.OAS {
 		routes.OASRoutes(app)
 	}
+	if config.Config.ServiceMap {
+		routes.ServiceMapRoutes(app)
+	}
 
 	routes.QueryRoutes(app)
 	routes.EntriesRoutes(app)
@@ -284,8 +299,8 @@ func setUIFlags() error {
 		return err
 	}
 
-	replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
-	replacedContent = strings.Replace(replacedContent, "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
+	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
 
 	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
 	if err != nil {

agent/pkg/api/main.go

@@ -5,6 +5,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/elastic"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"os"
@@ -13,16 +15,17 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
+	"mizuserver/pkg/servicemap"
 
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/oas"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
 
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+
 	basenine "github.com/up9inc/basenine/client/go"
 )
 
@@ -115,8 +118,6 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	}
 
 	for item := range outputItems {
-		providers.EntryAdded()
-
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
@@ -132,7 +133,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
+			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
 				mizuEntry.Rules = rules
@@ -145,7 +146,13 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		if err != nil {
 			panic(err)
 		}
+
+		providers.EntryAdded(len(data))
+
 		connection.SendText(string(data))
+
+		servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
+		elastic.GetInstance().PushEntry(mizuEntry)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
@@ -49,7 +48,7 @@ func init() {
 func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
 	app.GET("/ws", func(c *gin.Context) {
 		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
-	}, middlewares.RequiresAuth())
+	})
 
 	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
 		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)

agent/pkg/controllers/entries_controller.go

@@ -2,8 +2,8 @@ package controllers
 
 import (
 	"encoding/json"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/models"
-	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
 	"strconv"
@@ -127,7 +127,7 @@ func GetEntry(c *gin.Context) {
 	var rules []map[string]interface{}
 	var isRulesEnabled bool
 	if entry.Protocol.Name == "http" {
-		harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
 		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)

agent/pkg/controllers/install_controller.go

@@ -16,3 +16,8 @@ func IsSetupNecessary(c *gin.Context) {
 		c.JSON(http.StatusOK, IsInstallNeeded)
 	}
 }
+
+func SetupAdminUser(c *gin.Context) {
+	token, err, formErrorMessages := providers.CreateAdminUser(c.PostForm("password"), c.Request.Context())
+	handleRegistration(token, err, formErrorMessages, c)
+}

agent/pkg/controllers/service_map_controller.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"mizuserver/pkg/servicemap"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+)
+
+type ServiceMapController struct {
+	service servicemap.ServiceMap
+}
+
+func NewServiceMapController() *ServiceMapController {
+	return &ServiceMapController{
+		service: servicemap.GetInstance(),
+	}
+}
+
+func (s *ServiceMapController) Status(c *gin.Context) {
+	c.JSON(http.StatusOK, s.service.GetStatus())
+}
+
+func (s *ServiceMapController) Get(c *gin.Context) {
+	response := &servicemap.ServiceMapResponse{
+		Status: s.service.GetStatus(),
+		Nodes:  s.service.GetNodes(),
+		Edges:  s.service.GetEdges(),
+	}
+	c.JSON(http.StatusOK, response)
+}
+
+func (s *ServiceMapController) Reset(c *gin.Context) {
+	s.service.Reset()
+	s.Status(c)
+}

agent/pkg/controllers/service_map_controller_test.go

@@ -0,0 +1,147 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"mizuserver/pkg/servicemap"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	a    = "aService"
+	b    = "bService"
+	Ip   = "127.0.0.1"
+	Port = "80"
+)
+
+var (
+	TCPEntryA = &tapApi.TCP{
+		Name: a,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, a),
+	}
+	TCPEntryB = &tapApi.TCP{
+		Name: b,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, b),
+	}
+)
+
+var ProtocolHttp = &tapApi.Protocol{
+	Name:            "http",
+	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+	Abbreviation:    "HTTP",
+	Macro:           "http",
+	Version:         "1.1",
+	BackgroundColor: "#205cf5",
+	ForegroundColor: "#ffffff",
+	FontSize:        12,
+	ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc2616",
+	Ports:           []string{"80", "443", "8080"},
+	Priority:        0,
+}
+
+type ServiceMapControllerSuite struct {
+	suite.Suite
+
+	c *ServiceMapController
+	w *httptest.ResponseRecorder
+	g *gin.Context
+}
+
+func (s *ServiceMapControllerSuite) SetupTest() {
+	s.c = NewServiceMapController()
+	s.c.service.SetConfig(&shared.MizuAgentConfig{
+		ServiceMap: true,
+	})
+	s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+
+	s.w = httptest.NewRecorder()
+	s.g, _ = gin.CreateTestContext(s.w)
+}
+
+func (s *ServiceMapControllerSuite) TestGetStatus() {
+	assert := s.Assert()
+
+	s.c.Status(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var status servicemap.ServiceMapStatus
+	err := json.Unmarshal(s.w.Body.Bytes(), &status)
+	assert.NoError(err)
+	assert.Equal("enabled", status.Status)
+	assert.Equal(1, status.EntriesProcessedCount)
+	assert.Equal(2, status.NodeCount)
+	assert.Equal(1, status.EdgeCount)
+}
+
+func (s *ServiceMapControllerSuite) TestGet() {
+	assert := s.Assert()
+
+	s.c.Get(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var response servicemap.ServiceMapResponse
+	err := json.Unmarshal(s.w.Body.Bytes(), &response)
+	assert.NoError(err)
+
+	// response status
+	assert.Equal("enabled", response.Status.Status)
+	assert.Equal(1, response.Status.EntriesProcessedCount)
+	assert.Equal(2, response.Status.NodeCount)
+	assert.Equal(1, response.Status.EdgeCount)
+
+	// response nodes
+	aNode := servicemap.ServiceMapNode{
+		Id:    1,
+		Name:  TCPEntryA.IP,
+		Entry: TCPEntryA,
+		Count: 1,
+	}
+	bNode := servicemap.ServiceMapNode{
+		Id:    2,
+		Name:  TCPEntryB.IP,
+		Entry: TCPEntryB,
+		Count: 1,
+	}
+	assert.Contains(response.Nodes, aNode)
+	assert.Contains(response.Nodes, bNode)
+	assert.Len(response.Nodes, 2)
+
+	// response edges
+	assert.Equal([]servicemap.ServiceMapEdge{
+		{
+			Source:      aNode,
+			Destination: bNode,
+			Protocol:    ProtocolHttp,
+			Count:       1,
+		},
+	}, response.Edges)
+}
+
+func (s *ServiceMapControllerSuite) TestGetReset() {
+	assert := s.Assert()
+
+	s.c.Reset(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var status servicemap.ServiceMapStatus
+	err := json.Unmarshal(s.w.Body.Bytes(), &status)
+	assert.NoError(err)
+	assert.Equal("enabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+}
+
+func TestServiceMapControllerSuite(t *testing.T) {
+	suite.Run(t, new(ServiceMapControllerSuite))
+}

agent/pkg/controllers/user_controller.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/shared/logger"
+
+	ory "github.com/ory/kratos-client-go"
 )
 
 func Login(c *gin.Context) {
@@ -25,7 +27,12 @@ func Logout(c *gin.Context) {
 }
 
 func Register(c *gin.Context) {
-	if token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
+	token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context())
+	handleRegistration(token, err, formErrorMessages, c)
+}
+
+func handleRegistration(token *string, err error, formErrorMessages map[string][]ory.UiText, c *gin.Context) {
+	if err != nil {
 		if formErrorMessages != nil {
 			logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err)
 			c.AbortWithStatusJSON(400, formErrorMessages)
@@ -34,6 +41,6 @@ func Register(c *gin.Context) {
 			c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"})
 		}
 	} else {
-		c.JSON(200, gin.H{"token": token})
+		c.JSON(201, gin.H{"token": token})
 	}
 }

agent/pkg/elastic/esClient.go

@@ -0,0 +1,120 @@
+package elastic
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/tap/api"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type client struct {
+	es            *elasticsearch.Client
+	index         string
+	insertedCount int
+}
+
+var instance *client
+var once sync.Once
+
+func GetInstance() *client {
+	once.Do(func() {
+		instance = newClient()
+	})
+	return instance
+}
+
+func (client *client) Configure(config shared.ElasticConfig) {
+	if config.Url == "" || config.User == "" || config.Password == "" {
+		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
+		return
+	}
+	transport := http.DefaultTransport
+	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
+	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
+	cfg := elasticsearch.Config{
+		Addresses: []string{config.Url},
+		Username:  config.User,
+		Password:  config.Password,
+		Transport: transport,
+	}
+
+	es, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		logger.Log.Fatalf("Failed to initialize elastic client %v", err)
+	}
+
+	// Have the client instance return a response
+	res, err := es.Info()
+	if err != nil {
+		logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err)
+	} else {
+		client.es = es
+		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
+		client.insertedCount = 0
+		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
+	}
+	defer res.Body.Close()
+}
+
+func newClient() *client {
+	return &client{
+		es:    nil,
+		index: "",
+	}
+}
+
+type httpEntry struct {
+	Source      *api.TCP               `json:"src"`
+	Destination *api.TCP               `json:"dst"`
+	Outgoing    bool                   `json:"outgoing"`
+	CreatedAt   time.Time              `json:"createdAt"`
+	Request     map[string]interface{} `json:"request"`
+	Response    map[string]interface{} `json:"response"`
+	Summary     string                 `json:"summary"`
+	Method      string                 `json:"method"`
+	Status      int                    `json:"status"`
+	ElapsedTime int64                  `json:"elapsedTime"`
+	Path        string                 `json:"path"`
+}
+
+func (client *client) PushEntry(entry *api.Entry) {
+	if client.es == nil {
+		return
+	}
+
+	if entry.Protocol.Name != "http" {
+		return
+	}
+
+	entryToPush := httpEntry{
+		Source:      entry.Source,
+		Destination: entry.Destination,
+		Outgoing:    entry.Outgoing,
+		CreatedAt:   entry.StartTime,
+		Request:     entry.Request,
+		Response:    entry.Response,
+		Summary:     entry.Summary,
+		Method:      entry.Method,
+		Status:      entry.Status,
+		ElapsedTime: entry.ElapsedTime,
+		Path:        entry.Path,
+	}
+
+	entryJson, err := json.Marshal(entryToPush)
+	if err != nil {
+		logger.Log.Errorf("json.Marshal ERROR: %v", err)
+		return
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString(string(entryJson))
+	res, _ := client.es.Index(client.index, &buffer)
+	if res.StatusCode == 201 {
+		client.insertedCount += 1
+	}
+}

agent/pkg/har/types.go

@@ -0,0 +1,375 @@
+package har
+
+import (
+	"encoding/base64"
+	"github.com/up9inc/mizu/shared/logger"
+	"time"
+	"unicode/utf8"
+)
+
+/*
+HTTP Archive (HAR) format
+https://w3c.github.io/web-performance/specs/HAR/Overview.html
+*/
+
+// HAR is a container type for deserialization
+type HAR struct {
+	Log Log `json:"log"`
+}
+
+// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
+type Log struct {
+	// The object contains the following name/value pairs:
+
+	// Required. Version number of the format.
+	Version string `json:"version"`
+	// Required. An object of type creator that contains the name and version
+	// information of the log creator application.
+	Creator Creator `json:"creator"`
+	// Optional. An object of type browser that contains the name and version
+	// information of the user agent.
+	Browser Browser `json:"browser"`
+	// Optional. An array of objects of type page, each representing one exported
+	// (tracked) page. Leave out this field if the application does not support
+	// grouping by pages.
+	Pages []Page `json:"pages,omitempty"`
+	// Required. An array of objects of type entry, each representing one
+	// exported (tracked) HTTP request.
+	Entries []Entry `json:"entries"`
+	// Optional. A comment provided by the user or the application. Sorting
+	// entries by startedDateTime (starting from the oldest) is preferred way how
+	// to export data since it can make importing faster. However the reader
+	// application should always make sure the array is sorted (if required for
+	// the import).
+	Comment string `json:"comment"`
+}
+
+// Creator contains information about the log creator application
+type Creator struct {
+	// Required. The name of the application that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the application that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Browser that created the log
+type Browser struct {
+	// Required. The name of the browser that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the browser that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the browser.
+	Comment string `json:"comment"`
+}
+
+// Page object for every exported web page and one <entry> object for every HTTP request.
+// In case when an HTTP trace tool isn't able to group requests by a page,
+// the <pages> object is empty and individual requests doesn't have a parent page.
+type Page struct {
+	/* There is one <page> object for every exported web page and one <entry>
+	object for every HTTP request. In case when an HTTP trace tool isn't able to
+	group requests by a page, the <pages> object is empty and individual
+	requests doesn't have a parent page.
+	*/
+
+	// Date and time stamp for the beginning of the page load
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
+	StartedDateTime string `json:"startedDateTime"`
+	// Unique identifier of a page within the . Entries use it to refer the parent page.
+	ID string `json:"id"`
+	// Page title.
+	Title string `json:"title"`
+	// Detailed timing info about page load.
+	PageTiming PageTiming `json:"pageTiming"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTiming describes timings for various events (states) fired during the page load.
+// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
+type PageTiming struct {
+	// Content of the page loaded. Number of milliseconds since page load started
+	// (page.startedDateTime). Use -1 if the timing does not apply to the current
+	// request.
+	// Depeding on the browser, onContentLoad property represents DOMContentLoad
+	// event or document.readyState == interactive.
+	OnContentLoad int `json:"onContentLoad"`
+	// Page is loaded (onLoad event fired). Number of milliseconds since page
+	// load started (page.startedDateTime). Use -1 if the timing does not apply
+	// to the current request.
+	OnLoad int `json:"onLoad"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Entry is a unique, optional Reference to the parent page.
+// Leave out this field if the application does not support grouping by pages.
+type Entry struct {
+	Pageref string `json:"pageref,omitempty"`
+	// Date and time stamp of the request start
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
+	StartedDateTime string `json:"startedDateTime"`
+	// Total elapsed time of the request in milliseconds. This is the sum of all
+	// timings available in the timings object (i.e. not including -1 values) .
+	Time int `json:"time"`
+	// Detailed info about the request.
+	Request Request `json:"request"`
+	// Detailed info about the response.
+	Response Response `json:"response"`
+	// Info about cache usage.
+	Cache Cache `json:"cache"`
+	// Detailed timing info about request/response round trip.
+	PageTimings PageTimings `json:"pageTimings"`
+	// optional (new in 1.2) IP address of the server that was connected
+	// (result of DNS resolution).
+	ServerIPAddress string `json:"serverIPAddress,omitempty"`
+	// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
+	// the client port number. Note that a port number doesn't have to be unique
+	// identifier in cases where the port is shared for more connections. If the
+	// port isn't available for the application, any other unique connection ID
+	// can be used instead (e.g. connection index). Leave out this field if the
+	// application doesn't support this info.
+	Connection string `json:"connection,omitempty"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Request contains detailed info about performed request.
+type Request struct {
+	// Request method (GET, POST, ...).
+	Method string `json:"method"`
+	// Absolute URL of the request (fragments are not included).
+	URL string `json:"url"`
+	// Request HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// List of query parameter objects.
+	QueryString []NVP `json:"queryString"`
+	// Posted data.
+	PostData PostData `json:"postData"`
+	// Total number of bytes from the start of the HTTP request message until
+	// (and including) the double CRLF before the body. Set to -1 if the info
+	// is not available.
+	HeaderSize int `json:"headerSize"`
+	// Size of the request body (POST data payload) in bytes. Set to -1 if the
+	// info is not available.
+	BodySize int `json:"bodySize"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Response contains detailed info about the response.
+type Response struct {
+	// Response status.
+	Status int `json:"status"`
+	// Response status description.
+	StatusText string `json:"statusText"`
+	// Response HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// Details about the response body.
+	Content Content `json:"content"`
+	// Redirection target URL from the Location response header.
+	RedirectURL string `json:"redirectURL"`
+	// Total number of bytes from the start of the HTTP response message until
+	// (and including) the double CRLF before the body. Set to -1 if the info is
+	// not available.
+	// The size of received response-headers is computed only from headers that
+	// are really received from the server. Additional headers appended by the
+	// browser are not included in this number, but they appear in the list of
+	// header objects.
+	HeadersSize int `json:"headersSize"`
+	// Size of the received response body in bytes. Set to zero in case of
+	// responses coming from the cache (304). Set to -1 if the info is not
+	// available.
+	BodySize int `json:"bodySize"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Cookie contains list of all cookies (used in <request> and <response> objects).
+type Cookie struct {
+	// The name of the cookie.
+	Name string `json:"name"`
+	// The cookie value.
+	Value string `json:"value"`
+	// optional The path pertaining to the cookie.
+	Path string `json:"path,omitempty"`
+	// optional The host of the cookie.
+	Domain string `json:"domain,omitempty"`
+	// optional Cookie expiration time.
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
+	Expires string `json:"expires,omitempty"`
+	// optional Set to true if the cookie is HTTP only, false otherwise.
+	HTTPOnly bool `json:"httpOnly,omitempty"`
+	// optional (new in 1.2) True if the cookie was transmitted over ssl, false
+	// otherwise.
+	Secure bool `json:"secure,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// NVP is simply a name/value pair with a comment
+type NVP struct {
+	Name    string `json:"name"`
+	Value   string `json:"value"`
+	Comment string `json:"comment,omitempty"`
+}
+
+// PostData describes posted data, if any (embedded in <request> object).
+type PostData struct {
+	//  Mime type of posted data.
+	MimeType string `json:"mimeType"`
+	//  List of posted parameters (in case of URL encoded parameters).
+	Params []PostParam `json:"params"`
+	//  Plain text posted data
+	Text string `json:"text"`
+	// optional (new in 1.2) A comment provided by the user or the
+	// application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (d PostData) B64Decoded() (bool, []byte, string) {
+	// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
+	// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
+	return b64Decoded(d.Comment, d.Text)
+}
+
+// PostParam is a list of posted parameters, if any (embedded in <postData> object).
+type PostParam struct {
+	// name of a posted parameter.
+	Name string `json:"name"`
+	// optional value of a posted parameter or content of a posted file.
+	Value string `json:"value,omitempty"`
+	// optional name of a posted file.
+	FileName string `json:"fileName,omitempty"`
+	// optional content type of a posted file.
+	ContentType string `json:"contentType,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Content describes details about response content (embedded in <response> object).
+type Content struct {
+	// Length of the returned content in bytes. Should be equal to
+	// response.bodySize if there is no compression and bigger when the content
+	// has been compressed.
+	Size int `json:"size"`
+	// optional Number of bytes saved. Leave out this field if the information
+	// is not available.
+	Compression int `json:"compression,omitempty"`
+	// MIME type of the response text (value of the Content-Type response
+	// header). The charset attribute of the MIME type is included (if
+	// available).
+	MimeType string `json:"mimeType"`
+	// optional Response body sent from the server or loaded from the browser
+	// cache. This field is populated with textual content only. The text field
+	// is either HTTP decoded text or a encoded (e.g. "base64") representation of
+	// the response body. Leave out this field if the information is not
+	// available.
+	Text string `json:"text,omitempty"`
+	// optional (new in 1.2) Encoding used for response text field e.g
+	// "base64". Leave out this field if the text field is HTTP decoded
+	// (decompressed & unchunked), than trans-coded from its original character
+	// set into UTF-8.
+	Encoding string `json:"encoding,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (c Content) B64Decoded() (bool, []byte, string) {
+	return b64Decoded(c.Encoding, c.Text)
+}
+
+func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
+	if enc == "base64" {
+		decoded, err := base64.StdEncoding.DecodeString(text)
+		if err != nil {
+			logger.Log.Warningf("Failed to decode content as base64: %s", text)
+			return false, []byte(text), text
+		}
+		valid := utf8.Valid(decoded)
+		return !valid, decoded, string(decoded)
+	} else {
+		return false, nil, text
+	}
+}
+
+// Cache contains info about a request coming from browser cache.
+type Cache struct {
+	// optional State of a cache entry before the request. Leave out this field
+	// if the information is not available.
+	BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
+	// optional State of a cache entry after the request. Leave out this field if
+	// the information is not available.
+	AfterRequest CacheObject `json:"afterRequest,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// CacheObject is used by both beforeRequest and afterRequest
+type CacheObject struct {
+	// optional - Expiration time of the cache entry.
+	Expires string `json:"expires,omitempty"`
+	// The last time the cache entry was opened.
+	LastAccess string `json:"lastAccess"`
+	// Etag
+	ETag string `json:"eTag"`
+	// The number of times the cache entry has been opened.
+	HitCount int `json:"hitCount"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTimings describes various phases within request-response round trip.
+// All times are specified in milliseconds.
+type PageTimings struct {
+	Blocked int `json:"blocked,omitempty"`
+	// optional - Time spent in a queue waiting for a network connection. Use -1
+	// if the timing does not apply to the current request.
+	DNS int `json:"dns,omitempty"`
+	// optional - DNS resolution time. The time required to resolve a host name.
+	// Use -1 if the timing does not apply to the current request.
+	Connect int `json:"connect,omitempty"`
+	// optional - Time required to create TCP connection. Use -1 if the timing
+	// does not apply to the current request.
+	Send int `json:"send"`
+	// Time required to send HTTP request to the server.
+	Wait int `json:"wait"`
+	// Waiting for a response from the server.
+	Receive int `json:"receive"`
+	// Time required to read entire response from the server (or cache).
+	Ssl int `json:"ssl,omitempty"`
+	// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
+	// field is defined then the time is also included in the connect field (to
+	// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
+	// apply to the current request.
+	Comment string `json:"comment,omitempty"`
+	// optional (new in 1.2) - A comment provided by the user or the application.
+}
+
+// TestResult contains results for an individual HTTP request
+type TestResult struct {
+	URL       string    `json:"url"`
+	Status    int       `json:"status"` // 200, 500, etc.
+	StartTime time.Time `json:"startTime"`
+	EndTime   time.Time `json:"endTime"`
+	Latency   int       `json:"latency"` // milliseconds
+	Method    string    `json:"method"`
+	HarFile   string    `json:"harfile"`
+}
+
+// aliases for martian lib compatibility
+
+type Header = NVP
+type QueryString = NVP
+type Param = PostParam
+type Timings = PageTimings

agent/pkg/har/types_test.go

@@ -0,0 +1,38 @@
+package har
+
+import "testing"
+
+func TestContentEncoded(t *testing.T) {
+	testCases := []struct {
+		text        string
+		isBinary    bool
+		expectedStr string
+		binaryLen   int
+	}{
+		{"not-base64", false, "not-base64", 10},
+		{"dGVzdA==", false, "test", 4},
+		{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
+		{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+	}
+
+	for _, tc := range testCases {
+		c := Content{
+			Encoding: "base64",
+			Text:     tc.text,
+		}
+		isBinary, asBytes, asString := c.B64Decoded()
+		_ = asBytes
+
+		if tc.isBinary != isBinary {
+			t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
+		}
+
+		if !isBinary && tc.expectedStr != asString {
+			t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
+		}
+
+		if tc.binaryLen != len(asBytes) {
+			t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
+		}
+	}
+}

agent/pkg/har/utils.go

@@ -1,4 +1,4 @@
-package utils
+package har
 
 import (
 	"bytes"
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
 )
 
@@ -55,14 +54,14 @@ import (
 //	return cookies
 //}
 
-func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) {
+func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) {
 	var host, scheme, authority, path, status string
-	headers := make([]har.Header, 0, len(rawHeaders))
+	headers := make([]Header, 0, len(rawHeaders))
 
 	for _, header := range rawHeaders {
 		h := header.(map[string]interface{})
 
-		headers = append(headers, har.Header{
+		headers = append(headers, Header{
 			Name:  h["name"].(string),
 			Value: h["value"].(string),
 		})
@@ -87,8 +86,8 @@ func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, strin
 	return headers, host, scheme, authority, path, status
 }
 
-func BuildPostParams(rawParams []interface{}) []har.Param {
-	params := make([]har.Param, 0, len(rawParams))
+func BuildPostParams(rawParams []interface{}) []Param {
+	params := make([]Param, 0, len(rawParams))
 	for _, param := range rawParams {
 		p := param.(map[string]interface{})
 		name := ""
@@ -108,10 +107,10 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 			contentType = p["contentType"].(string)
 		}
 
-		params = append(params, har.Param{
+		params = append(params, Param{
 			Name:        name,
 			Value:       value,
-			Filename:    fileName,
+			FileName:    fileName,
 			ContentType: contentType,
 		})
 	}
@@ -119,9 +118,9 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 	return params
 }
 
-func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
+func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
 	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType, _ := postData["mimeType"]
@@ -134,10 +133,10 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 		postDataText = text.(string)
 	}
 
-	queryString := make([]har.QueryString, 0)
+	queryString := make([]QueryString, 0)
 	for _, _qs := range request["_queryString"].([]interface{}) {
 		qs := _qs.(map[string]interface{})
-		queryString = append(queryString, har.QueryString{
+		queryString = append(queryString, QueryString{
 			Name:  qs["name"].(string),
 			Value: qs["value"].(string),
 		})
@@ -148,21 +147,21 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
 	}
 
-	harParams := make([]har.Param, 0)
+	harParams := make([]Param, 0)
 	if postData["params"] != nil {
 		harParams = BuildPostParams(postData["params"].([]interface{}))
 	}
 
-	harRequest = &har.Request{
+	harRequest = &Request{
 		Method:      request["method"].(string),
 		URL:         url,
 		HTTPVersion: request["httpVersion"].(string),
-		HeadersSize: -1,
-		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
+		HeaderSize:  -1,
+		BodySize:    bytes.NewBufferString(postDataText).Len(),
 		QueryString: queryString,
 		Headers:     headers,
 		Cookies:     cookies,
-		PostData: &har.PostData{
+		PostData: PostData{
 			MimeType: mimeType.(string),
 			Params:   harParams,
 			Text:     postDataText,
@@ -172,9 +171,9 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 	return
 }
 
-func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
+func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
 	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType, _ := content["mimeType"]
@@ -188,11 +187,11 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er
 		bodyText = text.(string)
 	}
 
-	harContent := &har.Content{
+	harContent := &Content{
 		Encoding: encoding.(string),
 		MimeType: mimeType.(string),
-		Text:     []byte(bodyText),
-		Size:     int64(len(bodyText)),
+		Text:     bodyText,
+		Size:     len(bodyText),
 	}
 
 	status := int(response["status"].(float64))
@@ -206,20 +205,20 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er
 		}
 	}
 
-	harResponse = &har.Response{
+	harResponse = &Response{
 		HTTPVersion: response["httpVersion"].(string),
 		Status:      status,
 		StatusText:  response["statusText"].(string),
 		HeadersSize: -1,
-		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
+		BodySize:    bytes.NewBufferString(bodyText).Len(),
 		Headers:     headers,
 		Cookies:     cookies,
-		Content:     harContent,
+		Content:     *harContent,
 	}
 	return
 }
 
-func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
+func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) {
 	harRequest, err := NewRequest(request)
 	if err != nil {
 		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
@@ -236,16 +235,16 @@ func NewEntry(request map[string]interface{}, response map[string]interface{}, s
 		elapsedTime = 1
 	}
 
-	harEntry := har.Entry{
-		StartedDateTime: startTime,
-		Time:            elapsedTime,
-		Request:         harRequest,
-		Response:        harResponse,
-		Cache:           &har.Cache{},
-		Timings: &har.Timings{
+	harEntry := Entry{
+		StartedDateTime: startTime.Format(time.RFC3339),
+		Time:            int(elapsedTime),
+		Request:         *harRequest,
+		Response:        *harResponse,
+		Cache:           Cache{},
+		PageTimings: PageTimings{
 			Send:    -1,
 			Wait:    -1,
-			Receive: elapsedTime,
+			Receive: int(elapsedTime),
 		},
 	}
 

agent/pkg/middlewares/auth.go

@@ -0,0 +1,73 @@
+package middlewares
+
+import (
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/providers"
+
+	"github.com/gin-gonic/gin"
+	ory "github.com/ory/kratos-client-go"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func RequiresAuth() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// auth is irrelevant for ephermeral mizu
+		if !config.Config.StandaloneMode {
+			c.Next()
+			return
+		}
+
+		verifyKratosSessionForRequest(c)
+		if !c.IsAborted() {
+			c.Next()
+		}
+	}
+}
+
+func RequiresAdmin() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// auth is irrelevant for ephermeral mizu
+		if !config.Config.StandaloneMode {
+			c.Next()
+			return
+		}
+
+		session := verifyKratosSessionForRequest(c)
+		if c.IsAborted() {
+			return
+		}
+
+		traits := session.Identity.Traits.(map[string]interface{})
+		username := traits["username"].(string)
+
+		isAdmin, err := providers.CheckIfUserHasSystemRole(username, providers.AdminRole)
+		if err != nil {
+			logger.Log.Errorf("error checking user role %v", err)
+			c.AbortWithStatusJSON(403, gin.H{"error": "unknown auth error occured"})
+		} else if !isAdmin {
+			logger.Log.Warningf("user %s attempted to call an admin only endpoint with insufficient privileges", username)
+			c.AbortWithStatusJSON(403, gin.H{"error": "unauthorized"})
+		} else {
+			c.Next()
+		}
+	}
+}
+
+func verifyKratosSessionForRequest(c *gin.Context) *ory.Session {
+	token := c.GetHeader("x-session-token")
+	if token == "" {
+		c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
+		return nil
+	}
+
+	if session, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
+		logger.Log.Errorf("error verifying token %v", err)
+		c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
+		return nil
+	} else if session == nil {
+		c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
+		return nil
+	} else {
+		return session
+	}
+}

agent/pkg/middlewares/requiresAuth.go

@@ -1,49 +0,0 @@
-package middlewares
-
-import (
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/providers"
-	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/patrickmn/go-cache"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-const cachedValidTokensRetainmentTime = time.Minute * 1
-
-var cachedValidTokens = cache.New(cachedValidTokensRetainmentTime, cachedValidTokensRetainmentTime)
-
-func RequiresAuth() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		// auth is irrelevant for ephermeral mizu
-		if !config.Config.StandaloneMode {
-			c.Next()
-			return
-		}
-
-		token := c.GetHeader("x-session-token")
-		if token == "" {
-			c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
-			return
-		}
-
-		if _, isTokenCached := cachedValidTokens.Get(token); isTokenCached {
-			c.Next()
-			return
-		}
-
-		if isTokenValid, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
-			logger.Log.Errorf("error verifying token %s", err)
-			c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
-			return
-		} else if !isTokenValid {
-			c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
-			return
-		}
-
-		cachedValidTokens.Set(token, true, cachedValidTokensRetainmentTime)
-
-		c.Next()
-	}
-}

agent/pkg/models/models.go

@@ -2,11 +2,10 @@ package models
 
 import (
 	"encoding/json"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/rules"
 
-	tapApi "github.com/up9inc/mizu/tap/api"
-
-	"github.com/google/martian/har"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"

agent/pkg/oas/feeder_test.go

@@ -4,10 +4,10 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
 	"io"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"os"
 	"path/filepath"
 	"sort"
@@ -51,32 +51,42 @@ func fileSize(fname string) int64 {
 	return fi.Size()
 }
 
-func feedEntries(fromFiles []string) (err error) {
+func feedEntries(fromFiles []string, isSync bool) (count int, err error) {
+	badFiles := make([]string, 0)
+	cnt := 0
 	for _, file := range fromFiles {
 		logger.Log.Info("Processing file: " + file)
 		ext := strings.ToLower(filepath.Ext(file))
+		eCnt := 0
 		switch ext {
 		case ".har":
-			err = feedFromHAR(file)
+			eCnt, err = feedFromHAR(file, isSync)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
 				continue
 			}
 		case ".ldjson":
-			err = feedFromLDJSON(file)
+			eCnt, err = feedFromLDJSON(file, isSync)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
 				continue
 			}
 		default:
-			return errors.New("Unsupported file extension: " + ext)
+			return 0, errors.New("Unsupported file extension: " + ext)
 		}
+		cnt += eCnt
 	}
 
-	return nil
+	for _, f := range badFiles {
+		logger.Log.Infof("Bad file: %s", f)
+	}
+
+	return cnt, nil
 }
 
-func feedFromHAR(file string) error {
+func feedFromHAR(file string, isSync bool) (int, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -86,23 +96,43 @@ func feedFromHAR(file string) error {
 
 	data, err := ioutil.ReadAll(fd)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
 	var harDoc har.HAR
 	err = json.Unmarshal(data, &harDoc)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
+	cnt := 0
 	for _, entry := range harDoc.Log.Entries {
-		GetOasGeneratorInstance().PushEntry(entry)
+		cnt += 1
+		feedEntry(&entry, isSync)
 	}
 
-	return nil
+	return cnt, nil
 }
 
-func feedFromLDJSON(file string) error {
+func feedEntry(entry *har.Entry, isSync bool) {
+	if entry.Response.Status == 302 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+	}
+
+	if strings.Contains(entry.Request.URL, "taboola") {
+		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
+	} else {
+		//return
+	}
+
+	if isSync {
+		GetOasGeneratorInstance().entriesChan <- *entry // blocking variant, right?
+	} else {
+		GetOasGeneratorInstance().PushEntry(entry)
+	}
+}
+
+func feedFromLDJSON(file string, isSync bool) (int, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -113,8 +143,8 @@ func feedFromLDJSON(file string) error {
 	reader := bufio.NewReader(fd)
 
 	var meta map[string]interface{}
-
 	buf := strings.Builder{}
+	cnt := 0
 	for {
 		substr, isPrefix, err := reader.ReadLine()
 		if err == io.EOF {
@@ -132,26 +162,28 @@ func feedFromLDJSON(file string) error {
 		if meta == nil {
 			err := json.Unmarshal([]byte(line), &meta)
 			if err != nil {
-				return err
+				return 0, err
 			}
 		} else {
 			var entry har.Entry
 			err := json.Unmarshal([]byte(line), &entry)
 			if err != nil {
 				logger.Log.Warningf("Failed decoding entry: %s", line)
+			} else {
+				cnt += 1
+				feedEntry(&entry, isSync)
 			}
-			GetOasGeneratorInstance().PushEntry(&entry)
 		}
 	}
 
-	return nil
+	return cnt, nil
 }
 
 func TestFilesList(t *testing.T) {
 	res, err := getFiles("./test_artifacts/")
 	t.Log(len(res))
 	t.Log(res)
-	if err != nil || len(res) != 2 {
+	if err != nil || len(res) != 3 {
 		t.Logf("Should return 2 files but returned %d", len(res))
 		t.FailNow()
 	}

agent/pkg/oas/gibberish.go

@@ -12,6 +12,7 @@ var (
 	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
 	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
 	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
+	patLongNum  = regexp.MustCompile(`\d{6,}`)
 )
 
 func IsGibberish(str string) bool {
@@ -27,7 +28,7 @@ func IsGibberish(str string) bool {
 		return true
 	}
 
-	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
+	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) || patLongNum.MatchString(str) {
 		return true
 	}
 

agent/pkg/oas/ignores.go

@@ -17,17 +17,19 @@ var ignoredHeaders = []string{
 	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
 	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
 	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
-	"x-wap-profile", "x-scheme",
-	"newrelic", "x-cloud-trace-context", "sentry-trace",
-	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
-	"last-modified", "content-language",
+	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
+	"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
+	"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
 	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
-	"age", "allow", "www-authenticate",
+	"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
+	"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
+	"duration_ms", // UP9 custom
 }
 
 var ignoredHeaderPrefixes = []string{
 	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
-	"x-forwarded-", "x-original-",
+	"x-forwarded-", "x-original-", "cf-",
 	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
 	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
 }

agent/pkg/oas/oas_generator.go

@@ -3,8 +3,8 @@ package oas
 import (
 	"context"
 	"encoding/json"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/har"
 	"net/url"
 	"sync"
 )
@@ -17,7 +17,7 @@ var (
 func GetOasGeneratorInstance() *oasGenerator {
 	syncOnce.Do(func() {
 		instance = newOasGenerator()
-		logger.Log.Debug("Oas Generator Initialized")
+		logger.Log.Debug("OAS Generator Initialized")
 	})
 	return instance
 }

agent/pkg/oas/specgen.go

@@ -1,14 +1,13 @@
 package oas
 
 import (
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
 	"github.com/google/uuid"
 	"github.com/up9inc/mizu/shared/logger"
 	"mime"
+	"mizuserver/pkg/har"
 	"net/url"
 	"strconv"
 	"strings"
@@ -31,7 +30,7 @@ func NewGen(server string) *SpecGen {
 	spec.Version = "3.1.0"
 
 	info := openapi.Info{Title: server}
-	info.Version = "0.0"
+	info.Version = "1.0"
 	spec.Info = &info
 	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
 
@@ -176,11 +175,18 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 
 	if isExtIgnored(urlParsed.Path) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+		return "", nil
 	}
 
-	ctype := getRespCtype(entry.Response)
+	if entry.Request.Method == "OPTIONS" {
+		logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path)
+		return "", nil
+	}
+
+	ctype := getRespCtype(&entry.Response)
 	if isCtypeIgnored(ctype) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+		return "", nil
 	}
 
 	if entry.Response.Status < 100 {
@@ -193,9 +199,19 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 		return "", nil
 	}
 
-	split := strings.Split(urlParsed.Path, "/")
+	if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
+		logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	var split []string
+	if urlParsed.RawPath != "" {
+		split = strings.Split(urlParsed.RawPath, "/")
+	} else {
+		split = strings.Split(urlParsed.Path, "/")
+	}
 	node := g.tree.getOrSet(split, new(openapi.PathObj))
-	opObj, err := handleOpObj(entry, node.ops)
+	opObj, err := handleOpObj(entry, node.pathObj)
 
 	if opObj != nil {
 		return opObj.OperationID, err
@@ -216,12 +232,12 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation
 		return nil, nil
 	}
 
-	err = handleRequest(entry.Request, opObj, isSuccess)
+	err = handleRequest(&entry.Request, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handleResponse(entry.Response, opObj, isSuccess)
+	err = handleResponse(&entry.Response, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
@@ -233,26 +249,22 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
 
 	qstrGW := nvParams{
-		In: openapi.InQuery,
-		Pairs: func() []NVPair {
-			return qstrToNVP(req.QueryString)
-		},
+		In:             openapi.InQuery,
+		Pairs:          req.QueryString,
 		IsIgnored:      func(name string) bool { return false },
 		GeneralizeName: func(name string) string { return name },
 	}
 	handleNameVals(qstrGW, &opObj.Parameters)
 
 	hdrGW := nvParams{
-		In: openapi.InHeader,
-		Pairs: func() []NVPair {
-			return hdrToNVP(req.Headers)
-		},
+		In:             openapi.InHeader,
+		Pairs:          req.Headers,
 		IsIgnored:      isHeaderIgnored,
 		GeneralizeName: strings.ToLower,
 	}
 	handleNameVals(hdrGW, &opObj.Parameters)
 
-	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+	if req.PostData.Text != "" && isSuccess {
 		reqBody, err := getRequestBody(req, opObj, isSuccess)
 		if err != nil {
 			return err
@@ -342,43 +354,34 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err
 	}
 
 	var text string
+	var isBinary bool
 	if reqResp.Req != nil {
-		text = reqResp.Req.PostData.Text
+		isBinary, _, text = reqResp.Req.PostData.B64Decoded()
 	} else {
-		text = decRespText(reqResp.Resp.Content)
+		isBinary, _, text = reqResp.Resp.Content.B64Decoded()
 	}
 
-	var exampleMsg []byte
-	// try treating it as json
-	any, isJSON := anyJSON(text)
-	if isJSON {
-		// re-marshal with forced indent
-		exampleMsg, err = json.MarshalIndent(any, "", "\t")
-		if err != nil {
-			panic("Failed to re-marshal value, super-strange")
-		}
-	} else {
-		exampleMsg, err = json.Marshal(text)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	content.Example = exampleMsg
-	return respContent[ctype], nil
-}
-
-func decRespText(content *har.Content) (res string) {
-	res = string(content.Text)
-	if content.Encoding == "base64" {
-		data, err := base64.StdEncoding.DecodeString(res)
-		if err != nil {
-			logger.Log.Warningf("error decoding response text as base64: %s", err)
+	if !isBinary && text != "" {
+		var exampleMsg []byte
+		// try treating it as json
+		any, isJSON := anyJSON(text)
+		if isJSON {
+			// re-marshal with forced indent
+			exampleMsg, err = json.MarshalIndent(any, "", "\t")
+			if err != nil {
+				panic("Failed to re-marshal value, super-strange")
+			}
 		} else {
-			res = string(data)
+			exampleMsg, err = json.Marshal(text)
+			if err != nil {
+				return nil, err
+			}
 		}
+
+		content.Example = exampleMsg
 	}
-	return
+
+	return respContent[ctype], nil
 }
 
 func getRespCtype(resp *har.Response) string {

agent/pkg/oas/specgen_test.go

@@ -3,48 +3,56 @@ package oas
 import (
 	"encoding/json"
 	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared/logger"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"os"
 	"strings"
 	"testing"
+	"time"
 )
 
 // if started via env, write file into subdir
-func writeFiles(label string, spec *openapi.OpenAPI) {
+func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) {
+	content, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		panic(err)
+	}
+
 	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
 		path := "./oas-samples"
 		err := os.MkdirAll(path, 0o755)
 		if err != nil {
 			panic(err)
 		}
-		content, err := json.MarshalIndent(spec, "", "\t")
-		if err != nil {
-			panic(err)
-		}
 		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
 		if err != nil {
 			panic(err)
 		}
+		t.Logf("Written: %s", label)
+	} else {
+		t.Logf("%s", string(content))
 	}
 }
 
 func TestEntries(t *testing.T) {
+	logger.InitLoggerStderrOnly(logging.INFO)
 	files, err := getFiles("./test_artifacts/")
-	// files, err = getFiles("/media/bigdisk/UP9")
 	if err != nil {
 		t.Log(err)
 		t.FailNow()
 	}
 	GetOasGeneratorInstance().Start()
+	loadStartingOAS()
 
-	if err := feedEntries(files); err != nil {
+	cnt, err := feedEntries(files, true)
+	if err != nil {
 		t.Log(err)
 		t.Fail()
 	}
 
-	loadStartingOAS()
+	waitQueueProcessed()
 
 	svcs := strings.Builder{}
 	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
@@ -78,33 +86,35 @@ func TestEntries(t *testing.T) {
 			t.FailNow()
 		}
 
-		specText, _ := json.MarshalIndent(spec, "", "\t")
-		t.Logf("%s", string(specText))
+		outputSpec(svc, spec, t)
 
 		err = spec.Validate()
 		if err != nil {
 			t.Log(err)
 			t.FailNow()
 		}
-		writeFiles(svc, spec)
 
 		return true
 	})
 
+	logger.Log.Infof("Total entries: %d", cnt)
 }
 
-func TestFileLDJSON(t *testing.T) {
+func TestFileSingle(t *testing.T) {
 	GetOasGeneratorInstance().Start()
-	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
-	err := feedFromLDJSON(file)
+	// loadStartingOAS()
+	file := "test_artifacts/params.har"
+	files := []string{file}
+	cnt, err := feedEntries(files, true)
 	if err != nil {
 		logger.Log.Warning("Failed processing file: " + err.Error())
 		t.Fail()
 	}
 
-	loadStartingOAS()
+	waitQueueProcessed()
 
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
 		gen := val.(*SpecGen)
 		spec, err := gen.GetSpec()
 		if err != nil {
@@ -112,8 +122,7 @@ func TestFileLDJSON(t *testing.T) {
 			t.FailNow()
 		}
 
-		specText, _ := json.MarshalIndent(spec, "", "\t")
-		t.Logf("%s", string(specText))
+		outputSpec(svc, spec, t)
 
 		err = spec.Validate()
 		if err != nil {
@@ -123,6 +132,19 @@ func TestFileLDJSON(t *testing.T) {
 
 		return true
 	})
+
+	logger.Log.Infof("Processed entries: %d", cnt)
+}
+
+func waitQueueProcessed() {
+	for {
+		time.Sleep(100 * time.Millisecond)
+		queue := len(GetOasGeneratorInstance().entriesChan)
+		logger.Log.Infof("Queue: %d", queue)
+		if queue < 1 {
+			break
+		}
+	}
 }
 
 func loadStartingOAS() {
@@ -155,20 +177,29 @@ func loadStartingOAS() {
 
 func TestEntriesNegative(t *testing.T) {
 	files := []string{"invalid"}
-	err := feedEntries(files)
+	_, err := feedEntries(files, false)
 	if err == nil {
 		t.Logf("Should have failed")
 		t.Fail()
 	}
 }
 
+func TestEntriesPositive(t *testing.T) {
+	files := []string{"test_artifacts/params.har"}
+	_, err := feedEntries(files, false)
+	if err != nil {
+		t.Logf("Failed")
+		t.Fail()
+	}
+}
+
 func TestLoadValidHAR(t *testing.T) {
 	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
 	var entry *har.Entry
 	var err = json.Unmarshal([]byte(inp), &entry)
 	if err != nil {
 		t.Logf("Failed to decode entry: %s", err)
-		// t.FailNow() demonstrates the problem of library
+		t.FailNow() // demonstrates the problem of `martian` HAR library
 	}
 }
 

agent/pkg/oas/test_artifacts/example.har

@@ -13,10 +13,14 @@
         "time": 1022,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/proxies/",
+          "url": "http://trcc-api-service/proxies;matrixparam=example/",
           "httpVersion": "",
           "cookies": [],
           "headers": [
+            {
+              "name": "X-Custom-Demo-Header",
+              "value": "demo"
+            },
             {
               "name": "Sec-Fetch-Dest",
               "value": "empty"
@@ -124,6 +128,10 @@
           "httpVersion": "",
           "cookies": [],
           "headers": [
+            {
+              "name": "X-Custom-Demo-Header2",
+              "value": "demo2"
+            },
             {
               "name": "Vary",
               "value": "Origin"
@@ -24568,7 +24576,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 200,
+          "status": 308,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],
@@ -24635,7 +24643,7 @@
         "time": 1,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs",
+          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs.png",
           "httpVersion": "",
           "cookies": [],
           "headers": [
@@ -24894,7 +24902,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 200,
+          "status": 0,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],

agent/pkg/oas/test_artifacts/params.har

@@ -0,0 +1,127 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": {
+      "name": "mitmproxy har_dump",
+      "version": "0.1",
+      "comment": "mitmproxy version mitmproxy 4.0.4"
+    },
+    "entries": [
+      {
+        "startedDateTime": "2019-09-06T06:14:43.864529+00:00",
+        "time": 111,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "headersSize": 1542,
+          "bodySize": 0,
+          "queryString": []
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "mimeType": "text/html",
+            "text": "",
+            "size": 0
+          },
+          "redirectURL": "",
+          "headersSize": 245,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 22,
+          "receive": 2,
+          "wait": 87,
+          "connect": -1,
+          "ssl": -1
+        },
+        "serverIPAddress": "54.210.29.33"
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:18.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:19.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      }
+    ]
+  }
+}

agent/pkg/oas/tree.go

@@ -3,6 +3,7 @@ package oas
 import (
 	"github.com/chanced/openapi"
 	"github.com/up9inc/mizu/shared/logger"
+	"net/url"
 	"strconv"
 	"strings"
 )
@@ -10,25 +11,37 @@ import (
 type NodePath = []string
 
 type Node struct {
-	constant *string
-	param    *openapi.ParameterObj
-	ops      *openapi.PathObj
-	parent   *Node
-	children []*Node
+	constant  *string
+	pathParam *openapi.ParameterObj
+	pathObj   *openapi.PathObj
+	parent    *Node
+	children  []*Node
 }
 
-func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
-	if pathObjToSet == nil {
+func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *Node) {
+	if existingPathObj == nil {
 		panic("Invalid function call")
 	}
 
 	pathChunk := path[0]
+	potentialMatrix := strings.SplitN(pathChunk, ";", 2)
+	if len(potentialMatrix) > 1 {
+		pathChunk = potentialMatrix[0]
+		logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
+	}
+
 	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	pathChunk, err := url.PathUnescape(pathChunk)
+	if err != nil {
+		logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
+		// any side effects on continuing?
+	}
+
 	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
 
 	var paramObj *openapi.ParameterObj
-	if chunkIsParam && pathObjToSet != nil {
-		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
+		_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
 	}
 
 	if paramObj == nil {
@@ -46,34 +59,30 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
 		n.children = append(n.children, node)
 
 		if paramObj != nil {
-			node.param = paramObj
+			node.pathParam = paramObj
 		} else if chunkIsGibberish {
-			initParams(&pathObjToSet.Parameters)
 
 			newParam := n.createParam()
-			node.param = newParam
-
-			appended := append(*pathObjToSet.Parameters, newParam)
-			pathObjToSet.Parameters = &appended
+			node.pathParam = newParam
 		} else {
 			node.constant = &pathChunk
 		}
 	}
 
-	// add example if it's a param
-	if node.param != nil && !chunkIsParam {
-		exmp := &node.param.Examples
+	// add example if it's a gibberish chunk
+	if node.pathParam != nil && !chunkIsParam {
+		exmp := &node.pathParam.Examples
 		err := fillParamExample(&exmp, pathChunk)
 		if err != nil {
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
 	}
 
-	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
+	// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
 	if len(path) > 1 {
-		return node.getOrSet(path[1:], pathObjToSet)
-	} else if node.ops == nil {
-		node.ops = pathObjToSet
+		return node.getOrSet(path[1:], existingPathObj)
+	} else if node.pathObj == nil {
+		node.pathObj = existingPathObj
 	}
 
 	return node
@@ -82,18 +91,24 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
 func (n *Node) createParam() *openapi.ParameterObj {
 	name := "param"
 
-	// REST assumption, not always correct
-	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
-		name = *n.constant
-		name = name[:len(name)-2] + "Id"
-	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
-		name = *n.constant
-		name = name[:len(name)-1] + "Id"
+	if n.constant != nil { // the node is already a param
+		// REST assumption, not always correct
+		if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+			name = *n.constant
+			name = name[:len(name)-2] + "Id"
+		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+			name = *n.constant
+			name = name[:len(name)-1] + "Id"
+		} else if isAlpha(*n.constant) {
+			name = *n.constant + "Id"
+		}
+
+		name = cleanNonAlnum([]byte(name))
 	}
 
 	newParam := createSimpleParam(name, "path", "string")
 	x := n.countParentParams()
-	if x > 1 {
+	if x > 0 {
 		newParam.Name = newParam.Name + strconv.Itoa(x)
 	}
 
@@ -111,7 +126,7 @@ func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish b
 			// TODO: check the regex pattern of param? for exceptions etc
 
 			if paramObj != nil {
-				// TODO: mergeParam(subnode.param, paramObj)
+				// TODO: mergeParam(subnode.pathParam, paramObj)
 				return subnode
 			} else {
 				return subnode
@@ -145,15 +160,16 @@ func (n *Node) listPaths() *openapi.Paths {
 	var strChunk string
 	if n.constant != nil {
 		strChunk = *n.constant
-	} else if n.param != nil {
-		strChunk = "{" + n.param.Name + "}"
+	} else if n.pathParam != nil {
+		strChunk = "{" + n.pathParam.Name + "}"
 	} else {
 		// this is the root node
 	}
 
 	// add self
-	if n.ops != nil {
-		paths.Items[openapi.PathValue(strChunk)] = n.ops
+	if n.pathObj != nil {
+		fillPathParams(n, n.pathObj)
+		paths.Items[openapi.PathValue(strChunk)] = n.pathObj
 	}
 
 	// recurse into children
@@ -173,6 +189,29 @@ func (n *Node) listPaths() *openapi.Paths {
 	return paths
 }
 
+func fillPathParams(n *Node, pathObj *openapi.PathObj) {
+	// collect all path parameters from parent hierarchy
+	node := n
+	for {
+		if node.pathParam != nil {
+			initParams(&pathObj.Parameters)
+
+			idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
+			if paramObj == nil {
+				appended := append(*pathObj.Parameters, node.pathParam)
+				pathObj.Parameters = &appended
+			} else {
+				(*pathObj.Parameters)[idx] = paramObj
+			}
+		}
+
+		node = node.parent
+		if node == nil {
+			break
+		}
+	}
+}
+
 type PathAndOp struct {
 	path string
 	op   *openapi.Operation
@@ -192,7 +231,7 @@ func (n *Node) countParentParams() int {
 	res := 0
 	node := n
 	for {
-		if node.param != nil {
+		if node.pathParam != nil {
 			res++
 		}
 

agent/pkg/oas/tree_test.go

@@ -0,0 +1,37 @@
+package oas
+
+import (
+	"github.com/chanced/openapi"
+	"strings"
+	"testing"
+)
+
+func TestTree(t *testing.T) {
+	testCases := []struct {
+		inp       string
+		numParams int
+		label     string
+	}{
+		{"/", 0, ""},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
+		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, ""},
+	}
+
+	tree := new(Node)
+	for _, tc := range testCases {
+		split := strings.Split(tc.inp, "/")
+		pathObj := new(openapi.PathObj)
+		node := tree.getOrSet(split, pathObj)
+		
+		fillPathParams(node, pathObj)
+
+		if node.constant != nil && *node.constant != tc.label {
+			t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
+		}
+
+		if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
+			t.Errorf("Wrong num of params, expected: %d", tc.numParams)
+		}
+	}
+}

agent/pkg/oas/utils.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/har"
 	"strconv"
 	"strings"
 )
@@ -71,9 +71,10 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
 	return &newParam
 }
 
-func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
 	caseInsensitive := in == openapi.InHeader
-	for _, param := range *params {
+	for i, param := range *params {
+		idx = i
 		paramObj, err := param.ResolveParameter(paramResolver)
 		if err != nil {
 			logger.Log.Warningf("Failed to resolve reference: %s", err)
@@ -89,7 +90,8 @@ func findParamByName(params *openapi.ParameterList, in openapi.In, name string)
 			break
 		}
 	}
-	return pathParam
+
+	return idx, pathParam
 }
 
 func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
@@ -107,37 +109,16 @@ func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj
 	return nil
 }
 
-type NVPair struct {
-	Name  string
-	Value string
-}
-
 type nvParams struct {
 	In             openapi.In
-	Pairs          func() []NVPair
+	Pairs          []har.NVP
 	IsIgnored      func(name string) bool
 	GeneralizeName func(name string) string
 }
 
-func qstrToNVP(list []har.QueryString) []NVPair {
-	res := make([]NVPair, len(list))
-	for idx, val := range list {
-		res[idx] = NVPair{Name: val.Name, Value: val.Value}
-	}
-	return res
-}
-
-func hdrToNVP(list []har.Header) []NVPair {
-	res := make([]NVPair, len(list))
-	for idx, val := range list {
-		res[idx] = NVPair{Name: val.Name, Value: val.Value}
-	}
-	return res
-}
-
 func handleNameVals(gw nvParams, params **openapi.ParameterList) {
 	visited := map[string]*openapi.ParameterObj{}
-	for _, pair := range gw.Pairs() {
+	for _, pair := range gw.Pairs {
 		if gw.IsIgnored(pair.Name) {
 			continue
 		}
@@ -145,7 +126,7 @@ func handleNameVals(gw nvParams, params **openapi.ParameterList) {
 		nameGeneral := gw.GeneralizeName(pair.Name)
 
 		initParams(params)
-		param := findParamByName(*params, gw.In, pair.Name)
+		_, param := findParamByName(*params, gw.In, pair.Name)
 		if param == nil {
 			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
 			appended := append(**params, param)
@@ -342,3 +323,26 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) {
 
 	return nil, false
 }
+
+func cleanNonAlnum(s []byte) string {
+	j := 0
+	for _, b := range s {
+		if ('a' <= b && b <= 'z') ||
+			('A' <= b && b <= 'Z') ||
+			('0' <= b && b <= '9') ||
+			b == ' ' {
+			s[j] = b
+			j++
+		}
+	}
+	return string(s[:j])
+}
+
+func isAlpha(s string) bool {
+	for _, r := range s {
+		if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') {
+			return false
+		}
+	}
+	return true
+}

agent/pkg/providers/install_provider.go

@@ -2,9 +2,14 @@ package providers
 
 import (
 	"context"
+	"errors"
 	"mizuserver/pkg/config"
+
+	ory "github.com/ory/kratos-client-go"
 )
 
+const AdminUsername = "admin"
+
 func IsInstallNeeded() (bool, error) {
 	if !config.Config.StandaloneMode { // install not needed in ephermeral mizu
 		return false, nil
@@ -16,3 +21,27 @@ func IsInstallNeeded() (bool, error) {
 		return !anyUserExists, nil
 	}
 }
+
+func CreateAdminUser(password string, ctx context.Context) (token *string, err error, formErrorMessages map[string][]ory.UiText) {
+	if isInstallNeeded, err := IsInstallNeeded(); err != nil {
+		return nil, err, nil
+	} else if !isInstallNeeded {
+		return nil, errors.New("The admin user has already been created"), nil
+	}
+
+	token, identityId, err, formErrors := RegisterUser(AdminUsername, password, ctx)
+	if err != nil {
+		return nil, err, formErrors
+	}
+
+	err = SetUserSystemRole(AdminUsername, AdminRole)
+
+	if err != nil {
+		//Delete the user to prevent a half-setup situation where admin user is created without admin privileges
+		DeleteUser(identityId, ctx)
+
+		return nil, err, nil
+	}
+
+	return token, nil, nil
+}

agent/pkg/providers/stats_provider.go

@@ -7,6 +7,7 @@ import (
 
 type GeneralStats struct {
 	EntriesCount        int
+	EntriesVolumeInGB   float64
 	FirstEntryTimestamp int
 	LastEntryTimestamp  int
 }
@@ -21,8 +22,9 @@ func GetGeneralStats() GeneralStats {
 	return generalStats
 }
 
-func EntryAdded() {
+func EntryAdded(size int) {
 	generalStats.EntriesCount++
+	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
 
 	currentTimestamp := int(time.Now().Unix())
 
@@ -32,5 +34,3 @@ func EntryAdded() {
 
 	generalStats.LastEntryTimestamp = currentTimestamp
 }
-
-

agent/pkg/providers/stats_provider_test.go

@@ -12,6 +12,10 @@ func TestNoEntryAddedCount(t *testing.T) {
 	if entriesStats.EntriesCount != 0 {
 		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
 	}
+
+	if entriesStats.EntriesVolumeInGB != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
+	}
 }
 
 func TestEntryAddedCount(t *testing.T) {
@@ -20,7 +24,7 @@ func TestEntryAddedCount(t *testing.T) {
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
 			for i := 0; i < entriesCount; i++ {
-				providers.EntryAdded()
+				providers.EntryAdded(0)
 			}
 
 			entriesStats := providers.GetGeneralStats()
@@ -29,7 +33,38 @@ func TestEntryAddedCount(t *testing.T) {
 				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
 			}
 
+			if entriesStats.EntriesVolumeInGB != 0 {
+				t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
+			}
+
 			t.Cleanup(providers.ResetGeneralStats)
 		})
 	}
 }
+
+func TestEntryAddedVolume(t *testing.T) {
+	// 6 bytes + 4 bytes
+	tests := [][]byte{[]byte("volume"), []byte("test")}
+	var expectedEntriesCount int
+	var expectedVolumeInGB float64
+
+	for _, data := range tests {
+		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {
+			expectedEntriesCount++
+			expectedVolumeInGB += float64(len(data)) / (1 << 30)
+
+			providers.EntryAdded(len(data))
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != expectedEntriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", expectedEntriesCount, entriesStats.EntriesCount)
+			}
+
+			if entriesStats.EntriesVolumeInGB != expectedVolumeInGB {
+				t.Errorf("unexpected result - expected: %v, actual: %v", expectedVolumeInGB, entriesStats.EntriesVolumeInGB)
+			}
+		})
+	}
+
+}

agent/pkg/providers/user_provider.go

@@ -66,17 +66,17 @@ func PerformLogin(username string, password string, ctx context.Context) (*strin
 	return result.SessionToken, nil
 }
 
-func VerifyToken(token string, ctx context.Context) (bool, error) {
+func VerifyToken(token string, ctx context.Context) (*ory.Session, error) {
 	flow, _, err := client.V0alpha2Api.ToSession(ctx).XSessionToken(token).Execute()
 	if err != nil {
-		return false, err
+		return nil, err
 	}
 
 	if flow == nil {
-		return false, nil
+		return nil, nil
 	}
 
-	return true, nil
+	return flow, nil
 }
 
 func DeleteUser(identityId string, ctx context.Context) error {

agent/pkg/providers/user_role_provider.go

@@ -0,0 +1,180 @@
+package providers
+
+/*
+This provider abstracts keto role management down to what we need for mizu
+
+Keto, in the configuration we use it, is basically a tuple database. Each tuple consists of 4 strings (namespace, object, relation, subjectID) - for example ("workspaces", "sock-shop-workspace", "viewer", "ramiberman")
+
+namespace - used to organize tuples into groups - we currently use "system" for defining admins and "workspaces" for defining workspace permissions
+objects - represents something one can have permissions to (files, mizu workspaces etc)
+relation - represents the permission (viewer, editor, owner etc) - we currently use only viewer and admin
+subject - represents the user or group that has the permission - we currently use usernames
+
+more on keto here: https://www.ory.sh/keto/docs/
+*/
+
+import (
+	"fmt"
+	"mizuserver/pkg/utils"
+
+	ketoClient "github.com/ory/keto-client-go/client"
+	ketoRead "github.com/ory/keto-client-go/client/read"
+	ketoWrite "github.com/ory/keto-client-go/client/write"
+	ketoModels "github.com/ory/keto-client-go/models"
+)
+
+const (
+	ketoHost      = "localhost"
+	ketoReadPort  = 4466
+	ketoWritePort = 4467
+)
+
+var (
+	readClient              = getKetoClient(fmt.Sprintf("%s:%d", ketoHost, ketoReadPort))
+	writeClient             = getKetoClient(fmt.Sprintf("%s:%d", ketoHost, ketoWritePort))
+	systemRoleNamespace     = "system"
+	workspacesRoleNamespace = "workspaces"
+
+	systemObject = "system"
+
+	AdminRole  = "admin"
+	ViewerRole = "viewer"
+)
+
+func GetUserSystemRoles(username string) ([]string, error) {
+	return getObjectRelationsForSubjectID(systemRoleNamespace, systemObject, username)
+}
+
+func CheckIfUserHasSystemRole(username string, role string) (bool, error) {
+	systemRoles, err := GetUserSystemRoles(username)
+	if err != nil {
+		return false, err
+	}
+
+	for _, systemRole := range systemRoles {
+		if systemRole == role {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func GetUserWorkspaceRole(username string, workspace string) ([]string, error) {
+	return getObjectRelationsForSubjectID(workspacesRoleNamespace, workspace, username)
+}
+
+func SetUserWorkspaceRole(username string, workspace string, role string) error {
+	return createObjectRelationForSubjectID(workspacesRoleNamespace, workspace, username, role)
+}
+
+func SetUserSystemRole(username string, role string) error {
+	return createObjectRelationForSubjectID(systemRoleNamespace, systemObject, username, role)
+}
+
+func DeleteAllUserWorkspaceRoles(username string) error {
+	return deleteAllNamespacedRelationsForSubjectID(workspacesRoleNamespace, username)
+}
+
+func createObjectRelationForSubjectID(namespace string, object string, subjectID string, relation string) error {
+	tuple := ketoModels.RelationQuery{
+		Namespace: &namespace,
+		Object:    object,
+		Relation:  relation,
+		SubjectID: subjectID,
+	}
+
+	_, err := writeClient.Write.CreateRelationTuple(ketoWrite.
+		NewCreateRelationTupleParams().
+		WithPayload(&tuple))
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getObjectRelationsForSubjectID(namespace string, object string, subjectID string) ([]string, error) {
+	relationTuples, err := queryRelationTuples(&namespace, &object, &subjectID, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	relations := make([]string, 0)
+
+	for _, clientRelation := range relationTuples {
+		relations = append(relations, *clientRelation.Relation)
+	}
+
+	return utils.UniqueStringSlice(relations), nil
+}
+
+func deleteAllNamespacedRelationsForSubjectID(namespace string, subjectID string) error {
+	relationTuples, err := queryRelationTuples(&namespace, nil, &subjectID, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, clientRelation := range relationTuples {
+		_, err := writeClient.Write.DeleteRelationTuple(ketoWrite.
+			NewDeleteRelationTupleParams().
+			WithNamespace(*clientRelation.Namespace).
+			WithObject(*clientRelation.Object).
+			WithRelation(*clientRelation.Relation).
+			WithSubjectID(&clientRelation.SubjectID))
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func queryRelationTuples(namespace *string, object *string, subjectID *string, role *string) ([]*ketoModels.InternalRelationTuple, error) {
+	relationTuplesQuery := ketoRead.NewGetRelationTuplesParams()
+	if namespace != nil {
+		relationTuplesQuery = relationTuplesQuery.WithNamespace(*namespace)
+	}
+	if object != nil {
+		relationTuplesQuery = relationTuplesQuery.WithObject(object)
+	}
+	if subjectID != nil {
+		relationTuplesQuery = relationTuplesQuery.WithSubjectID(subjectID)
+	}
+	if role != nil {
+		relationTuplesQuery = relationTuplesQuery.WithRelation(role)
+	}
+
+	return recursiveKetoPagingTraverse(relationTuplesQuery, make([]*ketoModels.InternalRelationTuple, 0), "")
+}
+
+func recursiveKetoPagingTraverse(queryParams *ketoRead.GetRelationTuplesParams, tuples []*ketoModels.InternalRelationTuple, pagingToken string) ([]*ketoModels.InternalRelationTuple, error) {
+	params := queryParams
+	if pagingToken != "" {
+		params = queryParams.WithPageToken(&pagingToken)
+	}
+
+	clientRelationsResponse, err := readClient.Read.GetRelationTuples(params)
+
+	if err != nil {
+		return nil, err
+	}
+
+	tuples = append(tuples, clientRelationsResponse.Payload.RelationTuples...)
+
+	if clientRelationsResponse.Payload.NextPageToken != "" {
+		return recursiveKetoPagingTraverse(queryParams, tuples, clientRelationsResponse.Payload.NextPageToken)
+	}
+
+	return tuples, nil
+}
+
+func getKetoClient(url string) *ketoClient.OryKeto {
+	return ketoClient.NewHTTPClientWithConfig(nil,
+		ketoClient.
+			DefaultTransportConfig().
+			WithSchemes([]string{"http"}).
+			WithHost(url))
+}

agent/pkg/routes/config_routes.go

@@ -1,15 +1,16 @@
 package routes
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/controllers"
 	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
 )
 
 func ConfigRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/config")
 	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.POST("/tapConfig", controllers.PostTapConfig)
-	routeGroup.GET("/tapConfig", controllers.GetTapConfig)
+	routeGroup.POST("/tap", middlewares.RequiresAdmin(), controllers.PostTapConfig)
+	routeGroup.GET("/tap", controllers.GetTapConfig)
 }

agent/pkg/routes/install_routes.go

@@ -10,4 +10,5 @@ func InstallRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/install")
 
 	routeGroup.GET("/isNeeded", controllers.IsSetupNecessary)
+	routeGroup.POST("/admin", controllers.SetupAdminUser)
 }

agent/pkg/routes/service_map_routes.go

@@ -0,0 +1,19 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
+)
+
+func ServiceMapRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/servicemap")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	controller := controllers.NewServiceMapController()
+
+	routeGroup.GET("/status", controller.Status)
+	routeGroup.GET("/get", controller.Get)
+	routeGroup.GET("/reset", controller.Reset)
+}

agent/pkg/rules/rulesHTTP.go

@@ -4,15 +4,15 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/har"
 	"reflect"
 	"regexp"
 	"strings"
 
 	"github.com/up9inc/mizu/shared/logger"
 
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
-	jsonpath "github.com/yalp/jsonpath"
+	"github.com/yalp/jsonpath"
 )
 
 type RulesMatched struct {

agent/pkg/servicemap/models.go

@@ -0,0 +1,32 @@
+package servicemap
+
+import (
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type ServiceMapStatus struct {
+	Status                string `json:"status"`
+	EntriesProcessedCount int    `json:"entriesProcessedCount"`
+	NodeCount             int    `json:"nodeCount"`
+	EdgeCount             int    `json:"edgeCount"`
+}
+
+type ServiceMapResponse struct {
+	Status ServiceMapStatus `json:"status"`
+	Nodes  []ServiceMapNode `json:"nodes"`
+	Edges  []ServiceMapEdge `json:"edges"`
+}
+
+type ServiceMapNode struct {
+	Id    int         `json:"id"`
+	Name  string      `json:"name"`
+	Entry *tapApi.TCP `json:"entry"`
+	Count int         `json:"count"`
+}
+
+type ServiceMapEdge struct {
+	Source      ServiceMapNode   `json:"source"`
+	Destination ServiceMapNode   `json:"destination"`
+	Count       int              `json:"count"`
+	Protocol    *tapApi.Protocol `json:"protocol"`
+}

agent/pkg/servicemap/servicemap.go

@@ -0,0 +1,271 @@
+package servicemap
+
+import (
+	"sync"
+
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	ServiceMapEnabled  = "enabled"
+	ServiceMapDisabled = "disabled"
+	UnresolvedNodeName = "unresolved"
+)
+
+var instance *serviceMap
+var once sync.Once
+
+func GetInstance() ServiceMap {
+	once.Do(func() {
+		instance = newServiceMap()
+		logger.Log.Debug("Service Map Initialized")
+	})
+	return instance
+}
+
+type serviceMap struct {
+	config           *shared.MizuAgentConfig
+	graph            *graph
+	entriesProcessed int
+}
+
+type ServiceMap interface {
+	SetConfig(config *shared.MizuAgentConfig)
+	IsEnabled() bool
+	NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol)
+	GetStatus() ServiceMapStatus
+	GetNodes() []ServiceMapNode
+	GetEdges() []ServiceMapEdge
+	GetEntriesProcessedCount() int
+	GetNodesCount() int
+	GetEdgesCount() int
+	Reset()
+}
+
+func newServiceMap() *serviceMap {
+	return &serviceMap{
+		config:           nil,
+		entriesProcessed: 0,
+		graph:            newDirectedGraph(),
+	}
+}
+
+type key string
+
+type entryData struct {
+	key   key
+	entry *tapApi.TCP
+}
+
+type nodeData struct {
+	id    int
+	entry *tapApi.TCP
+	count int
+}
+
+type edgeProtocol struct {
+	protocol *tapApi.Protocol
+	count    int
+}
+
+type edgeData struct {
+	data map[key]*edgeProtocol
+}
+
+type graph struct {
+	Nodes map[key]*nodeData
+	Edges map[key]map[key]*edgeData
+}
+
+func newDirectedGraph() *graph {
+	return &graph{
+		Nodes: make(map[key]*nodeData),
+		Edges: make(map[key]map[key]*edgeData),
+	}
+}
+
+func newNodeData(id int, e *tapApi.TCP) *nodeData {
+	return &nodeData{
+		id:    id,
+		entry: e,
+		count: 1,
+	}
+}
+
+func newEdgeData(p *tapApi.Protocol) *edgeData {
+	return &edgeData{
+		data: map[key]*edgeProtocol{
+			key(p.Name): {
+				protocol: p,
+				count:    1,
+			},
+		},
+	}
+}
+
+func (s *serviceMap) nodeExists(k key) (*nodeData, bool) {
+	n, ok := s.graph.Nodes[k]
+	return n, ok
+}
+
+func (s *serviceMap) addNode(k key, e *tapApi.TCP) (*nodeData, bool) {
+	nd, exists := s.nodeExists(k)
+	if !exists {
+		s.graph.Nodes[k] = newNodeData(len(s.graph.Nodes)+1, e)
+		return s.graph.Nodes[k], true
+	}
+	return nd, false
+}
+
+func (s *serviceMap) addEdge(u, v *entryData, p *tapApi.Protocol) {
+	if n, ok := s.addNode(u.key, u.entry); !ok {
+		n.count++
+	}
+	if n, ok := s.addNode(v.key, v.entry); !ok {
+		n.count++
+	}
+
+	if _, ok := s.graph.Edges[u.key]; !ok {
+		s.graph.Edges[u.key] = make(map[key]*edgeData)
+	}
+
+	// new edge u -> v pair
+	// protocol is the same for u and v
+	if e, ok := s.graph.Edges[u.key][v.key]; ok {
+		// edge data already exists for u -> v pair
+		// we have a new protocol for this u -> v pair
+
+		k := key(p.Name)
+		if pd, pOk := e.data[k]; pOk {
+			// protocol key already exists, just increment the count
+			pd.count++
+		} else {
+			// new protocol key
+			e.data[k] = &edgeProtocol{
+				protocol: p,
+				count:    1,
+			}
+		}
+	} else {
+		// new edge data for u -> v pair
+		s.graph.Edges[u.key][v.key] = newEdgeData(p)
+	}
+
+	s.entriesProcessed++
+}
+
+func (s *serviceMap) SetConfig(config *shared.MizuAgentConfig) {
+	s.config = config
+}
+
+func (s *serviceMap) IsEnabled() bool {
+	if s.config != nil && s.config.ServiceMap {
+		return true
+	}
+	return false
+}
+
+func (s *serviceMap) NewTCPEntry(src *tapApi.TCP, dst *tapApi.TCP, p *tapApi.Protocol) {
+	if !s.IsEnabled() {
+		return
+	}
+
+	srcEntry := &entryData{
+		key:   key(src.IP),
+		entry: src,
+	}
+	if len(srcEntry.entry.Name) == 0 {
+		srcEntry.entry.Name = UnresolvedNodeName
+	}
+
+	dstEntry := &entryData{
+		key:   key(dst.IP),
+		entry: dst,
+	}
+	if len(dstEntry.entry.Name) == 0 {
+		dstEntry.entry.Name = UnresolvedNodeName
+	}
+
+	s.addEdge(srcEntry, dstEntry, p)
+}
+
+func (s *serviceMap) GetStatus() ServiceMapStatus {
+	status := ServiceMapDisabled
+	if s.IsEnabled() {
+		status = ServiceMapEnabled
+	}
+
+	return ServiceMapStatus{
+		Status:                status,
+		EntriesProcessedCount: s.entriesProcessed,
+		NodeCount:             s.GetNodesCount(),
+		EdgeCount:             s.GetEdgesCount(),
+	}
+}
+
+func (s *serviceMap) GetNodes() []ServiceMapNode {
+	var nodes []ServiceMapNode
+	for i, n := range s.graph.Nodes {
+		nodes = append(nodes, ServiceMapNode{
+			Id:    n.id,
+			Name:  string(i),
+			Entry: n.entry,
+			Count: n.count,
+		})
+	}
+	return nodes
+}
+
+func (s *serviceMap) GetEdges() []ServiceMapEdge {
+	var edges []ServiceMapEdge
+	for u, m := range s.graph.Edges {
+		for v := range m {
+			for _, p := range s.graph.Edges[u][v].data {
+				edges = append(edges, ServiceMapEdge{
+					Source: ServiceMapNode{
+						Id:    s.graph.Nodes[u].id,
+						Name:  string(u),
+						Entry: s.graph.Nodes[u].entry,
+						Count: s.graph.Nodes[u].count,
+					},
+					Destination: ServiceMapNode{
+						Id:    s.graph.Nodes[v].id,
+						Name:  string(v),
+						Entry: s.graph.Nodes[v].entry,
+						Count: s.graph.Nodes[v].count,
+					},
+					Count:    p.count,
+					Protocol: p.protocol,
+				})
+			}
+		}
+	}
+	return edges
+}
+
+func (s *serviceMap) GetEntriesProcessedCount() int {
+	return s.entriesProcessed
+}
+
+func (s *serviceMap) GetNodesCount() int {
+	return len(s.graph.Nodes)
+}
+
+func (s *serviceMap) GetEdgesCount() int {
+	var count int
+	for u, m := range s.graph.Edges {
+		for v := range m {
+			for range s.graph.Edges[u][v].data {
+				count++
+			}
+		}
+	}
+	return count
+}
+
+func (s *serviceMap) Reset() {
+	s.entriesProcessed = 0
+	s.graph = newDirectedGraph()
+}

agent/pkg/servicemap/servicemap_test.go

@@ -0,0 +1,405 @@
+package servicemap
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	a    = "aService"
+	b    = "bService"
+	c    = "cService"
+	d    = "dService"
+	Ip   = "127.0.0.1"
+	Port = "80"
+)
+
+var (
+	TCPEntryA = &tapApi.TCP{
+		Name: a,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, a),
+	}
+	TCPEntryB = &tapApi.TCP{
+		Name: b,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, b),
+	}
+	TCPEntryC = &tapApi.TCP{
+		Name: c,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, c),
+	}
+	TCPEntryD = &tapApi.TCP{
+		Name: d,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, d),
+	}
+	TCPEntryUnresolved = &tapApi.TCP{
+		Name: "",
+		Port: Port,
+		IP:   Ip,
+	}
+	TCPEntryUnresolved2 = &tapApi.TCP{
+		Name: "",
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, UnresolvedNodeName),
+	}
+	ProtocolHttp = &tapApi.Protocol{
+		Name:            "http",
+		LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+		Abbreviation:    "HTTP",
+		Macro:           "http",
+		Version:         "1.1",
+		BackgroundColor: "#205cf5",
+		ForegroundColor: "#ffffff",
+		FontSize:        12,
+		ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc2616",
+		Ports:           []string{"80", "443", "8080"},
+		Priority:        0,
+	}
+	ProtocolRedis = &tapApi.Protocol{
+		Name:            "redis",
+		LongName:        "Redis Serialization Protocol",
+		Abbreviation:    "REDIS",
+		Macro:           "redis",
+		Version:         "3.x",
+		BackgroundColor: "#a41e11",
+		ForegroundColor: "#ffffff",
+		FontSize:        11,
+		ReferenceLink:   "https://redis.io/topics/protocol",
+		Ports:           []string{"6379"},
+		Priority:        3,
+	}
+)
+
+type ServiceMapDisabledSuite struct {
+	suite.Suite
+
+	instance ServiceMap
+}
+
+type ServiceMapEnabledSuite struct {
+	suite.Suite
+
+	instance ServiceMap
+}
+
+func (s *ServiceMapDisabledSuite) SetupTest() {
+	s.instance = GetInstance()
+}
+
+func (s *ServiceMapEnabledSuite) SetupTest() {
+	s.instance = GetInstance()
+	s.instance.SetConfig(&shared.MizuAgentConfig{
+		ServiceMap: true,
+	})
+}
+
+func (s *ServiceMapDisabledSuite) TestServiceMapInstance() {
+	assert := s.Assert()
+
+	assert.NotNil(s.instance)
+}
+
+func (s *ServiceMapDisabledSuite) TestServiceMapSingletonInstance() {
+	assert := s.Assert()
+
+	instance2 := GetInstance()
+
+	assert.NotNil(s.instance)
+	assert.NotNil(instance2)
+	assert.Equal(s.instance, instance2)
+}
+
+func (s *ServiceMapDisabledSuite) TestServiceMapIsEnabledShouldReturnFalseByDefault() {
+	assert := s.Assert()
+
+	enabled := s.instance.IsEnabled()
+
+	assert.False(enabled)
+}
+
+func (s *ServiceMapDisabledSuite) TestGetStatusShouldReturnDisabledByDefault() {
+	assert := s.Assert()
+
+	status := s.instance.GetStatus()
+
+	assert.Equal("disabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+}
+
+func (s *ServiceMapDisabledSuite) TestNewTCPEntryShouldDoNothingWhenDisabled() {
+	assert := s.Assert()
+
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryC, TCPEntryD, ProtocolHttp)
+	status := s.instance.GetStatus()
+
+	assert.Equal("disabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+}
+
+// Enabled
+
+func (s *ServiceMapEnabledSuite) TestServiceMapIsEnabled() {
+	assert := s.Assert()
+
+	enabled := s.instance.IsEnabled()
+
+	assert.True(enabled)
+}
+
+func (s *ServiceMapEnabledSuite) TestServiceMap() {
+	assert := s.Assert()
+
+	// A -> B - HTTP
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+
+	nodes := s.instance.GetNodes()
+	edges := s.instance.GetEdges()
+
+	// Counts for the first entry
+	assert.Equal(1, s.instance.GetEntriesProcessedCount())
+	assert.Equal(2, s.instance.GetNodesCount())
+	assert.Equal(2, len(nodes))
+	assert.Equal(1, s.instance.GetEdgesCount())
+	assert.Equal(1, len(edges))
+	//http protocol
+	assert.Equal(1, edges[0].Count)
+	assert.Equal(ProtocolHttp.Name, edges[0].Protocol.Name)
+
+	// same A -> B - HTTP, http protocol count should be 2, edges count should be 1
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+
+	// Counts for a second entry
+	assert.Equal(2, s.instance.GetEntriesProcessedCount())
+	assert.Equal(2, s.instance.GetNodesCount())
+	assert.Equal(2, len(nodes))
+	// edges count should still be 1, but http protocol count should be 2
+	assert.Equal(1, s.instance.GetEdgesCount())
+	assert.Equal(1, len(edges))
+	// http protocol
+	assert.Equal(2, edges[0].Count) //http
+	assert.Equal(ProtocolHttp.Name, edges[0].Protocol.Name)
+
+	// same A -> B - REDIS, http protocol count should be 2 and redis protocol count should 1, edges count should be 2
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolRedis)
+
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+
+	// Counts after second entry
+	assert.Equal(3, s.instance.GetEntriesProcessedCount())
+	assert.Equal(2, s.instance.GetNodesCount())
+	assert.Equal(2, len(nodes))
+	// edges count should be 2, http protocol count should be 2 and redis protocol should be 1
+	assert.Equal(2, s.instance.GetEdgesCount())
+	assert.Equal(2, len(edges))
+	// http and redis protocols
+	httpIndex := -1
+	redisIndex := -1
+	for i, e := range edges {
+		if e.Protocol.Name == ProtocolHttp.Name {
+			httpIndex = i
+			continue
+		}
+		if e.Protocol.Name == ProtocolRedis.Name {
+			redisIndex = i
+		}
+	}
+	assert.NotEqual(-1, httpIndex)
+	assert.NotEqual(-1, redisIndex)
+	// http protocol
+	assert.Equal(2, edges[httpIndex].Count)
+	assert.Equal(ProtocolHttp.Name, edges[httpIndex].Protocol.Name)
+	// redis protocol
+	assert.Equal(1, edges[redisIndex].Count)
+	assert.Equal(ProtocolRedis.Name, edges[redisIndex].Protocol.Name)
+
+	// other entries
+	s.instance.NewTCPEntry(TCPEntryUnresolved, TCPEntryA, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryB, TCPEntryUnresolved2, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryC, TCPEntryD, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryC, ProtocolHttp)
+
+	status := s.instance.GetStatus()
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+	expectedEntriesProcessedCount := 7
+	expectedNodeCount := 6
+	expectedEdgeCount := 6
+
+	// Counts after all entries
+	assert.Equal(expectedEntriesProcessedCount, s.instance.GetEntriesProcessedCount())
+	assert.Equal(expectedNodeCount, s.instance.GetNodesCount())
+	assert.Equal(expectedNodeCount, len(nodes))
+	assert.Equal(expectedEdgeCount, s.instance.GetEdgesCount())
+	assert.Equal(expectedEdgeCount, len(edges))
+
+	// Status
+	assert.Equal("enabled", status.Status)
+	assert.Equal(expectedEntriesProcessedCount, status.EntriesProcessedCount)
+	assert.Equal(expectedNodeCount, status.NodeCount)
+	assert.Equal(expectedEdgeCount, status.EdgeCount)
+
+	// Nodes
+	aNode := -1
+	bNode := -1
+	cNode := -1
+	dNode := -1
+	unresolvedNode := -1
+	unresolvedNode2 := -1
+	var validateNode = func(node ServiceMapNode, entryName string, count int) int {
+		// id
+		assert.GreaterOrEqual(node.Id, 1)
+		assert.LessOrEqual(node.Id, expectedNodeCount)
+
+		// entry
+		// node.Name is the key of the node, key = entry.IP
+		// entry.Name is the name of the service and could be unresolved
+		assert.Equal(node.Name, node.Entry.IP)
+		assert.Equal(Port, node.Entry.Port)
+		assert.Equal(entryName, node.Entry.Name)
+
+		// count
+		assert.Equal(count, node.Count)
+
+		return node.Id
+	}
+
+	for _, v := range nodes {
+		if strings.HasSuffix(v.Name, a) {
+			aNode = validateNode(v, a, 5)
+			continue
+		}
+		if strings.HasSuffix(v.Name, b) {
+			bNode = validateNode(v, b, 4)
+			continue
+		}
+		if strings.HasSuffix(v.Name, c) {
+			cNode = validateNode(v, c, 2)
+			continue
+		}
+		if strings.HasSuffix(v.Name, d) {
+			dNode = validateNode(v, d, 1)
+			continue
+		}
+		if v.Name == Ip {
+			unresolvedNode = validateNode(v, UnresolvedNodeName, 1)
+			continue
+		}
+		if strings.HasSuffix(v.Name, UnresolvedNodeName) {
+			unresolvedNode2 = validateNode(v, UnresolvedNodeName, 1)
+			continue
+		}
+	}
+
+	// Make sure we found all the nodes
+	nodeIds := [...]int{aNode, bNode, cNode, dNode, unresolvedNode, unresolvedNode2}
+	for _, v := range nodeIds {
+		assert.NotEqual(-1, v)
+	}
+
+	// Edges
+	abEdge := -1
+	uaEdge := -1
+	buEdge := -1
+	cdEdge := -1
+	acEdge := -1
+	var validateEdge = func(edge ServiceMapEdge, sourceEntryName string, destEntryName string, protocolName string, protocolCount int) {
+		// source
+		assert.Contains(nodeIds, edge.Source.Id)
+		assert.LessOrEqual(edge.Source.Id, expectedNodeCount)
+		assert.Equal(edge.Source.Name, edge.Source.Entry.IP)
+		assert.Equal(sourceEntryName, edge.Source.Entry.Name)
+
+		// destination
+		assert.Contains(nodeIds, edge.Destination.Id)
+		assert.LessOrEqual(edge.Destination.Id, expectedNodeCount)
+		assert.Equal(edge.Destination.Name, edge.Destination.Entry.IP)
+		assert.Equal(destEntryName, edge.Destination.Entry.Name)
+
+		// protocol
+		assert.Equal(protocolName, edge.Protocol.Name)
+		assert.Equal(protocolCount, edge.Count)
+	}
+
+	for i, v := range edges {
+		if v.Source.Entry.Name == a && v.Destination.Entry.Name == b && v.Protocol.Name == "http" {
+			validateEdge(v, a, b, ProtocolHttp.Name, 2)
+			abEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == a && v.Destination.Entry.Name == b && v.Protocol.Name == "redis" {
+			validateEdge(v, a, b, ProtocolRedis.Name, 1)
+			abEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == UnresolvedNodeName && v.Destination.Entry.Name == a {
+			validateEdge(v, UnresolvedNodeName, a, ProtocolHttp.Name, 1)
+			uaEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == b && v.Destination.Entry.Name == UnresolvedNodeName {
+			validateEdge(v, b, UnresolvedNodeName, ProtocolHttp.Name, 1)
+			buEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == c && v.Destination.Entry.Name == d {
+			validateEdge(v, c, d, ProtocolHttp.Name, 1)
+			cdEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == a && v.Destination.Entry.Name == c {
+			validateEdge(v, a, c, ProtocolHttp.Name, 1)
+			acEdge = i
+			continue
+		}
+	}
+
+	// Make sure we found all the edges
+	for _, v := range [...]int{abEdge, uaEdge, buEdge, cdEdge, acEdge} {
+		assert.NotEqual(-1, v)
+	}
+
+	// Reset
+	s.instance.Reset()
+	status = s.instance.GetStatus()
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+
+	// Counts after reset
+	assert.Equal(0, s.instance.GetEntriesProcessedCount())
+	assert.Equal(0, s.instance.GetNodesCount())
+	assert.Equal(0, s.instance.GetEdgesCount())
+
+	// Status after reset
+	assert.Equal("enabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+
+	// Nodes after reset
+	assert.Equal([]ServiceMapNode(nil), nodes)
+
+	// Edges after reset
+	assert.Equal([]ServiceMapEdge(nil), edges)
+}
+
+func TestServiceMapSuite(t *testing.T) {
+	suite.Run(t, new(ServiceMapDisabledSuite))
+	suite.Run(t, new(ServiceMapEnabledSuite))
+}

agent/pkg/up9/main.go

@@ -3,10 +3,10 @@ package up9
 import (
 	"bytes"
 	"compress/zlib"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
@@ -15,7 +15,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/martian/har"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
@@ -247,7 +246,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
 				continue
 			}
-			harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+			harEntry, err := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
 			if err != nil {
 				continue
 			}
@@ -259,11 +258,6 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 				harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name)
 			}
 
-			// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
-			if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
-				continue
-			}
-
 			batch = append(batch, *harEntry)
 
 			now := time.Now()

agent/pkg/utils/utils.go

@@ -85,3 +85,18 @@ func SaveJsonFile(filePath string, value interface{}) error {
 
 	return nil
 }
+
+func UniqueStringSlice(s []string) []string {
+	uniqueSlice := make([]string, 0)
+	uniqueMap := map[string]bool{}
+
+	for _, val := range s {
+		if uniqueMap[val] == true {
+			continue
+		}
+		uniqueMap[val] = true
+		uniqueSlice = append(uniqueSlice, val)
+	}
+
+	return uniqueSlice
+}

cli/Makefile

@@ -30,15 +30,12 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables
 build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
 	@mkdir -p bin && sed s/_SEM_VER_/$(SEM_VER)/g README.md.TEMPLATE >  bin/README.md
-	@$(MAKE) build GOOS=darwin GOARCH=amd64
 	@$(MAKE) build GOOS=linux GOARCH=amd64
+	@$(MAKE) build GOOS=linux GOARCH=arm64
+	@$(MAKE) build GOOS=darwin GOARCH=amd64
 	@$(MAKE) build GOOS=darwin GOARCH=arm64
 	@$(MAKE) build GOOS=windows GOARCH=amd64
 	@mv ./bin/mizu_windows_amd64 ./bin/mizu.exe
-	@# $(MAKE) GOOS=linux GOARCH=386
-	@# $(MAKE) GOOS=windows GOARCH=386
-	@# $(MAKE) GOOS=linux GOARCH=arm64
-	@# $(MAKE) GOOS=windows GOARCH=arm64
 	@echo "---------"
 	@find ./bin -ls
 

cli/README.md.TEMPLATE

@@ -3,22 +3,27 @@ Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/b
 
 ## Download Mizu for your platform
 
-**Mac** (Intel)
+**Mac** (x86-64/Intel)
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_amd64 && chmod 755 mizu
 ```
 
-**Mac** (Apple M1 silicon)
+**Mac** (AArch64/Apple M1 silicon)
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_arm64 && chmod 755 mizu
 ```
 
-**Linux** 
+**Linux** (x86-64)
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_amd64 && chmod 755 mizu
 ```
 
-**Windows** (Intel 64bit)
+**Linux** (AArch64)
+```
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_arm64 && chmod 755 mizu
+```
+
+**Windows** (x86-64)
 ```
 curl -LO https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu.exe
 ```

cli/apiserver/provider.go

@@ -23,8 +23,8 @@ type Provider struct {
 	client  *http.Client
 }
 
-const DefaultRetries = 20
-const DefaultTimeout = 5 * time.Second
+const DefaultRetries = 3
+const DefaultTimeout = 2 * time.Second
 
 func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	return &Provider{

cli/cmd/check.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/telemetry"
+)
+
+var checkCmd = &cobra.Command{
+	Use:   "check",
+	Short: "Check the Mizu installation for potential problems",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("check", nil)
+		runMizuCheck()
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(checkCmd)
+}

cli/cmd/checkRunner.go

@@ -0,0 +1,239 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/shared/semver"
+)
+
+func runMizuCheck() {
+	logger.Log.Infof("Mizu install checks\n===================")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel() // cancel will be called when this function exits
+
+	kubernetesProvider, kubernetesVersion, checkPassed := checkKubernetesApi()
+
+	if checkPassed {
+		checkPassed = checkKubernetesVersion(kubernetesVersion)
+	}
+
+	var isInstallCommand bool
+	if checkPassed {
+		checkPassed, isInstallCommand = checkMizuMode(ctx, kubernetesProvider)
+	}
+
+	if checkPassed {
+		checkPassed = checkAllResourcesExist(ctx, kubernetesProvider, isInstallCommand)
+	}
+
+	if checkPassed {
+		checkPassed = checkServerConnection(kubernetesProvider)
+	}
+
+	if checkPassed {
+		logger.Log.Infof("\nStatus check results are %v", fmt.Sprintf(uiUtils.Green, "√"))
+	} else {
+		logger.Log.Errorf("\nStatus check results are %v", fmt.Sprintf(uiUtils.Red, "✗"))
+	}
+}
+
+func checkKubernetesApi() (*kubernetes.Provider, *semver.SemVersion, bool) {
+	logger.Log.Infof("\nkubernetes-api\n--------------------")
+
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
+	if err != nil {
+		logger.Log.Errorf("%v can't initialize the client, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return nil, nil, false
+	}
+	logger.Log.Infof("%v can initialize the client", fmt.Sprintf(uiUtils.Green, "√"))
+
+	kubernetesVersion, err := kubernetesProvider.GetKubernetesVersion()
+	if err != nil {
+		logger.Log.Errorf("%v can't query the Kubernetes API, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return nil, nil, false
+	}
+	logger.Log.Infof("%v can query the Kubernetes API", fmt.Sprintf(uiUtils.Green, "√"))
+
+	return kubernetesProvider, kubernetesVersion, true
+}
+
+func checkMizuMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, bool) {
+	logger.Log.Infof("\nmizu-mode\n--------------------")
+
+	if exist, err := kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
+		logger.Log.Errorf("%v can't check mizu command, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return false, false
+	} else if exist {
+		logger.Log.Infof("%v mizu running with install command", fmt.Sprintf(uiUtils.Green, "√"))
+		return true, true
+	} else {
+		logger.Log.Infof("%v mizu running with tap command", fmt.Sprintf(uiUtils.Green, "√"))
+		return true, false
+	}
+}
+
+func checkKubernetesVersion(kubernetesVersion *semver.SemVersion) bool {
+	logger.Log.Infof("\nkubernetes-version\n--------------------")
+
+	if err := kubernetes.ValidateKubernetesVersion(kubernetesVersion); err != nil {
+		logger.Log.Errorf("%v not running the minimum Kubernetes API version, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return false
+	}
+
+	logger.Log.Infof("%v is running the minimum Kubernetes API version", fmt.Sprintf(uiUtils.Green, "√"))
+	return true
+}
+
+func checkServerConnection(kubernetesProvider *kubernetes.Provider) bool {
+	logger.Log.Infof("\nmizu-connectivity\n--------------------")
+
+	serverUrl := GetApiServerUrl()
+
+	apiServerProvider := apiserver.NewProvider(serverUrl, 1, apiserver.DefaultTimeout)
+	if err := apiServerProvider.TestConnection(); err == nil {
+		logger.Log.Infof("%v found Mizu server tunnel available and connected successfully to API server", fmt.Sprintf(uiUtils.Green, "√"))
+		return true
+	}
+
+	connectedToApiServer := false
+
+	if err := checkProxy(serverUrl, kubernetesProvider); err != nil {
+		logger.Log.Errorf("%v couldn't connect to API server using proxy, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+	} else {
+		connectedToApiServer = true
+		logger.Log.Infof("%v connected successfully to API server using proxy", fmt.Sprintf(uiUtils.Green, "√"))
+	}
+
+	if err := checkPortForward(serverUrl, kubernetesProvider); err != nil {
+		logger.Log.Errorf("%v couldn't connect to API server using port-forward, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+	} else {
+		connectedToApiServer = true
+		logger.Log.Infof("%v connected successfully to API server using port-forward", fmt.Sprintf(uiUtils.Green, "√"))
+	}
+
+	return connectedToApiServer
+}
+
+func checkProxy(serverUrl string, kubernetesProvider *kubernetes.Provider) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	httpServer, err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, cancel)
+	if err != nil {
+		return err
+	}
+
+	apiServerProvider := apiserver.NewProvider(serverUrl, apiserver.DefaultRetries, apiserver.DefaultTimeout)
+	if err := apiServerProvider.TestConnection(); err != nil {
+		return err
+	}
+
+	if err := httpServer.Shutdown(ctx); err != nil {
+		logger.Log.Debugf("Error occurred while stopping proxy, err: %v", err)
+	}
+
+	return nil
+}
+
+func checkPortForward(serverUrl string, kubernetesProvider *kubernetes.Provider) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	podRegex, _ := regexp.Compile(kubernetes.ApiServerPodName)
+	forwarder, err := kubernetes.NewPortForward(kubernetesProvider, config.Config.MizuResourcesNamespace, podRegex, config.Config.Tap.GuiPort, ctx, cancel)
+	if err != nil {
+		return err
+	}
+
+	apiServerProvider := apiserver.NewProvider(serverUrl, apiserver.DefaultRetries, apiserver.DefaultTimeout)
+	if err := apiServerProvider.TestConnection(); err != nil {
+		return err
+	}
+
+	forwarder.Close()
+
+	return nil
+}
+
+func checkAllResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider, isInstallCommand bool) bool {
+	logger.Log.Infof("\nmizu-existence\n--------------------")
+
+	exist, err := kubernetesProvider.DoesNamespaceExist(ctx, config.Config.MizuResourcesNamespace)
+	allResourcesExist := checkResourceExist(config.Config.MizuResourcesNamespace, "namespace", exist, err)
+
+	exist, err = kubernetesProvider.DoesConfigMapExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName)
+	allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err) && allResourcesExist
+
+	exist, err = kubernetesProvider.DoesServiceAccountExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName)
+	allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err) && allResourcesExist
+
+	if config.Config.IsNsRestrictedMode() {
+		exist, err = kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName)
+		allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err) && allResourcesExist
+
+		exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName)
+		allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err) && allResourcesExist
+	} else {
+		exist, err = kubernetesProvider.DoesClusterRoleExist(ctx, kubernetes.ClusterRoleName)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err) && allResourcesExist
+
+		exist, err = kubernetesProvider.DoesClusterRoleBindingExist(ctx, kubernetes.ClusterRoleBindingName)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err) && allResourcesExist
+	}
+
+	exist, err = kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+	allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err) && allResourcesExist
+
+	if isInstallCommand {
+		allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider) && allResourcesExist
+	} else {
+		allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider) && allResourcesExist
+	}
+
+	return allResourcesExist
+}
+
+func checkInstallResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
+	exist, err := kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleName)
+	installResourcesExist := checkResourceExist(kubernetes.DaemonRoleName, "role", exist, err)
+
+	exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName)
+	installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err) && installResourcesExist
+
+	exist, err = kubernetesProvider.DoesPersistentVolumeClaimExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName)
+	installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err) && installResourcesExist
+
+	exist, err = kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+	installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err) && installResourcesExist
+
+	return installResourcesExist
+}
+
+func checkTapResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
+	exist, err := kubernetesProvider.DoesPodExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+	tapResourcesExist := checkResourceExist(kubernetes.ApiServerPodName, "pod", exist, err)
+
+	return tapResourcesExist
+}
+
+func checkResourceExist(resourceName string, resourceType string, exist bool, err error) bool {
+	if err != nil {
+		logger.Log.Errorf("%v error checking if '%v' %v exists, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), resourceName, resourceType, err)
+		return false
+	} else if !exist {
+		logger.Log.Errorf("%v '%v' %v doesn't exist", fmt.Sprintf(uiUtils.Red, "✗"), resourceName, resourceType)
+		return false
+	} else {
+		logger.Log.Infof("%v '%v' %v exists", fmt.Sprintf(uiUtils.Green, "√"), resourceName, resourceType)
+	}
+
+	return true
+}

cli/cmd/cleanRunner.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 )
 
@@ -11,5 +10,5 @@ func performCleanCommand() {
 		return
 	}
 
-	finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout), config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+	finishMizuExecution(kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
 }

cli/cmd/common.go

@@ -5,19 +5,20 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
+	"regexp"
+	"time"
+
 	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/resources"
-	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
-	"path"
-	"time"
 
 	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
 )
@@ -26,15 +27,37 @@ func GetApiServerUrl() string {
 	return fmt.Sprintf("http://%s", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort))
 }
 
-func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
-	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx context.Context, cancel context.CancelFunc) {
+	httpServer, err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, cancel)
 	if err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
 			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
 		cancel()
+		return
 	}
 
-	logger.Log.Debugf("proxy ended")
+	apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
+	if err := apiProvider.TestConnection(); err != nil {
+		logger.Log.Debugf("Couldn't connect using proxy, stopping proxy and trying to create port-forward")
+		if err := httpServer.Shutdown(ctx); err != nil {
+			logger.Log.Debugf("Error occurred while stopping proxy %v", errormessage.FormatError(err))
+		}
+
+		podRegex, _ := regexp.Compile(kubernetes.ApiServerPodName)
+		if _, err := kubernetes.NewPortForward(kubernetesProvider, config.Config.MizuResourcesNamespace, podRegex, config.Config.Tap.GuiPort, ctx, cancel); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running port forward %v\n"+
+				"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
+			cancel()
+			return
+		}
+
+		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
+		if err := apiProvider.TestConnection(); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
+			cancel()
+			return
+		}
+	}
 }
 
 func getKubernetesProviderForCli() (*kubernetes.Provider, error) {
@@ -43,6 +66,23 @@ func getKubernetesProviderForCli() (*kubernetes.Provider, error) {
 		handleKubernetesProviderError(err)
 		return nil, err
 	}
+
+	if err := kubernetesProvider.ValidateNotProxy(); err != nil {
+		handleKubernetesProviderError(err)
+		return nil, err
+	}
+
+	kubernetesVersion, err := kubernetesProvider.GetKubernetesVersion()
+	if err != nil {
+		handleKubernetesProviderError(err)
+		return nil, err
+	}
+
+	if err := kubernetes.ValidateKubernetesVersion(kubernetesVersion); err != nil {
+		handleKubernetesProviderError(err)
+		return nil, err
+	}
+
 	return kubernetesProvider, nil
 }
 
@@ -55,8 +95,7 @@ func handleKubernetesProviderError(err error) {
 	}
 }
 
-func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) {
-	telemetry.ReportAPICalls(apiProvider)
+func finishMizuExecution(kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) {
 	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
 	defer cancel()
 	dumpLogsIfNeeded(removalCtx, kubernetesProvider)

cli/cmd/installRunner.go

@@ -4,10 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/shared/kubernetes"
-	core "k8s.io/api/core/v1"
-	"regexp"
-	"time"
 
 	"github.com/creasty/defaults"
 	"github.com/up9inc/mizu/cli/config"
@@ -47,6 +43,7 @@ func runMizuInstall() {
 	if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules,
 		serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(),
 		config.Config.MizuResourcesNamespace, config.Config.AgentImage,
+		config.Config.KratosImage, config.Config.KetoImage,
 		nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(),
 		config.Config.LogLevel(), false); err != nil {
 		var statusError *k8serrors.StatusError
@@ -62,20 +59,6 @@ func runMizuInstall() {
 		return
 	}
 
-	logger.Log.Infof("Waiting for Mizu server to start...")
-	readyChan := make(chan string)
-	readyErrorChan := make(chan error)
-	go watchApiServerPodReady(ctx, kubernetesProvider, readyChan, readyErrorChan)
-
-	select {
-	case readyMessage := <-readyChan:
-		logger.Log.Infof(readyMessage)
-	case err := <-readyErrorChan:
-		defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("%v", errormessage.FormatError(err)))
-		return
-	}
-
 	logger.Log.Infof(uiUtils.Magenta, "Installation completed, run `mizu view` to connect to the mizu daemon instance")
 }
 
@@ -91,63 +74,8 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
 		StandaloneMode:         true,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig
 }
-
-func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes.Provider, readyChan chan string, readyErrorChan chan error) {
-	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.ApiServerPodName))
-	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
-	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
-
-	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
-	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
-	for {
-		select {
-		case wEvent, ok := <-eventChan:
-			if !ok {
-				eventChan = nil
-				continue
-			}
-
-			switch wEvent.Type {
-			case kubernetes.EventAdded:
-				logger.Log.Debugf("Watching API Server pod ready loop, added")
-			case kubernetes.EventDeleted:
-				logger.Log.Debugf("Watching API Server pod ready loop, %s removed", kubernetes.ApiServerPodName)
-			case kubernetes.EventModified:
-				modifiedPod, err := wEvent.ToPod()
-				if err != nil {
-					readyErrorChan <- err
-					return
-				}
-
-				logger.Log.Debugf("Watching API Server pod ready loop, modified: %v", modifiedPod.Status.Phase)
-
-				if modifiedPod.Status.Phase == core.PodRunning {
-					readyChan <- fmt.Sprintf("%v pod is running", modifiedPod.Name)
-					return
-				}
-			case kubernetes.EventBookmark:
-				break
-			case kubernetes.EventError:
-				break
-			}
-		case err, ok := <-errorChan:
-			if !ok {
-				errorChan = nil
-				continue
-			}
-
-			readyErrorChan <- fmt.Errorf("[ERROR] Agent creation, watching %v namespace, error: %v", config.Config.MizuResourcesNamespace, err)
-			return
-		case <-timeAfter:
-			readyErrorChan <- fmt.Errorf("mizu API server was not ready in time")
-			return
-		case <-ctx.Done():
-			logger.Log.Debugf("Watching API Server pod ready loop, ctx done")
-			return
-		}
-	}
-}

cli/cmd/logs.go

@@ -23,7 +23,7 @@ var logsCmd = &cobra.Command{
 		if err != nil {
 			return nil
 		}
-		ctx, _ := context.WithCancel(context.Background())
+		ctx := context.Background()
 
 		if validationErr := config.Config.Logs.Validate(); validationErr != nil {
 			return errormessage.FormatError(validationErr)

cli/cmd/tap.go

@@ -12,7 +12,6 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
@@ -26,7 +25,6 @@ var tapCmd = &cobra.Command{
 	Long: `Record the ingoing traffic of a kubernetes pod.
 Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("tap", config.Config.Tap)
 		RunMizuTap()
 		return nil
 	},

cli/cmd/tapRunner.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/up9inc/mizu/cli/resources"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/utils"
 
 	"github.com/getkin/kin-openapi/openapi3"
@@ -23,7 +24,6 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -138,7 +138,7 @@ func RunMizuTap() {
 		return
 	}
 
-	defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+	defer finishTapExecution(kubernetesProvider)
 
 	go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel)
 	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
@@ -147,6 +147,12 @@ func RunMizuTap() {
 	utils.WaitForFinish(ctx, cancel)
 }
 
+func finishTapExecution(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportTapTelemetry(apiProvider, config.Config.Tap, state.startTime)
+
+	finishMizuExecution(kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+}
+
 func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 	mizuAgentConfig := shared.MizuAgentConfig{
 		MaxDBSizeBytes:         config.Config.Tap.MaxEntriesDBSizeBytes(),
@@ -158,6 +164,7 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		AgentDatabasePath:      shared.DataDirPath,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig
@@ -336,7 +343,7 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 
 				if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 					isPodReady = true
-					postApiServerStarted(ctx, kubernetesProvider, cancel, err)
+					postApiServerStarted(ctx, kubernetesProvider, cancel)
 				}
 			case kubernetes.EventBookmark:
 				break
@@ -399,7 +406,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 			case "FailedScheduling", "Failed":
 				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note))
 				cancel()
-				break
+				
 			}
 		case err, ok := <-errorChan:
 			if !ok {
@@ -415,21 +422,16 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 	}
 }
 
-func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) {
-	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel)
 
-	url := GetApiServerUrl()
-	if err := apiProvider.TestConnection(); err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
-		cancel()
-		return
-	}
 	options, _ := getMizuApiFilteringOptions()
-	if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
+	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
 		cancel()
 	}
 
+	url := GetApiServerUrl()
 	logger.Log.Infof("Mizu is available at %s", url)
 	if !config.Config.HeadlessMode {
 		uiUtils.OpenBrowser(url)

cli/cmd/viewRunner.go

@@ -27,7 +27,7 @@ func runMizuView() {
 	url := config.Config.View.Url
 
 	if url == "" {
-		exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+		exists, err := kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
 		if err != nil {
 			logger.Log.Errorf("Failed to found mizu service %v", err)
 			cancel()
@@ -47,8 +47,7 @@ func runMizuView() {
 			return
 		}
 		logger.Log.Infof("Establishing connection to k8s cluster...")
-		go startProxyReportErrorIfAny(kubernetesProvider, cancel)
-
+		startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel)
 	}
 
 	apiServerProvider := apiserver.NewProvider(url, apiserver.DefaultRetries, apiserver.DefaultTimeout)

cli/config/configStruct.go

@@ -2,14 +2,16 @@ package config
 
 import (
 	"fmt"
-	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/mizu"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/util/homedir"
 	"os"
 	"path"
 	"path/filepath"
+
+	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/shared"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/homedir"
 )
 
 const (
@@ -25,7 +27,9 @@ type ConfigStruct struct {
 	Logs                   configStructs.LogsConfig    `yaml:"logs"`
 	Auth                   configStructs.AuthConfig    `yaml:"auth"`
 	Config                 configStructs.ConfigConfig  `yaml:"config,omitempty"`
-	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
+	AgentImage             string                      `yaml:"agent-image,omitempty"`
+	KratosImage            string                      `yaml:"kratos-image,omitempty" readonly:""`
+	KetoImage              string                      `yaml:"keto-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
@@ -36,6 +40,7 @@ type ConfigStruct struct {
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
 	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
 	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
+	Elastic                shared.ElasticConfig        `yaml:"elastic"`
 }
 
 func (config *ConfigStruct) validate() error {
@@ -47,7 +52,9 @@ func (config *ConfigStruct) validate() error {
 }
 
 func (config *ConfigStruct) SetDefaults() {
-	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
+	config.KratosImage = shared.KratosImageDefault
+	config.KetoImage = shared.KetoImageDefault
+	config.AgentImage = fmt.Sprintf("%s:%s", shared.MizuAgentImageRepo, mizu.SemVer)
 	config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml")
 }
 

cli/go.sum

@@ -100,6 +100,7 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -336,6 +337,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

cli/resources/createResources.go

@@ -42,6 +42,8 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 		Namespace:             mizuResourcesNamespace,
 		PodName:               kubernetes.ApiServerPodName,
 		PodImage:              agentImage,
+		KratosImage:           "",
+		KetoImage:             "",
 		ServiceAccountName:    serviceAccountName,
 		IsNamespaceRestricted: isNsRestrictedMode,
 		SyncEntriesConfig:     syncEntriesConfig,
@@ -65,7 +67,7 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 	return mizuServiceAccountExists, nil
 }
 
-func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error {
+func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, kratosImage string, ketoImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error {
 	if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 		return err
 	}
@@ -95,6 +97,8 @@ func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kuberne
 		Namespace:             mizuResourcesNamespace,
 		PodName:               kubernetes.ApiServerPodName,
 		PodImage:              agentImage,
+		KratosImage:           kratosImage,
+		KetoImage:             ketoImage,
 		ServiceAccountName:    serviceAccountName,
 		IsNamespaceRestricted: isNsRestrictedMode,
 		SyncEntriesConfig:     syncEntriesConfig,
@@ -113,7 +117,7 @@ func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kuberne
 	if err != nil {
 		return err
 	}
-	logger.Log.Infof("service/%v created",  kubernetes.ApiServerPodName)
+	logger.Log.Infof("service/%v created", kubernetes.ApiServerPodName)
 
 	return nil
 }

cli/telemetry/telemetry.go

@@ -4,21 +4,21 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"net/http"
-	"os"
-
 	"github.com/denisbrodbeck/machineid"
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/shared/logger"
+	"net/http"
+	"os"
+	"time"
 )
 
 const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
 
 func ReportRun(cmd string, args interface{}) {
 	if !shouldRunTelemetry() {
-		logger.Log.Debugf("not reporting telemetry")
+		logger.Log.Debug("not reporting telemetry")
 		return
 	}
 
@@ -28,7 +28,7 @@ func ReportRun(cmd string, args interface{}) {
 		"args": string(argsBytes),
 	}
 
-	if err := sendTelemetry("Execution", argsMap); err != nil {
+	if err := sendTelemetry(argsMap); err != nil {
 		logger.Log.Debug(err)
 		return
 	}
@@ -36,30 +36,32 @@ func ReportRun(cmd string, args interface{}) {
 	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
 }
 
-func ReportAPICalls(apiProvider *apiserver.Provider) {
+func ReportTapTelemetry(apiProvider *apiserver.Provider, args interface{}, startTime time.Time) {
 	if !shouldRunTelemetry() {
-		logger.Log.Debugf("not reporting telemetry")
+		logger.Log.Debug("not reporting telemetry")
 		return
 	}
 
 	generalStats, err := apiProvider.GetGeneralStats()
 	if err != nil {
-		logger.Log.Debugf("[ERROR] failed get general stats from api server %v", err)
+		logger.Log.Debugf("[ERROR] failed to get general stats from api server %v", err)
 		return
 	}
-
+	argsBytes, _ := json.Marshal(args)
 	argsMap := map[string]interface{}{
-		"apiCallsCount":         generalStats["EntriesCount"],
-		"firstAPICallTimestamp": generalStats["FirstEntryTimestamp"],
-		"lastAPICallTimestamp":  generalStats["LastEntryTimestamp"],
+		"cmd":                    "tap",
+		"args":                   string(argsBytes),
+		"executionTimeInSeconds": int(time.Since(startTime).Seconds()),
+		"apiCallsCount":          generalStats["EntriesCount"],
+		"trafficVolumeInGB":      generalStats["EntriesVolumeInGB"],
 	}
 
-	if err := sendTelemetry("APICalls", argsMap); err != nil {
+	if err := sendTelemetry(argsMap); err != nil {
 		logger.Log.Debug(err)
 		return
 	}
 
-	logger.Log.Debugf("successfully reported telemetry of api calls")
+	logger.Log.Debug("successfully reported telemetry of tap command")
 }
 
 func shouldRunTelemetry() bool {
@@ -77,13 +79,12 @@ func shouldRunTelemetry() bool {
 	return true
 }
 
-func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error {
-	argsMap["telemetryType"] = telemetryType
+func sendTelemetry(argsMap map[string]interface{}) error {
 	argsMap["component"] = "mizu_cli"
 	argsMap["buildTimestamp"] = mizu.BuildTimestamp
 	argsMap["branch"] = mizu.Branch
 	argsMap["version"] = mizu.SemVer
-	argsMap["Platform"] = mizu.Platform
+	argsMap["platform"] = mizu.Platform
 
 	if machineId, err := machineid.ProtectedID("mizu"); err == nil {
 		argsMap["machineId"] = machineId

debug.Dockerfile

@@ -1,67 +0,0 @@
-# creates image in which mizu agent is remotely debuggable using delve
-FROM node:14-slim AS site-build
-
-WORKDIR /app/ui-build
-
-COPY ui/package.json .
-COPY ui/package-lock.json .
-RUN npm i
-COPY ui .
-RUN npm run build
-
-FROM golang:1.16-alpine AS builder
-# Set necessary environment variables needed for our image.
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
-
-RUN apk add libpcap-dev gcc g++ make bash perl-utils
-
-# Move to agent working directory (/agent-build).
-WORKDIR /app/agent-build
-
-COPY agent/go.mod agent/go.sum ./
-COPY shared/go.mod shared/go.mod ../shared/
-COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.* ../tap/api/
-RUN go mod download
-# cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
-
-ARG COMMIT_HASH
-ARG GIT_BRANCH
-ARG BUILD_TIMESTAMP
-ARG SEM_VER=0.0.0
-
-# Copy and build agent code
-COPY shared ../shared
-COPY tap ../tap
-COPY agent .
-# Include gcflags for debugging
-RUN go build -gcflags="all=-N -l" -o mizuagent .
-
-COPY devops/build_extensions_debug.sh ..
-RUN cd .. && /bin/bash build_extensions_debug.sh
-
-FROM golang:1.16-alpine
-
-# Set necessary environment variables needed for our image.
-RUN apk add bash libpcap-dev gcc g++
-
-WORKDIR /app
-
-# Copy binary and config files from /build to root folder of scratch container.
-COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent/build/extensions", "extensions"]
-COPY --from=site-build ["/app/ui-build/build", "site"]
-RUN mkdir /app/data/
-
-# install delve
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
-RUN go get github.com/go-delve/delve/cmd/dlv
-
-ENV GIN_MODE=debug
-
-# delve ports
-EXPOSE 2345 2346
-
-# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
-ENTRYPOINT "/app/mizuagent"

deploy/kubernetes/helm-chart/values.yaml

@@ -21,23 +21,23 @@ pod:
 container:
   mizuAgent:
     image:
-      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/mizufree"
-      tag: "0.21.29"
+      repository: "gcr.io/up9-docker-hub/mizu/main"
+      tag: "0.22.0"
   tapper:
     image:
-      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/mizufree"
-      tag: "0.21.29"
+      repository: "gcr.io/up9-docker-hub/mizu/main"
+      tag: "0.22.0"
   basenine:
     name: "basenine"
     port: 9099
     image:
-      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/basenine"
+      repository: "ghcr.io/up9inc/basenine"
       tag: "v0.3.0"
   kratos:
     name: "kratos"
     port: 4433
     image:
-      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/kratos"
+      repository: "gcr.io/up9-docker-hub/mizu-kratos/stable"
       tag: "0.0.0"
 
 deployment:

devops/build_extensions.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-for f in tap/extensions/*; do
-    if [ -d "$f" ]; then
-        echo Building extension: $f
-        extension=$(basename $f) && \
-        cd tap/extensions/${extension} && \
-        go build -buildmode=plugin -o ../${extension}.so .  && \
-        cd ../../..  && \
-        mkdir -p agent/build/extensions  && \
-        cp tap/extensions/${extension}.so agent/build/extensions
-    fi
-done

devops/build_extensions_debug.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-for f in tap/extensions/*; do
-    if [ -d "$f" ]; then
-        extension=$(basename $f) && \
-        cd tap/extensions/${extension} && \
-        go build -gcflags="all=-N -l" -buildmode=plugin -o ../${extension}.so .  && \
-        cd ../../..  && \
-        mkdir -p agent/build/extensions  && \
-        cp tap/extensions/${extension}.so agent/build/extensions
-    fi
-done

devops/linux-arm64-musl-go-libpcap/Dockerfile

@@ -0,0 +1,18 @@
+FROM dockcross/linux-arm64-musl:latest AS builder-from-amd64-to-arm64v8
+
+# Install Go
+RUN curl https://go.dev/dl/go1.16.13.linux-amd64.tar.gz -Lo ./go.linux-amd64.tar.gz
+RUN curl https://go.dev/dl/go1.16.13.linux-amd64.tar.gz.asc -Lo ./go.linux-amd64.tar.gz.asc
+RUN curl https://dl.google.com/dl/linux/linux_signing_key.pub  -Lo linux_signing_key.pub
+RUN gpg --import linux_signing_key.pub && gpg --verify ./go.linux-amd64.tar.gz.asc ./go.linux-amd64.tar.gz
+RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go.linux-amd64.tar.gz
+ENV PATH "$PATH:/usr/local/go/bin"
+
+# Compile libpcap from source
+RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz -Lo ./libpcap.tar.gz
+RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz.sig -Lo ./libpcap.tar.gz.sig
+RUN curl https://www.tcpdump.org/release/signing-key.asc -Lo ./signing-key.asc
+RUN gpg --import signing-key.asc && gpg --verify libpcap.tar.gz.sig libpcap.tar.gz
+RUN tar -xzf libpcap.tar.gz && mv ./libpcap-* ./libpcap
+RUN cd ./libpcap && ./configure --host=arm && make
+RUN cp /work/libpcap/libpcap.a /usr/xcc/aarch64-linux-musl-cross/lib/gcc/aarch64-linux-musl/*/

devops/linux-arm64-musl-go-libpcap/build-push.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+docker build . -t up9inc/linux-arm64-musl-go-libpcap && docker push up9inc/linux-arm64-musl-go-libpcap

shared/consts.go

@@ -14,8 +14,9 @@ const (
 	GoGCEnvVar                       = "GOGC"
 	DefaultApiServerPort             = 8899
 	LogLevelEnvVar                   = "LOG_LEVEL"
+	MizuAgentImageRepo               = "docker.io/up9inc/mizu"
 	BasenineHost                     = "127.0.0.1"
 	BaseninePort                     = "9099"
-	BasenineImageRepo                = "ghcr.io/up9inc/basenine"
-	BasenineImageTag                 = "v0.3.0"
+	KratosImageDefault               = "gcr.io/up9-docker-hub/mizu-kratos/stable:0.0.0"
+	KetoImageDefault                 = "gcr.io/up9-docker-hub/mizu-keto/stable:0.0.0"
 )

shared/kubernetes/provider.go

@@ -76,14 +76,6 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
 
-	if err := validateNotProxy(kubernetesConfig, restClientConfig); err != nil {
-		return nil, err
-	}
-
-	if err := validateKubernetesVersion(clientSet); err != nil {
-		return nil, err
-	}
-
 	return &Provider{
 		clientSet:        clientSet,
 		kubernetesConfig: kubernetesConfig,
@@ -177,6 +169,8 @@ type ApiServerOptions struct {
 	Namespace             string
 	PodName               string
 	PodImage              string
+	KratosImage           string
+	KetoImage             string
 	ServiceAccountName    string
 	IsNamespaceRestricted bool
 	SyncEntriesConfig     *shared.SyncEntriesConfig
@@ -280,7 +274,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 		},
 		{
 			Name:            "basenine",
-			Image:           fmt.Sprintf("%s:%s", shared.BasenineImageRepo, shared.BasenineImageTag),
+			Image:           opts.PodImage,
 			ImagePullPolicy: opts.ImagePullPolicy,
 			VolumeMounts:    volumeMounts,
 			ReadinessProbe: &core.Probe{
@@ -304,7 +298,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 					"memory": memRequests,
 				},
 			},
-			Command:    []string{"/basenine"},
+			Command:    []string{"basenine"},
 			Args:       []string{"-addr", "0.0.0.0", "-port", shared.BaseninePort, "-persistent"},
 			WorkingDir: shared.DataDirPath,
 		},
@@ -313,7 +307,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 	if createAuthContainer {
 		containers = append(containers, core.Container{
 			Name:            "kratos",
-			Image:           "gcr.io/up9-docker-hub/mizu-kratos/stable:0.0.0",
+			Image:           opts.KratosImage,
 			ImagePullPolicy: opts.ImagePullPolicy,
 			VolumeMounts:    volumeMounts,
 			ReadinessProbe: &core.Probe{
@@ -341,6 +335,35 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 			},
 		})
 
+		containers = append(containers, core.Container{
+			Name:            "keto",
+			Image:           opts.KetoImage,
+			ImagePullPolicy: opts.ImagePullPolicy,
+			VolumeMounts:    volumeMounts,
+			ReadinessProbe: &core.Probe{
+				FailureThreshold: 3,
+				Handler: core.Handler{
+					HTTPGet: &core.HTTPGetAction{
+						Path:   "/health/ready",
+						Port:   intstr.FromInt(4466),
+						Scheme: core.URISchemeHTTP,
+					},
+				},
+				PeriodSeconds:    1,
+				SuccessThreshold: 1,
+				TimeoutSeconds:   1,
+			},
+			Resources: core.ResourceRequirements{
+				Limits: core.ResourceList{
+					"cpu":    cpuLimit,
+					"memory": memLimit,
+				},
+				Requests: core.ResourceList{
+					"cpu":    cpuRequests,
+					"memory": memRequests,
+				},
+			},
+		})
 	}
 
 	pod := &core.Pod{
@@ -416,11 +439,61 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 	return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
 }
 
-func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) {
+func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) {
+	namespaceResource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(namespaceResource, err)
+}
+
+func (provider *Provider) DoesConfigMapExist(ctx context.Context, namespace string, name string) (bool, error) {
+	configMapResource, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(configMapResource, err)
+}
+
+func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, name string) (bool, error) {
+	serviceAccountResource, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(serviceAccountResource, err)
+}
+
+func (provider *Provider) DoesPersistentVolumeClaimExist(ctx context.Context, namespace string, name string) (bool, error) {
+	persistentVolumeClaimResource, err := provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(persistentVolumeClaimResource, err)
+}
+
+func (provider *Provider) DoesDeploymentExist(ctx context.Context, namespace string, name string) (bool, error) {
+	deploymentResource, err := provider.clientSet.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(deploymentResource, err)
+}
+
+func (provider *Provider) DoesPodExist(ctx context.Context, namespace string, name string) (bool, error) {
+	podResource, err := provider.clientSet.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(podResource, err)
+}
+
+func (provider *Provider) DoesServiceExist(ctx context.Context, namespace string, name string) (bool, error) {
 	serviceResource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
 	return provider.doesResourceExist(serviceResource, err)
 }
 
+func (provider *Provider) DoesClusterRoleExist(ctx context.Context, name string) (bool, error) {
+	clusterRoleResource, err := provider.clientSet.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(clusterRoleResource, err)
+}
+
+func (provider *Provider) DoesClusterRoleBindingExist(ctx context.Context, name string) (bool, error) {
+	clusterRoleBindingResource, err := provider.clientSet.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(clusterRoleBindingResource, err)
+}
+
+func (provider *Provider) DoesRoleExist(ctx context.Context, namespace string, name string) (bool, error) {
+	roleResource, err := provider.clientSet.RbacV1().Roles(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(roleResource, err)
+}
+
+func (provider *Provider) DoesRoleBindingExist(ctx context.Context, namespace string, name string) (bool, error) {
+	roleBindingResource, err := provider.clientSet.RbacV1().RoleBindings(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(roleBindingResource, err)
+}
+
 func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) {
 	// Getting NotFound error is the expected behavior when a resource does not exist.
 	if k8serrors.IsNotFound(err) {
@@ -1042,6 +1115,45 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names
 	return provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, volumeClaim, metav1.CreateOptions{})
 }
 
+// ValidateNotProxy We added this after a customer tried to run mizu from lens, which used len's kube config, which have cluster server configuration, which points to len's local proxy.
+// The workaround was to use the user's local default kube config.
+// For now - we are blocking the option to run mizu through a proxy to k8s server
+func (provider *Provider) ValidateNotProxy() error {
+	kubernetesUrl, err := url.Parse(provider.clientConfig.Host)
+	if err != nil {
+		logger.Log.Debugf("ValidateNotProxy - error while parsing kubernetes host, err: %v", err)
+		return nil
+	}
+
+	restProxyClientConfig, _ := provider.kubernetesConfig.ClientConfig()
+	restProxyClientConfig.Host = kubernetesUrl.Host
+
+	clientProxySet, err := getClientSet(restProxyClientConfig)
+	if err == nil {
+		proxyServerVersion, err := clientProxySet.ServerVersion()
+		if err != nil {
+			return nil
+		}
+
+		if *proxyServerVersion == (version.Info{}) {
+			return &ClusterBehindProxyError{}
+		}
+	}
+
+	return nil
+}
+
+func (provider *Provider) GetKubernetesVersion() (*semver.SemVersion, error) {
+	serverVersion, err := provider.clientSet.ServerVersion()
+	if err != nil {
+		logger.Log.Debugf("error while getting kubernetes server version, err: %v", err)
+		return nil, err
+	}
+
+	serverVersionSemVer := semver.SemVersion(serverVersion.GitVersion)
+	return &serverVersionSemVer, nil
+}
+
 func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) {
 	clientSet, err := kubernetes.NewForConfig(config)
 	if err != nil {
@@ -1051,6 +1163,15 @@ func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) {
 	return clientSet, nil
 }
 
+func ValidateKubernetesVersion(serverVersionSemVer *semver.SemVersion) error {
+	minKubernetesServerVersionSemVer := semver.SemVersion(MinKubernetesServerVersion)
+	if minKubernetesServerVersionSemVer.GreaterThan(*serverVersionSemVer) {
+		return fmt.Errorf("kubernetes server version %v is not supported, supporting only kubernetes server version of %v or higher", serverVersionSemVer, MinKubernetesServerVersion)
+	}
+
+	return nil
+}
+
 func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
 	logger.Log.Debugf("Using kube config %s", kubeConfigPath)
 	configPathList := filepath.SplitList(kubeConfigPath)
@@ -1072,47 +1193,3 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
 func isPodRunning(pod *core.Pod) bool {
 	return pod.Status.Phase == core.PodRunning
 }
-
-// We added this after a customer tried to run mizu from lens, which used len's kube config, which have cluster server configuration, which points to len's local proxy.
-// The workaround was to use the user's local default kube config.
-// For now - we are blocking the option to run mizu through a proxy to k8s server
-func validateNotProxy(kubernetesConfig clientcmd.ClientConfig, restClientConfig *restclient.Config) error {
-	kubernetesUrl, err := url.Parse(restClientConfig.Host)
-	if err != nil {
-		logger.Log.Debugf("validateNotProxy - error while parsing kubernetes host, err: %v", err)
-		return nil
-	}
-
-	restProxyClientConfig, _ := kubernetesConfig.ClientConfig()
-	restProxyClientConfig.Host = kubernetesUrl.Host
-
-	clientProxySet, err := getClientSet(restProxyClientConfig)
-	if err == nil {
-		proxyServerVersion, err := clientProxySet.ServerVersion()
-		if err != nil {
-			return nil
-		}
-
-		if *proxyServerVersion == (version.Info{}) {
-			return &ClusterBehindProxyError{}
-		}
-	}
-
-	return nil
-}
-
-func validateKubernetesVersion(clientSet *kubernetes.Clientset) error {
-	serverVersion, err := clientSet.ServerVersion()
-	if err != nil {
-		logger.Log.Debugf("error while getting kubernetes server version, err: %v", err)
-		return nil
-	}
-
-	serverVersionSemVer := semver.SemVersion(serverVersion.GitVersion)
-	minKubernetesServerVersionSemVer := semver.SemVersion(MinKubernetesServerVersion)
-	if minKubernetesServerVersionSemVer.GreaterThan(serverVersionSemVer) {
-		return fmt.Errorf("kubernetes server version %v is not supported, supporting only kubernetes server version of %v or higher", serverVersion.GitVersion, MinKubernetesServerVersion)
-	}
-
-	return nil
-}

shared/kubernetes/proxy.go

@@ -1,9 +1,17 @@
 package kubernetes
 
 import (
+	"bytes"
+	"context"
 	"fmt"
+	"github.com/up9inc/mizu/shared"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/client-go/tools/portforward"
+	"k8s.io/client-go/transport/spdy"
 	"net"
 	"net/http"
+	"net/url"
+	"regexp"
 	"strings"
 	"time"
 
@@ -14,8 +22,8 @@ import (
 const k8sProxyApiPrefix = "/"
 const mizuServicePort = 80
 
-func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
-	logger.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
+func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16, mizuNamespace string, mizuServiceName string, cancel context.CancelFunc) (*http.Server, error) {
+	logger.Log.Debugf("Starting proxy using proxy method. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
 	filter := &proxy.FilterServer{
 		AcceptPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
 		RejectPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),
@@ -25,7 +33,7 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
 
 	proxyHandler, err := proxy.NewProxyHandler(k8sProxyApiPrefix, filter, &kubernetesProvider.clientConfig, time.Second*2)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	mux := http.NewServeMux()
 	mux.Handle(k8sProxyApiPrefix, getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
@@ -33,14 +41,21 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
 
 	l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", proxyHost, int(mizuPort)))
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	server := http.Server{
+	server := &http.Server{
 		Handler: mux,
 	}
 
-	return server.Serve(l)
+	go func() {
+		if err := server.Serve(l); err != nil && err != http.ErrServerClosed {
+			logger.Log.Errorf("Error creating proxy, %v", err)
+			cancel()
+		}
+	}()
+
+	return server, nil
 }
 
 func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
@@ -69,3 +84,52 @@ func getRerouteHttpHandlerMizuStatic(proxyHandler http.Handler, mizuNamespace st
 		proxyHandler.ServeHTTP(w, r)
 	})
 }
+
+func NewPortForward(kubernetesProvider *Provider, namespace string, podRegex *regexp.Regexp, localPort uint16, ctx context.Context, cancel context.CancelFunc) (*portforward.PortForwarder, error) {
+	pods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, podRegex, []string{namespace})
+	if err != nil {
+		return nil, err
+	} else if len(pods) == 0 {
+		return nil, fmt.Errorf("didn't find pod to port-forward")
+	}
+
+	podName := pods[0].Name
+
+	logger.Log.Debugf("Starting proxy using port-forward method. namespace: [%v], pod name: [%s], port: [%v]", namespace, podName, localPort)
+
+	dialer, err := getHttpDialer(kubernetesProvider, namespace, podName)
+	if err != nil {
+		return nil, err
+	}
+
+	stopChan, readyChan := make(chan struct{}, 1), make(chan struct{}, 1)
+	out, errOut := new(bytes.Buffer), new(bytes.Buffer)
+
+	forwarder, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", localPort, shared.DefaultApiServerPort)}, stopChan, readyChan, out, errOut)
+	if err != nil {
+		return nil, err
+	}
+
+	go func() {
+		if err = forwarder.ForwardPorts(); err != nil {
+			logger.Log.Errorf("kubernetes port-forwarding error: %v", err)
+			cancel()
+		}
+	}()
+
+	return forwarder, nil
+}
+
+func getHttpDialer(kubernetesProvider *Provider, namespace string, podName string) (httpstream.Dialer, error) {
+	roundTripper, upgrader, err := spdy.RoundTripperFor(&kubernetesProvider.clientConfig)
+	if err != nil {
+		logger.Log.Errorf("Error creating http dialer")
+		return nil, err
+	}
+
+	path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", namespace, podName)
+	hostIP := strings.TrimLeft(kubernetesProvider.clientConfig.Host, "htps:/") // no need specify "t" twice
+	serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP}
+
+	return spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &serverURL), nil
+}

shared/models.go

@@ -43,6 +43,13 @@ type MizuAgentConfig struct {
 	StandaloneMode         bool          `json:"standaloneMode"`
 	ServiceMap             bool          `json:"serviceMap"`
 	OAS                    bool          `json:"oas"`
+	Elastic                ElasticConfig `json:"elastic"`
+}
+
+type ElasticConfig struct {
+	User     string `yaml:"user,omitempty" default:"" readonly:""`
+	Password string `yaml:"password,omitempty" default:"" readonly:""`
+	Url      string `yaml:"url,omitempty" default:"" readonly:""`
 }
 
 type WebSocketMessageMetadata struct {

tap/api/api.go

@@ -6,13 +6,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/google/martian/har"
 	"io/ioutil"
 	"net/http"
-	"plugin"
 	"sync"
 	"time"
-
-	"github.com/google/martian/har"
 )
 
 type Protocol struct {
@@ -38,7 +36,6 @@ type TCP struct {
 type Extension struct {
 	Protocol   *Protocol
 	Path       string
-	Plug       *plugin.Plugin
 	Dissector  Dissector
 	MatcherMap *sync.Map
 }

tap/extensions/amqp/helpers.go

@@ -1,4 +1,4 @@
-package main
+package amqp
 
 import (
 	"encoding/json"

tap/extensions/amqp/main.go

@@ -1,4 +1,4 @@
-package main
+package amqp
 
 import (
 	"bufio"
@@ -328,3 +328,7 @@ func (d dissecting) Macros() map[string]string {
 }
 
 var Dissector dissecting
+
+func NewDissector() api.Dissector {
+	return Dissector
+}

tap/extensions/amqp/read.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 // Source code and contact info at http://github.com/streadway/amqp
 
-package main
+package amqp
 
 import (
 	"bytes"

tap/extensions/amqp/spec091.go

@@ -6,7 +6,7 @@
 /* GENERATED FILE - DO NOT EDIT */
 /* Rebuild from the spec/gen.go tool */
 
-package main
+package amqp
 
 import (
 	"encoding/binary"

tap/extensions/amqp/structs.go

@@ -1,4 +1,4 @@
-package main
+package amqp
 
 import (
 	"encoding/json"

tap/extensions/amqp/types.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 // Source code and contact info at http://github.com/streadway/amqp
 
-package main
+package amqp
 
 import (
 	"fmt"

tap/extensions/amqp/write.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 // Source code and contact info at http://github.com/streadway/amqp
 
-package main
+package amqp
 
 import (
 	"bufio"

tap/extensions/http/handlers.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"bufio"
@@ -66,7 +66,7 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 			streamID,
 			"HTTP2",
 		)
-		item = reqResMatcher.registerRequest(ident, &messageHTTP1, superTimer.CaptureTime)
+		item = reqResMatcher.registerRequest(ident, &messageHTTP1, superTimer.CaptureTime, messageHTTP1.ProtoMinor)
 		if item != nil {
 			item.ConnectionInfo = &api.ConnectionInfo{
 				ClientIP:   tcpID.SrcIP,
@@ -86,7 +86,7 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 			streamID,
 			"HTTP2",
 		)
-		item = reqResMatcher.registerResponse(ident, &messageHTTP1, superTimer.CaptureTime)
+		item = reqResMatcher.registerResponse(ident, &messageHTTP1, superTimer.CaptureTime, messageHTTP1.ProtoMinor)
 		if item != nil {
 			item.ConnectionInfo = &api.ConnectionInfo{
 				ClientIP:   tcpID.DstIP,
@@ -135,7 +135,7 @@ func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 		counterPair.Request,
 		"HTTP1",
 	)
-	item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime)
+	item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime, req.ProtoMinor)
 	if item != nil {
 		item.ConnectionInfo = &api.ConnectionInfo{
 			ClientIP:   tcpID.SrcIP,
@@ -175,7 +175,7 @@ func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 		counterPair.Response,
 		"HTTP1",
 	)
-	item := reqResMatcher.registerResponse(ident, res, superTimer.CaptureTime)
+	item := reqResMatcher.registerResponse(ident, res, superTimer.CaptureTime, res.ProtoMinor)
 	if item != nil {
 		item.ConnectionInfo = &api.ConnectionInfo{
 			ClientIP:   tcpID.DstIP,

tap/extensions/http/helpers.go

@@ -1,8 +1,9 @@
-package main
+package http
 
 import (
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"strconv"
 
 	"github.com/up9inc/mizu/tap/api"
@@ -18,16 +19,55 @@ func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}
 	return
 }
 
+func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{}) {
+	newMapSlice = make([]interface{}, 0)
+	valuesMap := make(map[string][]interface{})
+	for _, item := range mapSlice {
+		h := item.(map[string]interface{})
+		key := h["name"].(string)
+		valuesMap[key] = append(valuesMap[key], h["value"])
+	}
+
+	for key, values := range valuesMap {
+		h := make(map[string]interface{})
+		h["name"] = key
+		if len(values) == 1 {
+			h["value"] = values[0]
+		} else {
+			h["value"] = values
+		}
+		newMapSlice = append(newMapSlice, h)
+	}
+
+	return
+}
+
 func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
 	for _, item := range mapSlice {
 		h := item.(map[string]interface{})
-		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string))
-		table = append(table, api.TableData{
-			Name:     h["name"].(string),
-			Value:    h["value"],
-			Selector: selector,
-		})
+		key := h["name"].(string)
+		value := h["value"]
+		switch reflect.TypeOf(value).Kind() {
+		case reflect.Slice:
+			fallthrough
+		case reflect.Array:
+			for i, el := range value.([]interface{}) {
+				selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i)
+				table = append(table, api.TableData{
+					Name:     fmt.Sprintf("%s [%d]", key, i),
+					Value:    el,
+					Selector: selector,
+				})
+			}
+		default:
+			selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
+			table = append(table, api.TableData{
+				Name:     key,
+				Value:    value,
+				Selector: selector,
+			})
+		}
 	}
 
 	obj, _ := json.Marshal(table)

tap/extensions/http/http2_assembler.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"bufio"
@@ -235,8 +235,8 @@ func checkIsHTTP2ServerStream(b *bufio.Reader) (bool, error) {
 		return false, err
 	}
 
-	// If response starts with this text, it is HTTP/1.x
-	if bytes.Compare(buf, []byte("HTTP/1.0 ")) == 0 || bytes.Compare(buf, []byte("HTTP/1.1 ")) == 0 {
+	// If response starts with HTTP/1. then it's not HTTP/2
+	if bytes.HasPrefix(buf, []byte("HTTP/1.")) {
 		return false, nil
 	}