fixed redact acceptance test (#472)

* Update main.go, status_controller.go, and 2 more files...

* Update status_controller.go and mizuTapperSyncer.go

.github/workflows/inactive-issues-close.yaml

@@ -0,0 +1,22 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v3
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -29,3 +29,6 @@ build
 
 # pprof
 pprof/*
+
+# Database Files
+*.bin

Dockerfile

@@ -13,7 +13,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash
+RUN apk add libpcap-dev gcc g++ make bash perl-utils
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -24,7 +24,7 @@ COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
+RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 ARG COMMIT_HASH
 ARG GIT_BRANCH
@@ -41,16 +41,24 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
+# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.9/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.9/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+RUN shasum -a 256 -c basenine_linux_amd64.sha256
+RUN chmod +x ./basenine_linux_amd64
+
 COPY devops/build_extensions.sh ..
 RUN cd .. && /bin/bash build_extensions.sh
 
 FROM alpine:3.14
 
 RUN apk add bash libpcap-dev tcpdump
+
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 RUN mkdir /app/data/

README.md

@@ -83,7 +83,7 @@ To tap all pods in current namespace -
 ```
 
 
-To tap specific pod - 
+### To tap specific pod
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -96,7 +96,7 @@ To tap specific pod -
  ^C
 ```
 
-To tap multiple pods using regex - 
+### To tap multiple pods using regex
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -111,6 +111,27 @@ To tap multiple pods using regex -
  Web interface is now available at http://localhost:8899
  ^C
 ```
+### To run mizu mizu daemon mode (detached from cli)
+```bash
+$ mizu tap "^ca.*" --daemon
+  Mizu will store up to 200MB of traffic, old traffic will be cleared once the limit is reached.
+  Tapping pods in namespaces "sock-shop"
+  Waiting for mizu to be ready... (may take a few minutes)
+  +carts-66c77f5fbb-fq65r
+  +catalogue-5f4cb7cf5-7zrmn
+  ..
+
+$ mizu view
+  Establishing connection to k8s cluster...
+  Mizu is available at http://localhost:8899
+  ^C
+  ..
+
+$ mizu clean # mizu will continue running in cluster until clean is executed
+  Removing mizu resources
+```
+
+`mizu view` provides one way to access Mizu. For other options, see [Accessing Mizu Wiki Page](https://github.com/up9inc/mizu/wiki/Accessing-Mizu).
 
 ## Configuration
 
@@ -184,6 +205,8 @@ and when changed it will support accessing by IP
 
 Mizu can be ran detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run indefinitely in the cluster.
 
+Please note that daemon mode requires you to have RBAC creation permissions, see the [permissions](docs/PERMISSIONS.md) doc for more details.
+
 In order to access a daemon mizu you will have to run `mizu view` after running the `tap --daemon` command.
 
 To stop the detached mizu instance and clean all cluster side resources, run `mizu clean`

acceptanceTests/Makefile

@@ -1,2 +1,2 @@
 test: ## Run acceptance tests.
-	@go test ./... -timeout 1h
+	@go test ./... -timeout 1h -v

acceptanceTests/config_test.go

@@ -2,11 +2,12 @@ package acceptanceTests
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"testing"
+
+	"gopkg.in/yaml.v3"
 )
 
 type tapConfig struct {

acceptanceTests/go.mod

@@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tests
 go 1.16
 
 require (
+	github.com/gorilla/websocket v1.4.2
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

acceptanceTests/go.sum

@@ -211,6 +211,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -303,6 +304,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=

acceptanceTests/tap_test.go

@@ -66,21 +66,18 @@ func TestTap(t *testing.T) {
 			entriesCheckFunc := func() error {
 				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-				entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
-				requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-				if requestErr != nil {
-					return fmt.Errorf("failed to get entries, err: %v", requestErr)
+				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
+				if err != nil {
+					return err
 				}
-
-				entries := requestResult.([]interface{})
-				if len(entries) == 0 {
-					return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+				err = checkEntriesAtLeast(entries, 1)
+				if err != nil {
+					return err
 				}
-
-				entry :=  entries[0].(map[string]interface{})
+				entry := entries[0]
 
 				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
-				requestResult, requestErr = executeHttpGetRequest(entryUrl)
+				requestResult, requestErr := executeHttpGetRequest(entryUrl)
 				if requestErr != nil {
 					return fmt.Errorf("failed to get entry, err: %v", requestErr)
 				}
@@ -150,10 +147,7 @@ func TestTapAllNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
@@ -202,19 +196,7 @@ func TestTapAllNamespaces(t *testing.T) {
 	}
 
 	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
 			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
 			return
 		}
@@ -226,10 +208,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
 		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
@@ -288,19 +267,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 	}
 
 	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
 			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
 			return
 		}
@@ -313,10 +280,7 @@ func TestTapRegex(t *testing.T) {
 	}
 
 	regexPodName := "httpbin2"
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: regexPodName, Namespace: "mizu-tests"},
 	}
 
@@ -371,19 +335,7 @@ func TestTapRegex(t *testing.T) {
 	}
 
 	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
 			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
 			return
 		}
@@ -431,7 +383,7 @@ func TestTapDryRun(t *testing.T) {
 		resultChannel <- "fail"
 	}()
 
-	testResult := <- resultChannel
+	testResult := <-resultChannel
 	if testResult != "success" {
 		t.Errorf("unexpected result - dry run cmd not done")
 	}
@@ -475,9 +427,10 @@ func TestTapRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
@@ -486,51 +439,39 @@ func TestTapRedact(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
+		firstEntry := entries[0]
 
 		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		requestResult, requestErr := executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
+		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		request := entry["request"].(map[string]interface{})
 
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		headers :=  entryDetails["headers"].([]interface{})
+		headers := request["_headers"].([]interface{})
 		for _, headerInterface := range headers {
 			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Agent" {
+			if header["name"].(string) != "User-Header" {
 				continue
 			}
 
-			userAgent := header["value"].(string)
-			if userAgent != "[REDACTED]" {
+			userHeader := header["value"].(string)
+			if userHeader != "[REDACTED]" {
 				return fmt.Errorf("unexpected result - user agent is not redacted")
 			}
 		}
 
-		postData := entryDetails["postData"].(map[string]interface{})
+		postData := request["postData"].(map[string]interface{})
 		textDataStr := postData["text"].(string)
 
 		var textData map[string]string
@@ -590,9 +531,10 @@ func TestTapNoRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
@@ -601,51 +543,39 @@ func TestTapNoRedact(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
+		firstEntry := entries[0]
 
 		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		requestResult, requestErr := executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
+		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		request := entry["request"].(map[string]interface{})
 
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		headers :=  entryDetails["headers"].([]interface{})
+		headers := request["_headers"].([]interface{})
 		for _, headerInterface := range headers {
 			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Agent" {
+			if header["name"].(string) != "User-Header" {
 				continue
 			}
 
-			userAgent := header["value"].(string)
-			if userAgent == "[REDACTED]" {
+			userHeader := header["value"].(string)
+			if userHeader == "[REDACTED]" {
 				return fmt.Errorf("unexpected result - user agent is redacted")
 			}
 		}
 
-		postData := entryDetails["postData"].(map[string]interface{})
+		postData := request["postData"].(map[string]interface{})
 		textDataStr := postData["text"].(string)
 
 		var textData map[string]string
@@ -716,38 +646,26 @@ func TestTapRegexMasking(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
+		firstEntry := entries[0]
 
 		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		requestResult, requestErr := executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
+		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		request := entry["request"].(map[string]interface{})
 
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		postData := entryDetails["postData"].(map[string]interface{})
+		postData := request["postData"].(map[string]interface{})
 		textData := postData["text"].(string)
 
 		if textData != "[REDACTED]" {
@@ -805,7 +723,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 
 	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
-	headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
+	headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
 	for i := 0; i < defaultEntriesCount; i++ {
 		if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
@@ -823,38 +741,27 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	ignoredUserAgentsCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
 
 		for _, entryInterface := range entries {
-			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
-			requestResult, requestErr = executeHttpGetRequest(entryUrl)
+			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"])
+			requestResult, requestErr := executeHttpGetRequest(entryUrl)
 			if requestErr != nil {
 				return fmt.Errorf("failed to get entry, err: %v", requestErr)
 			}
 
-			data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-			entryJson := data["entry"].(string)
+			entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+			request := entry["request"].(map[string]interface{})
 
-			var entry map[string]interface{}
-			if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-				return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-			}
-
-			entryRequest := entry["request"].(map[string]interface{})
-			entryPayload := entryRequest["payload"].(map[string]interface{})
-			entryDetails := entryPayload["details"].(map[string]interface{})
-
-			entryHeaders :=  entryDetails["headers"].([]interface{})
-			for _, headerInterface := range entryHeaders {
+			headers := request["_headers"].([]interface{})
+			for _, headerInterface := range headers {
 				header := headerInterface.(map[string]interface{})
 				if header["name"].(string) != ignoredUserAgentCustomHeader {
 					continue
@@ -922,21 +829,21 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 
-	var dumpsLogsPath string
+	var dumpLogsPath string
 	for _, file := range files {
 		fileName := file.Name()
 		if strings.Contains(fileName, "mizu_logs") {
-			dumpsLogsPath = path.Join(mizuFolderPath, fileName)
+			dumpLogsPath = path.Join(mizuFolderPath, fileName)
 			break
 		}
 	}
 
-	if dumpsLogsPath == "" {
+	if dumpLogsPath == "" {
 		t.Errorf("dump logs file not found")
 		return
 	}
 
-	zipReader, zipError := zip.OpenReader(dumpsLogsPath)
+	zipReader, zipError := zip.OpenReader(dumpLogsPath)
 	if zipError != nil {
 		t.Errorf("failed to get zip reader, err: %v", zipError)
 		return
@@ -973,3 +880,251 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 }
+
+func TestDaemonSeeTraffic(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []int{50}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			cliPath, cliPathErr := getCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapDaemonCmdArgs := getDefaultTapCommandArgsWithDaemonMode()
+
+			tapNamespace := getDefaultTapNamespace()
+			tapDaemonCmdArgs = append(tapDaemonCmdArgs, tapNamespace...)
+
+			tapCmd := exec.Command(cliPath, tapDaemonCmdArgs...)
+
+			viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...)
+
+			t.Cleanup(func() {
+				daemonCleanup(t, viewCmd)
+			})
+
+			t.Logf("running command: %v", tapCmd.String())
+			if err := tapCmd.Run(); err != nil {
+				t.Errorf("error occured while running the tap command, err: %v", err)
+				return
+			}
+
+			t.Logf("running command: %v", viewCmd.String())
+			if err := viewCmd.Start(); err != nil {
+				t.Errorf("error occured while running the view command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := getApiServerUrl(defaultApiServerPort)
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+
+			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+			for i := 0; i < entriesCount; i++ {
+				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			entriesCheckFunc := func() error {
+				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
+				if err != nil {
+					return err
+				}
+				err = checkEntriesAtLeast(entries, 1)
+				if err != nil {
+					return err
+				}
+				entry := entries[0]
+
+				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
+				requestResult, requestErr := executeHttpGetRequest(entryUrl)
+				if requestErr != nil {
+					return fmt.Errorf("failed to get entry, err: %v", requestErr)
+				}
+
+				if requestResult == nil {
+					return fmt.Errorf("unexpected nil entry result")
+				}
+
+				return nil
+			}
+			if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
+				t.Errorf("%v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestDaemonMultipleNamespacesSeePods(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode()
+	var namespacesCmd []string
+	for _, expectedPod := range expectedPods {
+		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
+	}
+	tapCmdArgs = append(tapCmdArgs, namespacesCmd...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+
+	viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...)
+
+	t.Cleanup(func() {
+		daemonCleanup(t, viewCmd)
+	})
+
+	t.Logf("running command: %v", tapCmd.String())
+	if err := tapCmd.Run(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	t.Logf("running command: %v", viewCmd.String())
+	if err := viewCmd.Start(); err != nil {
+		t.Errorf("error occured while running the view command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
+}
+
+func TestDaemonSingleNamespaceSeePods(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
+	}
+	unexpectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode()
+	var namespacesCmd []string
+	for _, expectedPod := range expectedPods {
+		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
+	}
+	tapCmdArgs = append(tapCmdArgs, namespacesCmd...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+
+	viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...)
+
+	t.Cleanup(func() {
+		daemonCleanup(t, viewCmd)
+	})
+
+	t.Logf("running command: %v", tapCmd.String())
+	if err := tapCmd.Run(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	t.Logf("running command: %v", viewCmd.String())
+	if err := viewCmd.Start(); err != nil {
+		t.Errorf("error occured while running the view command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	for _, unexpectedPod := range unexpectedPods {
+		if isPodDescriptorInPodArray(pods, unexpectedPod) {
+			t.Errorf("unexpected result - unexpected pod found, pod namespace: %v, pod name: %v", unexpectedPod.Namespace, unexpectedPod.Name)
+			return
+		}
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
+}

acceptanceTests/testsUtils.go

@@ -3,6 +3,7 @@ package acceptanceTests
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -10,9 +11,12 @@ import (
 	"os/exec"
 	"path"
 	"strings"
+	"sync"
 	"syscall"
+	"testing"
 	"time"
 
+	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/shared"
 )
 
@@ -24,8 +28,26 @@ const (
 	defaultServiceName    = "httpbin"
 	defaultEntriesCount   = 50
 	waitAfterTapPodsReady = 3 * time.Second
+	cleanCommandTimeout   = 1 * time.Minute
 )
 
+type PodDescriptor struct {
+	Name      string
+	Namespace string
+}
+
+func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
+	for _, pod := range pods {
+		podNamespace := pod["namespace"].(string)
+		podName := pod["name"].(string)
+
+		if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
+			return true
+		}
+	}
+	return false
+}
+
 func getCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -59,7 +81,11 @@ func getProxyUrl(namespace string, service string) string {
 }
 
 func getApiServerUrl(port uint16) string {
-	return fmt.Sprintf("http://localhost:%v/mizu", port)
+	return fmt.Sprintf("http://localhost:%v", port)
+}
+
+func getWebSocketUrl(port uint16) string {
+	return fmt.Sprintf("ws://localhost:%v/ws", port)
 }
 
 func getDefaultCommandArgs() []string {
@@ -67,8 +93,9 @@ func getDefaultCommandArgs() []string {
 	telemetry := "telemetry=false"
 	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
 	imagePullPolicy := "image-pull-policy=Never"
+	headless := "headless=true"
 
-	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy}
+	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
 func getDefaultTapCommandArgs() []string {
@@ -78,6 +105,10 @@ func getDefaultTapCommandArgs() []string {
 	return append([]string{tapCommand}, defaultCmdArgs...)
 }
 
+func getDefaultTapCommandArgsWithDaemonMode() []string {
+	return append(getDefaultTapCommandArgs(), "--daemon")
+}
+
 func getDefaultTapCommandArgsWithRegex(regex string) []string {
 	tapCommand := "tap"
 	defaultCmdArgs := getDefaultCommandArgs()
@@ -103,6 +134,20 @@ func getDefaultConfigCommandArgs() []string {
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
+func getDefaultCleanCommandArgs() []string {
+	cleanCommand := "clean"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{cleanCommand}, defaultCmdArgs...)
+}
+
+func getDefaultViewCommandArgs() []string {
+	viewCommand := "view"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{viewCommand}, defaultCmdArgs...)
+}
+
 func retriesExecute(retriesCount int, executeFunc func() error) error {
 	var lastError interface{}
 
@@ -195,16 +240,57 @@ func executeHttpGetRequest(url string) (interface{}, error) {
 	return executeHttpRequest(response, requestErr)
 }
 
-func executeHttpPostRequest(url string, body interface{}) (interface{}, error) {
+func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
 	requestBody, jsonErr := json.Marshal(body)
 	if jsonErr != nil {
 		return nil, jsonErr
 	}
 
-	response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody))
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Add("Content-Type", "application/json")
+	for headerKey, headerValue := range headers {
+		request.Header.Add(headerKey, headerValue)
+	}
+
+	client := &http.Client{}
+	response, requestErr := client.Do(request)
 	return executeHttpRequest(response, requestErr)
 }
 
+func runMizuClean() error {
+	cliPath, err := getCliPath()
+	if err != nil {
+		return err
+	}
+
+	cleanCmdArgs := getDefaultCleanCommandArgs()
+
+	cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
+
+	commandDone := make(chan error)
+	go func() {
+		if err := cleanCmd.Run(); err != nil {
+			commandDone <- err
+		}
+		commandDone <- nil
+	}()
+
+	select {
+	case err = <-commandDone:
+		if err != nil {
+			return err
+		}
+	case <-time.After(cleanCommandTimeout):
+		return errors.New("clean command timed out")
+	}
+
+	return nil
+}
+
 func cleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
@@ -239,6 +325,87 @@ func getLogsPath() (string, error) {
 	return logsPath, nil
 }
 
+func daemonCleanup(t *testing.T, viewCmd *exec.Cmd) {
+	if err := runMizuClean(); err != nil {
+		t.Logf("error running mizu clean: %v", err)
+	}
+
+	if err := cleanupCommand(viewCmd); err != nil {
+		t.Logf("failed to cleanup view command, err: %v", err)
+	}
+}
+
+// waitTimeout waits for the waitgroup for the specified max timeout.
+// Returns true if waiting timed out.
+func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
+	channel := make(chan struct{})
+	go func() {
+		defer close(channel)
+		wg.Wait()
+	}()
+	select {
+	case <-channel:
+		return false // completed normally
+	case <-time.After(timeout):
+		return true // timed out
+	}
+}
+
+// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
+func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
+	if len(entries) < n {
+		return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
+	}
+	return nil
+}
+
+// getDBEntries retrieves the entries from the database before the given timestamp.
+// Also limits the results according to the limit parameter.
+// Timeout for the WebSocket connection is defined by the timeout parameter.
+func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
+	query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
+	webSocketUrl := getWebSocketUrl(defaultApiServerPort)
+
+	var connection *websocket.Conn
+	connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
+	if err != nil {
+		return
+	}
+	defer connection.Close()
+
+	handleWSConnection := func(wg *sync.WaitGroup) {
+		defer wg.Done()
+		for {
+			_, message, err := connection.ReadMessage()
+			if err != nil {
+				return
+			}
+
+			var data map[string]interface{}
+			if err = json.Unmarshal([]byte(message), &data); err != nil {
+				return
+			}
+
+			if data["messageType"] == "entry" {
+				entries = append(entries, data)
+			}
+		}
+	}
+
+	err = connection.WriteMessage(websocket.TextMessage, []byte(query))
+	if err != nil {
+		return
+	}
+
+	var wg sync.WaitGroup
+	go handleWSConnection(&wg)
+	wg.Add(1)
+
+	waitTimeout(&wg, timeout)
+
+	return
+}
+
 func Contains(slice []string, containsValue string) bool {
 	for _, sliceValue := range slice {
 		if sliceValue == containsValue {

agent/go.mod

@@ -3,8 +3,8 @@ module mizuserver
 go 1.16
 
 require (
+	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
 	github.com/djherbis/atime v1.0.0
-	github.com/fsnotify/fsnotify v1.4.9
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.2
@@ -16,13 +16,12 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/up9inc/basenine/client/go v0.0.0-20211114204315-4d028da5fda5
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	go.mongodb.org/mongo-driver v1.7.1
-	gorm.io/driver/sqlite v1.1.4
-	gorm.io/gorm v1.21.8
+	golang.org/x/text v0.3.5 // indirect
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
 	k8s.io/client-go v0.21.2

agent/go.sum

@@ -52,6 +52,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b h1:8m+eVxVVDDyJFidv7Ck1OwqnDaQR6pTSRGlCC2Dnw0A=
+github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b/go.mod h1:+tQQjzrp2501Nd6JXrb9s/XsNvFK3ZbxOnCdQl/vDRo=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -94,7 +96,6 @@ github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
 github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -111,7 +112,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY=
@@ -195,31 +195,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
-github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
-github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
-github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
-github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
-github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
-github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
-github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
-github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
-github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
-github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
-github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
-github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
-github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
-github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
-github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -251,7 +227,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -321,12 +296,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
-github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -336,14 +305,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
-github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -364,16 +329,12 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
-github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
-github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -395,7 +356,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
@@ -419,7 +379,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -447,8 +406,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -458,8 +415,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -468,7 +423,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -485,8 +439,9 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -495,24 +450,22 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/up9inc/basenine/client/go v0.0.0-20211114204315-4d028da5fda5 h1:JbLairDLEJpAC8bwmFuOAB+LYpY/oQbzGRSWRpkF7PQ=
+github.com/up9inc/basenine/client/go v0.0.0-20211114204315-4d028da5fda5/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
-github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
-github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA=
-go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -525,13 +478,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
@@ -609,7 +560,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -623,13 +573,10 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -646,6 +593,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -654,7 +602,6 @@ golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -683,13 +630,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -803,11 +746,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
-gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
-gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v1.21.8 h1:2CEwZSzogdhsKPlJ9OvBKTdlWIpELXb6HbfLfMNhSYI=
-gorm.io/gorm v1.21.8/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

agent/main.go

@@ -6,13 +6,10 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/up9inc/mizu/shared/kubernetes"
 	"io/ioutil"
-	v1 "k8s.io/api/core/v1"
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
 	"mizuserver/pkg/routes"
@@ -20,6 +17,7 @@ import (
 	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
@@ -28,10 +26,15 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/up9inc/mizu/shared/kubernetes"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/antelman107/net-wait-go/wait"
 	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
@@ -49,10 +52,12 @@ var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 var extensions []*tapApi.Extension             // global
 var extensionsMap map[string]*tapApi.Extension // global
 
+var startTime int64
+
 const (
-	socketConnectionRetries = 10
+	socketConnectionRetries    = 10
 	socketConnectionRetryDelay = time.Second * 2
-	socketHandshakeTimeout = time.Second * 2
+	socketHandshakeTimeout     = time.Second * 2
 )
 
 func main() {
@@ -109,7 +114,8 @@ func main() {
 
 		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
 	} else if *apiServerMode {
-		database.InitDataBase(config.Config.AgentDatabasePath)
+		startBasenineServer(shared.BasenineHost, shared.BaseninePort)
+		startTime = time.Now().UnixNano() / int64(time.Millisecond)
 		api.StartResolving(*namespace)
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
@@ -142,6 +148,53 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
+func startBasenineServer(host string, port string) {
+	cmd := exec.Command("basenine", "-addr", host, "-port", port, "-persistent")
+	cmd.Dir = config.Config.AgentDatabasePath
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Start()
+	if err != nil {
+		logger.Log.Panicf("Failed starting Basenine: %v", err)
+	}
+
+	if !wait.New(
+		wait.WithProto("tcp"),
+		wait.WithWait(200*time.Millisecond),
+		wait.WithBreak(50*time.Millisecond),
+		wait.WithDeadline(5*time.Second),
+		wait.WithDebug(true),
+	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
+		logger.Log.Panicf("Basenine is not available: %v", err)
+	}
+
+	// Make a channel to gracefully exit Basenine.
+	channel := make(chan os.Signal)
+	signal.Notify(channel, os.Interrupt, syscall.SIGTERM)
+
+	// Handle the channel.
+	go func() {
+		<-channel
+		cmd.Process.Signal(syscall.SIGTERM)
+	}()
+
+	// Limit the database size to default 200MB
+	err = basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
+	if err != nil {
+		logger.Log.Panicf("Error while limiting database size: %v", err)
+	}
+
+	for _, extension := range extensions {
+		macros := extension.Dissector.Macros()
+		for macro, expanded := range macros {
+			err = basenine.Macro(host, port, macro, expanded)
+			if err != nil {
+				logger.Log.Panicf("Error while adding a macro: %v", err)
+			}
+		}
+	}
+}
+
 func loadExtensions() {
 	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
 	extensionsDir := path.Join(dir, "./extensions/")
@@ -154,7 +207,7 @@ func loadExtensions() {
 	extensionsMap = make(map[string]*tapApi.Extension)
 	for i, file := range files {
 		filename := file.Name()
-		logger.Log.Infof("Loading extension: %s\n", filename)
+		logger.Log.Infof("Loading extension: %s", filename)
 		extension := &tapApi.Extension{
 			Path: path.Join(extensionsDir, filename),
 		}
@@ -166,7 +219,7 @@ func loadExtensions() {
 		var ok bool
 		dissector, ok = symDissector.(tapApi.Dissector)
 		if err != nil || !ok {
-			panic(fmt.Sprintf("Failed to load the extension: %s\n", extension.Path))
+			panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
 		}
 		dissector.Register(extension)
 		extension.Dissector = dissector
@@ -179,7 +232,7 @@ func loadExtensions() {
 	})
 
 	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v\n", extension)
+		logger.Log.Infof("Extension Properties: %+v", extension)
 	}
 
 	controllers.InitExtensionsMap(extensionsMap)
@@ -200,7 +253,8 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	app.Use(static.ServeRoot("/", "./site"))
 	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
-	api.WebSocketRoutes(app, &eventHandlers)
+	api.WebSocketRoutes(app, &eventHandlers, startTime)
+	routes.QueryRoutes(app)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
@@ -344,10 +398,11 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig {
 }
 
 func determineLogLevel() (logLevel logging.Level) {
-	logLevel = logging.INFO
-	if os.Getenv(shared.DebugModeEnvVar) == "1" {
-		logLevel = logging.DEBUG
+	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
+	if err != nil {
+		logLevel = logging.INFO
 	}
+
 	return
 }
 
@@ -361,7 +416,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 		socketConnection, _, err := dialer.Dial(socketAddress, nil)
 		if err != nil {
 			if i < retryAmount {
-				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay / time.Second)
+				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
 				time.Sleep(retryDelay)
 			}
 		} else {
@@ -371,8 +426,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 	return nil, lastErr
 }
 
-
-func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error){
+func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error) {
 	provider, err := kubernetes.NewProviderInCluster()
 	if err != nil {
 		return nil, err
@@ -385,7 +439,7 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e
 		AgentImage:               config.Config.AgentImage,
 		TapperResources:          config.Config.TapperResources,
 		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
-		DumpLogs:                 config.Config.DumpLogs,
+		LogLevel:                 config.Config.LogLevel,
 		IgnoredUserAgents:        config.Config.IgnoredUserAgents,
 		MizuApiFilteringOptions:  config.Config.MizuApiFilteringOptions,
 		MizuServiceAccountExists: true, //assume service account exists since daemon mode will not function without it anyway
@@ -405,7 +459,7 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e
 					return
 				}
 				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
-			case _, ok := <-tapperSyncer.TapPodChangesOut:
+			case tapPodChangeEvent, ok := <-tapperSyncer.TapPodChangesOut:
 				if !ok {
 					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
 					return
@@ -418,6 +472,7 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e
 				}
 				api.BroadcastToBrowserClients(serializedTapStatus)
 				providers.TapStatus.Pods = tapStatus.Pods
+				providers.ExpectedTapperAmount = tapPodChangeEvent.ExpectedTapperAmount
 			case <-ctx.Done():
 				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
 				return

agent/pkg/api/main.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"os"
@@ -14,15 +13,16 @@ import (
 	"strings"
 	"time"
 
-	"go.mongodb.org/mongo-driver/bson/primitive"
-
 	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
+
+	basenine "github.com/up9inc/basenine/client/go"
 )
 
 var k8sResolver *resolver.Resolver
@@ -76,7 +76,7 @@ func startReadingFiles(workingDir string) {
 		sort.Sort(utils.ByModTime(harFiles))
 
 		if len(harFiles) == 0 {
-			logger.Log.Infof("Waiting for new files\n")
+			logger.Log.Infof("Waiting for new files")
 			time.Sleep(3 * time.Second)
 			continue
 		}
@@ -99,11 +99,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
+	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		panic(err)
+	}
+	connection.InsertMode()
+
 	disableOASValidation := false
 	ctx := context.Background()
 	doc, contractContent, router, err := loadOAS(ctx)
 	if err != nil {
-		logger.Log.Infof("Disabled OAS validation: %s\n", err.Error())
+		logger.Log.Infof("Disabled OAS validation: %s", err.Error())
 		disableOASValidation = true
 	}
 
@@ -112,13 +118,13 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
 		baseEntry := extension.Dissector.Summarize(mizuEntry)
-		mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
+		mizuEntry.Base = baseEntry
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
-				json.Unmarshal([]byte(mizuEntry.Entry), &httpPair)
+				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
 				baseEntry.ContractStatus = contract.Status
@@ -128,18 +134,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			var pair tapApi.RequestResponsePair
-			json.Unmarshal([]byte(mizuEntry.Entry), &pair)
-			harEntry, err := utils.NewEntry(&pair)
+			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
 				baseEntry.Rules = rules
 			}
 		}
-		database.CreateEntry(mizuEntry)
 
-		baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)
-		BroadcastToBrowserClients(baseEntryBytes)
+		data, err := json.Marshal(mizuEntry)
+		if err != nil {
+			panic(err)
+		}
+		connection.SendText(string(data))
 	}
 }
 
@@ -148,7 +154,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedSource := connectionInfo.ClientIP
 		resolvedSource = k8sResolver.Resolve(unresolvedSource)
 		if resolvedSource == "" {
-			logger.Log.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
+			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
@@ -156,7 +162,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
 		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
 		if resolvedDestination == "" {
-			logger.Log.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
+			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
@@ -171,21 +177,3 @@ func CheckIsServiceIP(address string) bool {
 	}
 	return k8sResolver.CheckIsServiceIP(address)
 }
-
-// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
-func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int {
-	sizeBytes := len(mizuEntry.Entry)
-	sizeBytes += len(mizuEntry.EntryId)
-	sizeBytes += len(mizuEntry.Service)
-	sizeBytes += len(mizuEntry.Url)
-	sizeBytes += len(mizuEntry.Method)
-	sizeBytes += len(mizuEntry.RequestSenderIp)
-	sizeBytes += len(mizuEntry.ResolvedDestination)
-	sizeBytes += len(mizuEntry.ResolvedSource)
-	sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
-	sizeBytes += 8 // Timestamp bytes
-	sizeBytes += 8 // SizeBytes bytes
-	sizeBytes += 1 // IsOutgoing bytes
-
-	return sizeBytes
-}

agent/pkg/api/socket_routes.go

@@ -1,13 +1,18 @@
 package api
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
+	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
 )
@@ -39,17 +44,17 @@ func init() {
 	connectedWebsockets = make(map[int]*SocketConnection, 0)
 }
 
-func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
 	app.GET("/ws", func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, false)
+		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
 	})
 	app.GET("/wsTapper", func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, true)
+		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
 	})
 }
 
-func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
-	conn, err := websocketUpgrader.Upgrade(w, r, nil)
+func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
+	ws, err := websocketUpgrader.Upgrade(w, r, nil)
 	if err != nil {
 		logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
 		return
@@ -59,30 +64,110 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 
 	connectedWebsocketIdCounter++
 	socketId := connectedWebsocketIdCounter
-	connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
+	connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
 
 	websocketIdsLock.Unlock()
 
+	var connection *basenine.Connection
+	var isQuerySet bool
+
+	// `!isTapper` means it's a connection from the web UI
+	if !isTapper {
+		connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	data := make(chan []byte)
+	meta := make(chan []byte)
+
 	defer func() {
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
 		socketCleanup(socketId, connectedWebsockets[socketId])
 	}()
 
 	eventHandlers.WebSocketConnect(socketId, isTapper)
 
+	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
+	SendToSocket(socketId, startTimeBytes)
+
 	for {
-		_, msg, err := conn.ReadMessage()
+		_, msg, err := ws.ReadMessage()
 		if err != nil {
 			logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
 			break
 		}
-		eventHandlers.WebSocketMessage(socketId, msg)
+
+		if !isTapper && !isQuerySet {
+			query := string(msg)
+			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+			if err != nil {
+				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
+					Type:      "error",
+					AutoClose: 5000,
+					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
+				})
+				SendToSocket(socketId, toastBytes)
+				break
+			}
+
+			isQuerySet = true
+
+			handleDataChannel := func(c *basenine.Connection, data chan []byte) {
+				for {
+					bytes := <-data
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var dataMap map[string]interface{}
+					err = json.Unmarshal(bytes, &dataMap)
+
+					base := dataMap["base"].(map[string]interface{})
+					base["id"] = uint(dataMap["id"].(float64))
+
+					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
+					SendToSocket(socketId, baseEntryBytes)
+				}
+			}
+
+			handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
+				for {
+					bytes := <-meta
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var metadata *basenine.Metadata
+					err = json.Unmarshal(bytes, &metadata)
+					if err != nil {
+						logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+					}
+
+					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
+					SendToSocket(socketId, metadataBytes)
+				}
+			}
+
+			go handleDataChannel(connection, data)
+			go handleMetaChannel(connection, meta)
+
+			connection.Query(query, data, meta)
+		} else {
+			eventHandlers.WebSocketMessage(socketId, msg)
+		}
 	}
 }
 
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		logger.Log.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
+		logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
 	}
 
 	websocketIdsLock.Lock()

agent/pkg/api/socket_server_handlers.go

@@ -65,14 +65,14 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	var socketMessageBase shared.WebSocketMessageMetadata
 	err := json.Unmarshal(message, &socketMessageBase)
 	if err != nil {
-		logger.Log.Infof("Could not unmarshal websocket message %v\n", err)
+		logger.Log.Infof("Could not unmarshal websocket message %v", err)
 	} else {
 		switch socketMessageBase.MessageType {
 		case shared.WebSocketMessageTypeTappedEntry:
 			var tappedEntryMessage models.WebSocketTappedEntryMessage
 			err := json.Unmarshal(message, &tappedEntryMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
 				h.SocketOutChannel <- tappedEntryMessage.Data
@@ -81,7 +81,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			var statusMessage shared.WebSocketStatusMessage
 			err := json.Unmarshal(message, &statusMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
 				BroadcastToBrowserClients(message)
@@ -90,7 +90,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			var outboundLinkMessage models.WebsocketOutboundLinkMessage
 			err := json.Unmarshal(message, &outboundLinkMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				handleTLSLink(outboundLinkMessage)
 			}

agent/pkg/controllers/entries_controller.go

@@ -2,14 +2,17 @@ package controllers
 
 import (
 	"encoding/json"
-	"fmt"
-	"github.com/gin-gonic/gin"
-	tapApi "github.com/up9inc/mizu/tap/api"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/utils"
-	"mizuserver/pkg/validation"
 	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 var extensionsMap map[string]*tapApi.Extension // global
@@ -18,68 +21,36 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
 	extensionsMap = ref
 }
 
-func GetEntries(c *gin.Context) {
-	entriesFilter := &models.EntriesFilter{}
-
-	if err := c.BindQuery(entriesFilter); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-	err := validation.Validate(entriesFilter)
+func Error(c *gin.Context, err error) bool {
 	if err != nil {
-		c.JSON(http.StatusBadRequest, err)
+		logger.Log.Errorf("Error getting entry: %v", err)
+		c.Error(err)
+		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": true, "msg": err.Error()})
+		return true // signal that there was an error and the caller should return
 	}
-
-	order := database.OperatorToOrderMapping[entriesFilter.Operator]
-	operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
-	var entries []tapApi.MizuEntry
-	database.GetEntriesTable().
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
-		Limit(entriesFilter.Limit).
-		Find(&entries)
-
-	if len(entries) > 0 && order == database.OrderDesc {
-		// the entries always order from oldest to newest - we should reverse
-		utils.ReverseSlice(entries)
-	}
-
-	baseEntries := make([]tapApi.BaseEntryDetails, 0)
-	for _, entry := range entries {
-		baseEntryDetails := tapApi.BaseEntryDetails{}
-		if err := models.GetEntry(&entry, &baseEntryDetails); err != nil {
-			continue
-		}
-
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entry.Entry), &pair)
-		harEntry, err := utils.NewEntry(&pair)
-		if err == nil {
-			rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service)
-			baseEntryDetails.Rules = rules
-		}
-
-		baseEntries = append(baseEntries, baseEntryDetails)
-	}
-
-	c.JSON(http.StatusOK, baseEntries)
+	return false // no error, can continue
 }
 
 func GetEntry(c *gin.Context) {
-	var entryData tapApi.MizuEntry
-	database.GetEntriesTable().
-		Where(map[string]string{"entryId": c.Param("entryId")}).
-		First(&entryData)
+	id, _ := strconv.Atoi(c.Param("id"))
+	var entry tapApi.MizuEntry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id)
+	if Error(c, err) {
+		return // exit
+	}
+	err = json.Unmarshal(bytes, &entry)
+	if Error(c, err) {
+		return // exit
+	}
 
-	extension := extensionsMap[entryData.ProtocolName]
-	protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
+	extension := extensionsMap[entry.Protocol.Name]
+	protocol, representation, bodySize, _ := extension.Dissector.Represent(entry.Protocol, entry.Request, entry.Response)
 
 	var rules []map[string]interface{}
 	var isRulesEnabled bool
-	if entryData.ProtocolName == "http" {
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entryData.Entry), &pair)
-		harEntry, _ := utils.NewEntry(&pair)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service)
+	if entry.Protocol.Name == "http" {
+		harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Service)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)
 		json.Unmarshal(inrec, &rules)
@@ -89,7 +60,7 @@ func GetEntry(c *gin.Context) {
 		Protocol:       protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,
-		Data:           entryData,
+		Data:           entry,
 		Rules:          rules,
 		IsRulesEnabled: isRulesEnabled,
 	})

agent/pkg/controllers/query_controller.go

@@ -0,0 +1,31 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+)
+
+type ValidateResponse struct {
+	Valid   bool   `json:"valid"`
+	Message string `json:"message"`
+}
+
+func PostValidate(c *gin.Context) {
+	query := c.PostForm("query")
+	valid := true
+	message := ""
+
+	err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+	if err != nil {
+		valid = false
+		message = err.Error()
+	}
+
+	c.JSON(http.StatusOK, ValidateResponse{
+		Valid:   valid,
+		Message: message,
+	})
+}

agent/pkg/controllers/status_controller.go

@@ -2,7 +2,9 @@ package controllers
 
 import (
 	"encoding/json"
+	"fmt"
 	"mizuserver/pkg/api"
+	"mizuserver/pkg/config"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"mizuserver/pkg/up9"
@@ -15,6 +17,13 @@ import (
 )
 
 func HealthCheck(c *gin.Context) {
+	if config.Config.DaemonMode {
+		if providers.ExpectedTapperAmount != providers.TappersCount {
+			c.JSON(http.StatusInternalServerError, fmt.Sprintf("expecting more tappers than are actually connected (%d expected, %d connected)", providers.ExpectedTapperAmount, providers.TappersCount))
+			return
+		}
+	}
+
 	response := shared.HealthResponse{
 		TapStatus:    providers.TapStatus,
 		TappersCount: providers.TappersCount,
@@ -37,7 +46,7 @@ func PostTappedPods(c *gin.Context) {
 	providers.TapStatus.Pods = tapStatus.Pods
 	message := shared.CreateWebSocketStatusMessage(*tapStatus)
 	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v\n", err)
+		logger.Log.Errorf("Could not Marshal message %v", err)
 	} else {
 		api.BroadcastToBrowserClients(jsonBytes)
 	}

agent/pkg/database/main.go

@@ -1,78 +0,0 @@
-package database
-
-import (
-	"fmt"
-	"mizuserver/pkg/utils"
-	"time"
-
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-const (
-	OrderDesc  = "desc"
-	OrderAsc   = "asc"
-	LT         = "lt"
-	GT         = "gt"
-	TimeFormat = "2006-01-02 15:04:05.000000000"
-)
-
-var (
-	DB                      *gorm.DB
-	IsDBLocked              = false
-	OperatorToSymbolMapping = map[string]string{
-		LT: "<",
-		GT: ">",
-	}
-	OperatorToOrderMapping = map[string]string{
-		LT: OrderDesc,
-		GT: OrderAsc,
-	}
-)
-
-var DBPath string
-
-func GetEntriesTable() *gorm.DB {
-	return DB.Table("mizu_entries")
-}
-
-func CreateEntry(entry *tapApi.MizuEntry) {
-	if IsDBLocked {
-		return
-	}
-	GetEntriesTable().Create(entry)
-}
-
-func InitDataBase(databasePath string) *gorm.DB {
-	DBPath = databasePath
-	DB, _ = gorm.Open(sqlite.Open(databasePath), &gorm.Config{
-		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
-	})
-	_ = DB.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created
-	go StartEnforcingDatabaseSize()
-	return DB
-}
-
-func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry {
-	order := OrderDesc
-	protocolNameCondition := "1 = 1"
-	if protocolName != nil {
-		protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
-	}
-
-	var entries []tapApi.MizuEntry
-	GetEntriesTable().
-		Where(protocolNameCondition).
-		Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))).
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Find(&entries)
-
-	if len(entries) > 0 {
-		// the entries always order from oldest to newest so we should revers
-		utils.ReverseSlice(entries)
-	}
-	return entries
-}

agent/pkg/database/size_enforcer.go

@@ -1,102 +0,0 @@
-package database
-
-import (
-	"mizuserver/pkg/config"
-	"os"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/shared/units"
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-const percentageOfMaxSizeBytesToPrune = 15
-
-func StartEnforcingDatabaseSize() {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
-		return
-	}
-
-	checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
-		checkFileSize(config.Config.MaxDBSizeBytes)
-	})
-
-	go func() {
-		for {
-			select {
-			case event, ok := <-watcher.Events:
-				if !ok {
-					return // closed channel
-				}
-				if event.Op == fsnotify.Write {
-					checkFileSizeDebouncer.SetOn()
-				}
-			case err, ok := <-watcher.Errors:
-				if !ok {
-					return // closed channel
-				}
-				logger.Log.Errorf("filesystem watcher encountered error:%v", err)
-			}
-		}
-	}()
-
-	err = watcher.Add(DBPath)
-	if err != nil {
-		logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
-	}
-}
-
-func checkFileSize(maxSizeBytes int64) {
-	fileStat, err := os.Stat(DBPath)
-	if err != nil {
-		logger.Log.Errorf("Error checking %s file size: %v", DBPath, err)
-	} else {
-		if fileStat.Size() > maxSizeBytes {
-			pruneOldEntries(fileStat.Size())
-		}
-	}
-}
-
-func pruneOldEntries(currentFileSize int64) {
-	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
-	IsDBLocked = true
-	defer func() { IsDBLocked = false }()
-
-	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
-
-	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
-	if err != nil {
-		logger.Log.Errorf("Error getting 10000 first db rows: %v", err)
-		return
-	}
-
-	entryIdsToRemove := make([]uint, 0)
-	bytesToBeRemoved := int64(0)
-	for rows.Next() {
-		if bytesToBeRemoved >= amountOfBytesToTrim {
-			break
-		}
-		var entry tapApi.MizuEntry
-		err = DB.ScanRows(rows, &entry)
-		if err != nil {
-			logger.Log.Errorf("Error scanning db row: %v", err)
-			continue
-		}
-
-		entryIdsToRemove = append(entryIdsToRemove, entry.ID)
-		bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
-	}
-
-	if len(entryIdsToRemove) > 0 {
-		GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{})
-		// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
-		DB.Exec("VACUUM")
-		logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
-	} else {
-		logger.Log.Error("Found no rows to remove when pruning")
-	}
-}

agent/pkg/models/models.go

@@ -2,12 +2,12 @@ package models
 
 import (
 	"encoding/json"
+	"mizuserver/pkg/rules"
 
 	tapApi "github.com/up9inc/mizu/tap/api"
 
-	"mizuserver/pkg/rules"
-
 	"github.com/google/martian/har"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
 )
@@ -16,15 +16,9 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
-type EntriesFilter struct {
-	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
-	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
-	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
-}
-
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data *tapApi.BaseEntryDetails `json:"data,omitempty"`
+	Data map[string]interface{} `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
@@ -42,7 +36,28 @@ type AuthStatus struct {
 	Model string `json:"model"`
 }
 
-func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) {
+type ToastMessage struct {
+	Type      string `json:"type"`
+	AutoClose uint   `json:"autoClose"`
+	Text      string `json:"text"`
+}
+
+type WebSocketToastMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *ToastMessage `json:"data,omitempty"`
+}
+
+type WebSocketQueryMetadataMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *basenine.Metadata `json:"data,omitempty"`
+}
+
+type WebSocketStartTimeMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data int64 `json:"data"`
+}
+
+func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,
@@ -72,6 +87,36 @@ func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error)
 	return json.Marshal(message)
 }
 
+func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
+	message := &WebSocketToastMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeToast,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) {
+	message := &WebSocketQueryMetadataMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeQueryMetadata,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
+	message := &WebSocketStartTimeMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeStartTime,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
 // ExtendedHAR is the top level object of a HAR log.
 type ExtendedHAR struct {
 	Log *ExtendedLog `json:"log"`

agent/pkg/providers/stats_provider_test.go

@@ -2,16 +2,10 @@ package providers_test
 
 import (
 	"fmt"
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/providers"
 	"testing"
 )
 
-func init() {
-	database.InitDataBase(config.DefaultDatabasePath)
-}
-
 func TestNoEntryAddedCount(t *testing.T) {
 	entriesStats := providers.GetGeneralStats()
 

agent/pkg/providers/status_provider.go

@@ -19,7 +19,7 @@ var (
 	TapStatus      shared.TapStatus
 	authStatus     *models.AuthStatus
 	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
-
+	ExpectedTapperAmount = -1 //only relevant in daemon mode as cli manages tappers otherwise
 	tappersCountLock = sync.Mutex{}
 )
 

agent/pkg/resolver/resolver.go

@@ -164,10 +164,10 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
 		resolver.nameMap.Remove(key)
-		logger.Log.Infof("setting %s=nil\n", key)
+		logger.Log.Infof("setting %s=nil", key)
 	} else {
 		resolver.nameMap.Set(key, resolved)
-		logger.Log.Infof("setting %s=%s\n", key, resolved)
+		logger.Log.Infof("setting %s=%s", key, resolved)
 	}
 }
 
@@ -188,7 +188,7 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
 			var statusError *k8serrors.StatusError
 			if errors.As(err, &statusError) {
 				if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
-					logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
+					logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v", err)
 					return
 				}
 			}

agent/pkg/routes/entries_routes.go

@@ -1,14 +1,14 @@
 package routes
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/controllers"
+
+	"github.com/gin-gonic/gin"
 )
 
 // EntriesRoutes defines the group of har entries routes.
 func EntriesRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/entries")
 
-	routeGroup.GET("/", controllers.GetEntries)        // get entries (base/thin entries)
-	routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry
 }

agent/pkg/routes/query_routes.go

@@ -0,0 +1,13 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+
+	"github.com/gin-gonic/gin"
+)
+
+func QueryRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/query")
+
+	routeGroup.POST("/validate", controllers.PostValidate)
+}

agent/pkg/sensitiveDataFiltering/consts.go

@@ -1,10 +0,0 @@
-package sensitiveDataFiltering
-
-const maskedFieldPlaceholderValue = "[REDACTED]"
-
-//these values MUST be all lower case and contain no `-` or `_` characters
-var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
-	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
-	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
-	"zip", "zipcode", "address", "country", "firstname", "lastname",
-	"middlename", "fname", "lname", "birthdate"}

agent/pkg/up9/main.go

@@ -7,15 +7,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/google/martian/har"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
@@ -23,6 +24,7 @@ import (
 
 const (
 	AnalyzeCheckSleepTime = 5 * time.Second
+	SentCountLogInterval  = 100
 )
 
 type GuestToken struct {
@@ -110,14 +112,14 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus {
 }
 
 func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
-	logger.Log.Infof("Sync entries - started\n")
+	logger.Log.Infof("Sync entries - started")
 
 	var (
 		token, model string
 		guestMode    bool
 	)
 	if syncEntriesConfig.Token == "" {
-		logger.Log.Infof("Sync entries - creating anonymous token. env %s\n", syncEntriesConfig.Env)
+		logger.Log.Infof("Sync entries - creating anonymous token. env %s", syncEntriesConfig.Env)
 		guestToken, err := createAnonymousToken(syncEntriesConfig.Env)
 		if err != nil {
 			return fmt.Errorf("failed creating anonymous token, err: %v", err)
@@ -131,7 +133,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
 		model = syncEntriesConfig.Workspace
 		guestMode = false
 
-		logger.Log.Infof("Sync entries - upserting model. env %s, model %s\n", syncEntriesConfig.Env, model)
+		logger.Log.Infof("Sync entries - upserting model. env %s, model %s", syncEntriesConfig.Env, model)
 		if err := upsertModel(token, model, syncEntriesConfig.Env); err != nil {
 			return fmt.Errorf("failed upserting model, err: %v", err)
 		}
@@ -142,7 +144,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
 		return fmt.Errorf("invalid model name, model name: %s", model)
 	}
 
-	logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode)
+	logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v", token, model, guestMode)
 	go syncEntriesImpl(token, model, syncEntriesConfig.Env, syncEntriesConfig.UploadIntervalSec, guestMode)
 
 	return nil
@@ -204,44 +206,62 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 	analyzeInformation.AnalyzeDestination = envPrefix
 	analyzeInformation.SentCount = 0
 
-	sleepTime := time.Second * time.Duration(uploadIntervalSec)
+	// "http or grpc" filter indicates that we're only interested in HTTP and gRPC entries
+	query := "http or grpc"
 
-	var timeFrom time.Time
-	protocolFilter := "http"
+	logger.Log.Infof("Getting entries from the database")
 
-	for {
-		timeTo := time.Now()
-		logger.Log.Infof("Getting entries from %v, to %v\n", timeFrom.Format(time.RFC3339Nano), timeTo.Format(time.RFC3339Nano))
-		entriesArray := database.GetEntriesFromDb(timeFrom, timeTo, &protocolFilter)
+	var connection *basenine.Connection
+	var err error
+	connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		panic(err)
+	}
 
-		if len(entriesArray) > 0 {
-			result := make([]har.Entry, 0)
-			for _, data := range entriesArray {
-				var pair tapApi.RequestResponsePair
-				if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
-					continue
-				}
-				harEntry, err := utils.NewEntry(&pair)
-				if err != nil {
-					continue
-				}
-				if data.ResolvedSource != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource})
-				}
-				if data.ResolvedDestination != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination})
-					harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination)
-				}
+	data := make(chan []byte)
+	meta := make(chan []byte)
 
-				// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
-				if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
-					continue
-				}
+	defer func() {
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
+	}()
 
-				result = append(result, *harEntry)
+	handleDataChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, data chan []byte) {
+		defer wg.Done()
+		for {
+			dataBytes := <-data
+
+			if string(dataBytes) == basenine.CloseChannel {
+				return
 			}
 
-			logger.Log.Infof("About to upload %v entries\n", len(result))
+			var dataMap map[string]interface{}
+			err = json.Unmarshal(dataBytes, &dataMap)
+
+			result := make([]har.Entry, 0)
+			var entry tapApi.MizuEntry
+			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
+				continue
+			}
+			harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+			if err != nil {
+				continue
+			}
+			if entry.ResolvedSource != "" {
+				harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
+			}
+			if entry.ResolvedDestination != "" {
+				harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
+				harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.ResolvedDestination)
+			}
+
+			// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
+			if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
+				continue
+			}
+
+			result = append(result, *harEntry)
 
 			body, jMarshalErr := json.Marshal(result)
 			if jMarshalErr != nil {
@@ -273,18 +293,33 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 				logger.Log.Info("Stopping sync entries")
 				logger.Log.Fatal(postErr)
 			}
-			analyzeInformation.SentCount += len(entriesArray)
-			logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
+			analyzeInformation.SentCount += 1
 
-			logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
-		} else {
-			logger.Log.Infof("Nothing to upload")
+			if analyzeInformation.SentCount%SentCountLogInterval == 0 {
+				logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
+			}
 		}
-
-		logger.Log.Infof("Sleeping for %v...\n", sleepTime)
-		time.Sleep(sleepTime)
-		timeFrom = timeTo
 	}
+
+	handleMetaChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, meta chan []byte) {
+		defer wg.Done()
+		for {
+			metaBytes := <-meta
+
+			if string(metaBytes) == basenine.CloseChannel {
+				return
+			}
+		}
+	}
+
+	var wg sync.WaitGroup
+	go handleDataChannel(&wg, connection, data)
+	go handleMetaChannel(&wg, connection, meta)
+	wg.Add(2)
+
+	connection.Query(query, data, meta)
+
+	wg.Wait()
 }
 
 func UpdateAnalyzeStatus(callback func(data []byte)) {

agent/pkg/utils/har.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
 )
 
 // Keep it because we might want cookies in the future
@@ -120,13 +119,11 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 	return params
 }
 
-func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) {
-	reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{})
+func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
+	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
-	headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{}))
-
-	postData, _ := reqDetails["postData"].(map[string]interface{})
+	postData, _ := request["postData"].(map[string]interface{})
 	mimeType, _ := postData["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
@@ -138,7 +135,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	}
 
 	queryString := make([]har.QueryString, 0)
-	for _, _qs := range reqDetails["queryString"].([]interface{}) {
+	for _, _qs := range request["_queryString"].([]interface{}) {
 		qs := _qs.(map[string]interface{})
 		queryString = append(queryString, har.QueryString{
 			Name:  qs["name"].(string),
@@ -146,7 +143,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 		})
 	}
 
-	url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string))
+	url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
 		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
 	}
@@ -157,9 +154,9 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	}
 
 	harRequest = &har.Request{
-		Method:      reqDetails["method"].(string),
+		Method:      request["method"].(string),
 		URL:         url,
-		HTTPVersion: reqDetails["httpVersion"].(string),
+		HTTPVersion: request["httpVersion"].(string),
 		HeadersSize: -1,
 		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
 		QueryString: queryString,
@@ -175,13 +172,11 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	return
 }
 
-func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) {
-	resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{})
+func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
+	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
-	headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{}))
-
-	content, _ := resDetails["content"].(map[string]interface{})
+	content, _ := response["content"].(map[string]interface{})
 	mimeType, _ := content["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
@@ -200,9 +195,11 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 		Size:     int64(len(bodyText)),
 	}
 
-	status := int(resDetails["status"].(float64))
+	status := int(response["status"].(float64))
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
-		status, err = strconv.Atoi(_status)
+		if _status != "" {
+			status, err = strconv.Atoi(_status)
+		}
 		if err != nil {
 			logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
 			return nil, errors.New("failed converting response status to int for HAR")
@@ -210,9 +207,9 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	}
 
 	harResponse = &har.Response{
-		HTTPVersion: resDetails["httpVersion"].(string),
+		HTTPVersion: response["httpVersion"].(string),
 		Status:      status,
-		StatusText:  resDetails["statusText"].(string),
+		StatusText:  response["statusText"].(string),
 		HeadersSize: -1,
 		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
 		Headers:     headers,
@@ -222,34 +219,33 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	return
 }
 
-func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
-	harRequest, err := NewRequest(&pair.Request)
+func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
+	harRequest, err := NewRequest(request)
 	if err != nil {
 		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting request to HAR")
 	}
 
-	harResponse, err := NewResponse(&pair.Response)
+	harResponse, err := NewResponse(response)
 	if err != nil {
 		logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting response to HAR")
 	}
 
-	totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	if totalTime < 1 {
-		totalTime = 1
+	if elapsedTime < 1 {
+		elapsedTime = 1
 	}
 
 	harEntry := har.Entry{
-		StartedDateTime: pair.Request.CaptureTime,
-		Time:            totalTime,
+		StartedDateTime: startTime,
+		Time:            elapsedTime,
 		Request:         harRequest,
 		Response:        harResponse,
 		Cache:           &har.Cache{},
 		Timings: &har.Timings{
 			Send:    -1,
 			Wait:    -1,
-			Receive: totalTime,
+			Receive: elapsedTime,
 		},
 	}
 

agent/pkg/utils/truncating_logger.go

@@ -1,60 +0,0 @@
-package utils
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	loggerShared "github.com/up9inc/mizu/shared/logger"
-	"gorm.io/gorm/logger"
-	"gorm.io/gorm/utils"
-)
-
-// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
-type TruncatingLogger struct {
-	LogLevel      logger.LogLevel
-	SlowThreshold time.Duration
-}
-
-func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface {
-	truncatingLogger.LogLevel = logLevel
-	return truncatingLogger
-}
-
-func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Info {
-		return
-	}
-	loggerShared.Log.Errorf("gorm info: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Warn {
-		return
-	}
-	loggerShared.Log.Errorf("gorm warning: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Error {
-		return
-	}
-	loggerShared.Log.Errorf("gorm error: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
-	if truncatingLogger.LogLevel == logger.Silent {
-		return
-	}
-	elapsed := time.Since(begin)
-	if err != nil {
-		sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary
-		truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql))
-	} else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold {
-		sql, rows := fc()
-		truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
-	} else if truncatingLogger.LogLevel >= logger.Info {
-		sql, rows := fc()
-		truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
-	}
-}

cli/cmd/root.go

@@ -23,6 +23,7 @@ Further info is available at https://github.com/up9inc/mizu`,
 		if err := config.InitConfig(cmd); err != nil {
 			logger.Log.Fatal(err)
 		}
+
 		return nil
 	},
 }

cli/cmd/tapRunner.go

@@ -4,16 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/cli/cmd/goUtils"
 	"io/ioutil"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"path"
 	"regexp"
 	"strings"
 	"time"
 
+	"github.com/up9inc/mizu/cli/cmd/goUtils"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
@@ -199,7 +200,7 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 		AgentImage:               config.Config.AgentImage,
 		TapperResources:          config.Config.Tap.TapperResources,
 		ImagePullPolicy:          config.Config.ImagePullPolicy(),
-		DumpLogs:                 config.Config.DumpLogs,
+		LogLevel:                 config.Config.LogLevel(),
 		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
 		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
@@ -281,7 +282,7 @@ func createMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne
 	}
 
 	if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
-		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err)))
 	}
 
 	var err error
@@ -309,6 +310,7 @@ func createMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne
 		MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(),
 		Resources:             config.Config.Tap.ApiServerResources,
 		ImagePullPolicy:       config.Config.ImagePullPolicy(),
+		LogLevel:              config.Config.LogLevel(),
 	}
 
 	if config.Config.Tap.DaemonMode {
@@ -439,7 +441,7 @@ func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provid
 }
 
 func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
-	logger.Log.Infof("\nRemoving mizu resources\n")
+	logger.Log.Infof("\nRemoving mizu resources")
 
 	var leftoverResources []string
 
@@ -624,8 +626,10 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 					break
 				}
 
-				logger.Log.Infof("Mizu is available at %s\n", url)
-				uiUtils.OpenBrowser(url)
+				logger.Log.Infof("Mizu is available at %s", url)
+				if !config.Config.HeadlessMode {
+					uiUtils.OpenBrowser(url)
+				}
 				if err := apiProvider.ReportTappedPods(state.tapperSyncer.CurrentlyTappedPods); err != nil {
 					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
 				}

cli/cmd/viewRunner.go

@@ -56,9 +56,11 @@ func runMizuView() {
 		return
 	}
 
-	logger.Log.Infof("Mizu is available at %s\n", url)
+	logger.Log.Infof("Mizu is available at %s", url)
 
-	uiUtils.OpenBrowser(url)
+	if !config.Config.HeadlessMode {
+		uiUtils.OpenBrowser(url)
+	}
 
 	if isCompatible, err := version.CheckVersionCompatibility(apiServerProvider); err != nil {
 		logger.Log.Errorf("Failed to check versions compatibility %v", err)

cli/config/config.go

@@ -3,14 +3,15 @@ package config
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
-	"k8s.io/apimachinery/pkg/util/json"
 	"os"
 	"reflect"
 	"strconv"
 	"strings"
 
+	"github.com/up9inc/mizu/tap/api"
+	"k8s.io/apimachinery/pkg/util/json"
+
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 
@@ -51,6 +52,10 @@ func InitConfig(cmd *cobra.Command) error {
 
 	cmd.Flags().Visit(initFlag)
 
+	if err := Config.validate(); err != nil {
+		return fmt.Errorf("config validation failed, err: %v", err)
+	}
+
 	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
 	logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
 
@@ -391,12 +396,12 @@ func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.
 		TargetNamespaces:        targetNamespaces,
 		AgentImage:              Config.AgentImage,
 		PullPolicy:              Config.ImagePullPolicyStr,
-		DumpLogs:                Config.DumpLogs,
+		LogLevel:                Config.LogLevel(),
 		IgnoredUserAgents:       Config.Tap.IgnoredUserAgents,
 		TapperResources:         Config.Tap.TapperResources,
 		MizuResourcesNamespace:  Config.MizuResourcesNamespace,
 		MizuApiFilteringOptions: *mizuApiFilteringOptions,
-		AgentDatabasePath:       fmt.Sprintf("%s%s", shared.DataDirPath, "entries.db"),
+		AgentDatabasePath:       shared.DataDirPath,
 	}
 	return &config, nil
 }

cli/config/configStruct.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/mizu"
 	v1 "k8s.io/api/core/v1"
@@ -31,6 +32,16 @@ type ConfigStruct struct {
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
 	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
+	HeadlessMode           bool                        `yaml:"headless" default:"false"`
+	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
+}
+
+func(config *ConfigStruct) validate() error {
+	if _, err := logging.LogLevel(config.LogLevelStr); err != nil {
+		return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err)
+	}
+
+	return nil
 }
 
 func (config *ConfigStruct) SetDefaults() {
@@ -59,3 +70,8 @@ func (config *ConfigStruct) KubeConfigPath() string {
 	home := homedir.HomeDir()
 	return filepath.Join(home, ".kube", "config")
 }
+
+func (config *ConfigStruct) LogLevel() logging.Level {
+	logLevel, _ := logging.LogLevel(config.LogLevelStr)
+	return logLevel
+}

cli/config/configStructs/tapConfig.go

@@ -3,9 +3,10 @@ package configStructs
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/shared"
 	"regexp"
 
+	"github.com/up9inc/mizu/shared"
+
 	"github.com/up9inc/mizu/shared/units"
 )
 

cli/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/getkin/kin-openapi v0.79.0
 	github.com/google/go-github/v37 v37.0.0
 	github.com/google/uuid v1.1.2
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -78,6 +78,6 @@ func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath strin
 		logger.Log.Debugf("Successfully added file %s", GetLogFilePath())
 	}
 
-	logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath)
+	logger.Log.Infof("You can find the zip file with all logs in %s", filePath)
 	return nil
 }

debug.Dockerfile

@@ -12,7 +12,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash
+RUN apk add libpcap-dev gcc g++ make bash perl-utils
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -23,7 +23,7 @@ COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
+RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 ARG COMMIT_HASH
 ARG GIT_BRANCH
@@ -36,6 +36,12 @@ COPY tap ../tap
 COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
+# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.9/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.9/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+RUN shasum -a 256 -c basenine_linux_amd64.sha256
+RUN chmod +x ./basenine_linux_amd64
+
 COPY devops/build_extensions_debug.sh ..
 RUN cd .. && /bin/bash build_extensions_debug.sh
 
@@ -43,10 +49,12 @@ RUN cd .. && /bin/bash build_extensions_debug.sh
 FROM golang:1.16-alpine
 
 RUN apk add bash libpcap-dev tcpdump
+
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 

docs/PERMISSIONS.md

@@ -57,9 +57,11 @@ Mizu needs following permissions on your Kubernetes cluster to run properly
   - get
 ```
 
-## Permissions required for service / pod name resolving (opt)
+## Permissions required running with --daemon flag or (optional) for service / pod name resolving
 
-Optionally, for proper resolving of IP addresses to Kubernetes service name, Mizu needs below permissions:
+Mandatory permissions for running with `--daemon` flag.
+
+Optional for service/pod name resolving in non daemon mode
 
 ```yaml
 - apiGroups:

shared/consts.go

@@ -13,5 +13,7 @@ const (
 	ConfigFileName                   = "mizu-config.json"
 	GoGCEnvVar                       = "GOGC"
 	DefaultApiServerPort             = 8899
-	DebugModeEnvVar                  = "MIZU_DEBUG"
+	LogLevelEnvVar                   = "LOG_LEVEL"
+	BasenineHost                     = "localhost"
+	BaseninePort                     = "9099"
 )

shared/kubernetes/mizuTapperSyncer.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"context"
 	"fmt"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
@@ -15,18 +16,20 @@ import (
 const updateTappersDelay = 5 * time.Second
 
 type TappedPodChangeEvent struct {
-	Added   []core.Pod
-	Removed []core.Pod
+	Added                []core.Pod
+	Removed              []core.Pod
+	ExpectedTapperAmount int
 }
 
 // MizuTapperSyncer uses a k8s pod watch to update tapper daemonsets when targeted pods are removed or created
 type MizuTapperSyncer struct {
-	context             context.Context
-	CurrentlyTappedPods []core.Pod
-	config              TapperSyncerConfig
-	kubernetesProvider  *Provider
-	TapPodChangesOut    chan TappedPodChangeEvent
-	ErrorOut            chan K8sTapManagerError
+	context              context.Context
+	CurrentlyTappedPods  []core.Pod
+	config               TapperSyncerConfig
+	kubernetesProvider   *Provider
+	TapPodChangesOut     chan TappedPodChangeEvent
+	ErrorOut             chan K8sTapManagerError
+	nodeToTappedPodIPMap map[string][]string
 }
 
 type TapperSyncerConfig struct {
@@ -36,7 +39,7 @@ type TapperSyncerConfig struct {
 	AgentImage               string
 	TapperResources          shared.Resources
 	ImagePullPolicy          core.PullPolicy
-	DumpLogs                 bool
+	LogLevel                 logging.Level
 	IgnoredUserAgents        []string
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
@@ -159,9 +162,11 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
 		}
 		if len(addedPods) > 0 || len(removedPods) > 0 {
 			tapperSyncer.CurrentlyTappedPods = podsToTap
+			tapperSyncer.nodeToTappedPodIPMap = GetNodeHostToTappedPodIpsMap(tapperSyncer.CurrentlyTappedPods)
 			tapperSyncer.TapPodChangesOut <- TappedPodChangeEvent{
-				Added:   addedPods,
-				Removed: removedPods,
+				Added:                addedPods,
+				Removed:              removedPods,
+				ExpectedTapperAmount: len(tapperSyncer.nodeToTappedPodIPMap),
 			}
 			return nil, true
 		}
@@ -170,9 +175,7 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
 }
 
 func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
-	nodeToTappedPodIPMap := GetNodeHostToTappedPodIpsMap(tapperSyncer.CurrentlyTappedPods)
-
-	if len(nodeToTappedPodIPMap) > 0 {
+	if len(tapperSyncer.nodeToTappedPodIPMap) > 0 {
 		var serviceAccountName string
 		if tapperSyncer.config.MizuServiceAccountExists {
 			serviceAccountName = ServiceAccountName
@@ -187,16 +190,16 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
 			tapperSyncer.config.AgentImage,
 			TapperPodName,
 			fmt.Sprintf("%s.%s.svc.cluster.local", ApiServerPodName, tapperSyncer.config.MizuResourcesNamespace),
-			nodeToTappedPodIPMap,
+			tapperSyncer.nodeToTappedPodIPMap,
 			serviceAccountName,
 			tapperSyncer.config.TapperResources,
 			tapperSyncer.config.ImagePullPolicy,
 			tapperSyncer.config.MizuApiFilteringOptions,
-			tapperSyncer.config.DumpLogs,
+			tapperSyncer.config.LogLevel,
 		); err != nil {
 			return err
 		}
-		logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
+		logger.Log.Debugf("Successfully created %v tappers", len(tapperSyncer.nodeToTappedPodIPMap))
 	} else {
 		if err := tapperSyncer.kubernetesProvider.RemoveDaemonSet(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, TapperDaemonSetName); err != nil {
 			return err

shared/kubernetes/provider.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
@@ -45,6 +46,8 @@ type Provider struct {
 
 const (
 	fieldManagerName = "mizu-manager"
+	procfsVolumeName = "proc"
+	procfsMountPath  = "/hostproc"
 )
 
 func NewProvider(kubeConfigPath string) (*Provider, error) {
@@ -177,7 +180,7 @@ type ApiServerOptions struct {
 	MaxEntriesDBSizeBytes int64
 	Resources             shared.Resources
 	ImagePullPolicy       core.PullPolicy
-	DumpLogs              bool
+	LogLevel              logging.Level
 }
 
 func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, mountVolumeClaim bool, volumeClaimName string) (*core.Pod, error) {
@@ -239,22 +242,17 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 			},
 		})
 		volumeMounts = append(volumeMounts, core.VolumeMount{
-			Name:             volumeClaimName,
-			MountPath:        shared.DataDirPath,
+			Name:      volumeClaimName,
+			MountPath: shared.DataDirPath,
 		})
 	}
 
 	port := intstr.FromInt(shared.DefaultApiServerPort)
 
-	debugMode := ""
-	if opts.DumpLogs {
-		debugMode = "1"
-	}
-
 	pod := &core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      opts.PodName,
-			Labels:    map[string]string{"app": opts.PodName},
+			Name:   opts.PodName,
+			Labels: map[string]string{"app": opts.PodName},
 		},
 		Spec: core.PodSpec{
 			Containers: []core.Container{
@@ -262,16 +260,16 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 					Name:            opts.PodName,
 					Image:           opts.PodImage,
 					ImagePullPolicy: opts.ImagePullPolicy,
-					VolumeMounts: volumeMounts,
-					Command: command,
+					VolumeMounts:    volumeMounts,
+					Command:         command,
 					Env: []core.EnvVar{
 						{
 							Name:  shared.SyncEntriesConfigEnvVar,
 							Value: string(marshaledSyncEntriesConfig),
 						},
 						{
-							Name:  shared.DebugModeEnvVar,
-							Value: debugMode,
+							Name:  shared.LogLevelEnvVar,
+							Value: opts.LogLevel.String(),
 						},
 					},
 					Resources: core.ResourceRequirements{
@@ -305,7 +303,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 					},
 				},
 			},
-			Volumes: volumes,
+			Volumes:                       volumes,
 			DNSPolicy:                     core.DNSClusterFirstWithHostNet,
 			TerminationGracePeriodSeconds: new(int64),
 		},
@@ -318,7 +316,6 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 	return pod, nil
 }
 
-
 func (provider *Provider) CreatePod(ctx context.Context, namespace string, podSpec *core.Pod) (*core.Pod, error) {
 	return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, podSpec, metav1.CreateOptions{})
 }
@@ -333,14 +330,14 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string
 	}
 	deployment := &v1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      deploymentName,
+			Name: deploymentName,
 		},
 		Spec: v1.DeploymentSpec{
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{"app": podSpec.ObjectMeta.Labels["app"]},
 			},
-			Template:                *podTemplate,
-			Strategy:                v1.DeploymentStrategy{},
+			Template: *podTemplate,
+			Strategy: v1.DeploymentStrategy{},
 		},
 	}
 	return provider.clientSet.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{})
@@ -349,7 +346,7 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string
 func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) {
 	service := core.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      serviceName,
+			Name: serviceName,
 		},
 		Spec: core.ServiceSpec{
 			Ports:    []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}},
@@ -381,8 +378,8 @@ func (provider *Provider) doesResourceExist(resource interface{}, err error) (bo
 func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error {
 	serviceAccount := &core.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      serviceAccountName,
-			Labels:    map[string]string{"mizu-cli-version": version},
+			Name:   serviceAccountName,
+			Labels: map[string]string{"mizu-cli-version": version},
 		},
 	}
 	clusterRole := &rbac.ClusterRole{
@@ -434,8 +431,8 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
 func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error {
 	serviceAccount := &core.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      serviceAccountName,
-			Labels:    map[string]string{"mizu-cli-version": version},
+			Name:   serviceAccountName,
+			Labels: map[string]string{"mizu-cli-version": version},
 		},
 	}
 	role := &rbac.Role{
@@ -608,7 +605,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      configMapName,
+			Name: configMapName,
 		},
 		Data: configMapData,
 	}
@@ -618,7 +615,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, dumpLogs bool) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level) error {
 	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodIPMap) == 0 {
@@ -641,11 +638,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"--tap",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
 		"--nodefrag",
-	}
-
-	debugMode := ""
-	if dumpLogs {
-		debugMode = "1"
+		"--procfs", procfsMountPath,
 	}
 
 	agentContainer := applyconfcore.Container()
@@ -655,7 +648,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
-		applyconfcore.EnvVar().WithName(shared.DebugModeEnvVar).WithValue(debugMode),
+		applyconfcore.EnvVar().WithName(shared.LogLevelEnvVar).WithValue(logLevel.String()),
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
 		applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)),
 		applyconfcore.EnvVar().WithName(shared.GoGCEnvVar).WithValue("12800"),
@@ -719,6 +712,14 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	noScheduleToleration.WithOperator(core.TolerationOpExists)
 	noScheduleToleration.WithEffect(core.TaintEffectNoSchedule)
 
+	// Host procfs is needed inside the container because we need access to
+	//	the network namespaces of processes on the machine.
+	//
+	procfsVolume := applyconfcore.Volume()
+	procfsVolume.WithName(procfsVolumeName).WithHostPath(applyconfcore.HostPathVolumeSource().WithPath("/proc"))
+	volumeMount := applyconfcore.VolumeMount().WithName(procfsVolumeName).WithMountPath(procfsMountPath).WithReadOnly(true)
+	agentContainer.WithVolumeMounts(volumeMount)
+
 	volumeName := ConfigMapName
 	configMapVolume := applyconfcore.VolumeApplyConfiguration{
 		Name: &volumeName,
@@ -747,7 +748,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	podSpec.WithContainers(agentContainer)
 	podSpec.WithAffinity(affinity)
 	podSpec.WithTolerations(noExecuteToleration, noScheduleToleration)
-	podSpec.WithVolumes(&configMapVolume)
+	podSpec.WithVolumes(&configMapVolume, procfsVolume)
 
 	podTemplate := applyconfcore.PodTemplateSpec()
 	podTemplate.WithLabels(map[string]string{"app": tapperPodName})
@@ -842,9 +843,9 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names
 		ObjectMeta: metav1.ObjectMeta{
 			Name: volumeClaimName,
 		},
-		Spec:       core.PersistentVolumeClaimSpec{
-			AccessModes:      []core.PersistentVolumeAccessMode{core.ReadWriteOnce},
-			Resources:        core.ResourceRequirements{
+		Spec: core.PersistentVolumeClaimSpec{
+			AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce},
+			Resources: core.ResourceRequirements{
 				Limits: core.ResourceList{
 					core.ResourceStorage: *sizeLimitQuantity,
 				},

shared/kubernetes/proxy.go

@@ -28,9 +28,9 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
 		return err
 	}
 	mux := http.NewServeMux()
-	mux.Handle(k8sProxyApiPrefix, proxyHandler)
+	mux.Handle(k8sProxyApiPrefix, getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
 	mux.Handle("/static/", getRerouteHttpHandlerMizuStatic(proxyHandler, mizuNamespace, mizuServiceName))
-	mux.Handle("/mizu/", getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
+
 
 	l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", proxyHost, int(mizuPort)))
 	if err != nil {
@@ -45,16 +45,21 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
 }
 
 func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
-	return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort)
+	return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy", mizuNamespace, mizuServiceName, mizuServicePort)
 }
 
 func GetMizuApiServerProxiedHostAndPath(mizuPort uint16) string {
-	return fmt.Sprintf("localhost:%d/mizu", mizuPort)
+	return fmt.Sprintf("localhost:%d", mizuPort)
 }
 
 func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
+		proxiedPath := getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName)
+
+		//avoid redirecting several times
+		if !strings.Contains(r.URL.Path, proxiedPath) {
+			r.URL.Path = fmt.Sprintf("%s%s", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), r.URL.Path)
+		}
 		proxyHandler.ServeHTTP(w, r)
 	})
 }

shared/logger/logger.go

@@ -9,7 +9,7 @@ import (
 var Log = logging.MustGetLogger("mizu")
 
 var format = logging.MustStringFormatter(
-	`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
+	`%{time:2006-01-02T15:04:05.999Z-07:00} %{level:-5s} ▶ %{message} ▶ %{pid} %{shortfile} %{shortfunc}`,
 )
 
 func InitLogger(logPath string) {

shared/models.go

@@ -1,9 +1,10 @@
 package shared
 
 import (
+	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
-	"log"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -17,6 +18,9 @@ const (
 	WebSocketMessageTypeUpdateStatus  WebSocketMessageType = "status"
 	WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
 	WebsocketMessageTypeOutboundLink  WebSocketMessageType = "outboundLink"
+	WebSocketMessageTypeToast         WebSocketMessageType = "toast"
+	WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata"
+	WebSocketMessageTypeStartTime     WebSocketMessageType = "startTime"
 )
 
 type Resources struct {
@@ -33,7 +37,7 @@ type MizuAgentConfig struct {
 	TargetNamespaces        []string                    `json:"targetNamespaces"`
 	AgentImage              string                      `json:"agentImage"`
 	PullPolicy              string                      `json:"pullPolicy"`
-	DumpLogs                bool                        `json:"dumpLogs"`
+	LogLevel                logging.Level               `json:"logLevel"`
 	IgnoredUserAgents       []string                    `json:"ignoredUserAgents"`
 	TapperResources         Resources                   `json:"tapperResources"`
 	MizuResourcesNamespace  string                      `json:"mizuResourceNamespace"`
@@ -137,14 +141,12 @@ func (r *RulePolicy) validateType() bool {
 	permitedTypes := []string{"json", "header", "slo"}
 	_, found := Find(permitedTypes, r.Type)
 	if !found {
-		log.Printf("Error: %s. ", r.Name)
-		log.Printf("Only json, header and slo types are supported on rule definition. This rule will be ignored\n")
+		logger.Log.Errorf("Only json, header and slo types are supported on rule definition. This rule will be ignored. rule name: %s", r.Name)
 		found = false
 	}
 	if strings.ToLower(r.Type) == "slo" {
 		if r.ResponseTime <= 0 {
-			log.Printf("Error: %s. ", r.Name)
-			log.Printf("When type=slo, the field response-time should be specified and have a value >= 1\n\n")
+			logger.Log.Errorf("When rule type is slo, the field response-time should be specified and have a value >= 1. rule name: %s", r.Name)
 			found = false
 		}
 	}

tap/api/api.go

@@ -18,7 +18,8 @@ import (
 type Protocol struct {
 	Name            string   `json:"name"`
 	LongName        string   `json:"longName"`
-	Abbreviation    string   `json:"abbreviation"`
+	Abbreviation    string   `json:"abbr"`
+	Macro           string   `json:"macro"`
 	Version         string   `json:"version"`
 	BackgroundColor string   `json:"backgroundColor"`
 	ForegroundColor string   `json:"foregroundColor"`
@@ -28,6 +29,12 @@ type Protocol struct {
 	Priority        uint8    `json:"priority"`
 }
 
+type TCP struct {
+	IP   string `json:"ip"`
+	Port string `json:"port"`
+	Name string `json:"name"`
+}
+
 type Extension struct {
 	Protocol   *Protocol
 	Path       string
@@ -74,6 +81,7 @@ type OutputChannelItem struct {
 	Timestamp      int64
 	ConnectionInfo *ConnectionInfo
 	Pair           *RequestResponsePair
+	Summary        *BaseEntryDetails
 }
 
 type SuperTimer struct {
@@ -89,9 +97,10 @@ type Dissector interface {
 	Register(*Extension)
 	Ping()
 	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error
-	Analyze(item *OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *MizuEntry
+	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *MizuEntry
 	Summarize(entry *MizuEntry) *BaseEntryDetails
-	Represent(entry *MizuEntry) (protocol Protocol, object []byte, bodySize int64, err error)
+	Represent(pIn Protocol, request map[string]interface{}, response map[string]interface{}) (pOut Protocol, object []byte, bodySize int64, err error)
+	Macros() map[string]string
 }
 
 type Emitting struct {
@@ -109,39 +118,36 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 }
 
 type MizuEntry struct {
-	ID                      uint `gorm:"primarykey"`
-	CreatedAt               time.Time
-	UpdatedAt               time.Time
-	ProtocolName            string         `json:"protocolName" gorm:"column:protocolName"`
-	ProtocolLongName        string         `json:"protocolLongName" gorm:"column:protocolLongName"`
-	ProtocolAbbreviation    string         `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"`
-	ProtocolVersion         string         `json:"protocolVersion" gorm:"column:protocolVersion"`
-	ProtocolBackgroundColor string         `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"`
-	ProtocolForegroundColor string         `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"`
-	ProtocolFontSize        int8           `json:"protocolFontSize" gorm:"column:protocolFontSize"`
-	ProtocolReferenceLink   string         `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"`
-	Entry                   string         `json:"entry,omitempty" gorm:"column:entry"`
-	EntryId                 string         `json:"entryId" gorm:"column:entryId"`
-	Url                     string         `json:"url" gorm:"column:url"`
-	Method                  string         `json:"method" gorm:"column:method"`
-	Status                  int            `json:"status" gorm:"column:status"`
-	RequestSenderIp         string         `json:"requestSenderIp" gorm:"column:requestSenderIp"`
-	Service                 string         `json:"service" gorm:"column:service"`
-	Timestamp               int64          `json:"timestamp" gorm:"column:timestamp"`
-	ElapsedTime             int64          `json:"elapsedTime" gorm:"column:elapsedTime"`
-	Path                    string         `json:"path" gorm:"column:path"`
-	ResolvedSource          string         `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
-	ResolvedDestination     string         `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
-	SourceIp                string         `json:"sourceIp,omitempty" gorm:"column:sourceIp"`
-	DestinationIp           string         `json:"destinationIp,omitempty" gorm:"column:destinationIp"`
-	SourcePort              string         `json:"sourcePort,omitempty" gorm:"column:sourcePort"`
-	DestinationPort         string         `json:"destinationPort,omitempty" gorm:"column:destinationPort"`
-	IsOutgoing              bool           `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
-	ContractStatus          ContractStatus `json:"contractStatus,omitempty" gorm:"column:contractStatus"`
-	ContractRequestReason   string         `json:"contractRequestReason,omitempty" gorm:"column:contractRequestReason"`
-	ContractResponseReason  string         `json:"contractResponseReason,omitempty" gorm:"column:contractResponseReason"`
-	ContractContent         string         `json:"contractContent,omitempty" gorm:"column:contractContent"`
-	EstimatedSizeBytes      int            `json:"-" gorm:"column:estimatedSizeBytes"`
+	Id                     uint                   `json:"id"`
+	Protocol               Protocol               `json:"proto"`
+	Source                 *TCP                   `json:"src"`
+	Destination            *TCP                   `json:"dst"`
+	Outgoing               bool                   `json:"outgoing"`
+	Timestamp              int64                  `json:"timestamp"`
+	StartTime              time.Time              `json:"startTime"`
+	Request                map[string]interface{} `json:"request"`
+	Response               map[string]interface{} `json:"response"`
+	Base                   *BaseEntryDetails      `json:"base"`
+	Summary                string                 `json:"summary"`
+	Url                    string                 `json:"url"`
+	Method                 string                 `json:"method"`
+	Status                 int                    `json:"status"`
+	RequestSenderIp        string                 `json:"requestSenderIp"`
+	Service                string                 `json:"service"`
+	ElapsedTime            int64                  `json:"elapsedTime"`
+	Path                   string                 `json:"path"`
+	ResolvedSource         string                 `json:"resolvedSource,omitempty"`
+	ResolvedDestination    string                 `json:"resolvedDestination,omitempty"`
+	SourceIp               string                 `json:"sourceIp,omitempty"`
+	DestinationIp          string                 `json:"destinationIp,omitempty"`
+	SourcePort             string                 `json:"sourcePort,omitempty"`
+	DestinationPort        string                 `json:"destinationPort,omitempty"`
+	IsOutgoing             bool                   `json:"isOutgoing,omitempty"`
+	ContractStatus         ContractStatus         `json:"contractStatus,omitempty"`
+	ContractRequestReason  string                 `json:"contractRequestReason,omitempty"`
+	ContractResponseReason string                 `json:"contractResponseReason,omitempty"`
+	ContractContent        string                 `json:"contractContent,omitempty"`
+	HTTPPair               string                 `json:"httpPair,omitempty"`
 }
 
 type MizuEntryWrapper struct {
@@ -154,7 +160,7 @@ type MizuEntryWrapper struct {
 }
 
 type BaseEntryDetails struct {
-	Id              string          `json:"id,omitempty"`
+	Id              uint            `json:"id"`
 	Protocol        Protocol        `json:"protocol,omitempty"`
 	Url             string          `json:"url,omitempty"`
 	RequestSenderIp string          `json:"requestSenderIp,omitempty"`
@@ -194,17 +200,8 @@ type DataUnmarshaler interface {
 }
 
 func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
-	bed.Protocol = Protocol{
-		Name:            entry.ProtocolName,
-		LongName:        entry.ProtocolLongName,
-		Abbreviation:    entry.ProtocolAbbreviation,
-		Version:         entry.ProtocolVersion,
-		BackgroundColor: entry.ProtocolBackgroundColor,
-		ForegroundColor: entry.ProtocolForegroundColor,
-		FontSize:        entry.ProtocolFontSize,
-		ReferenceLink:   entry.ProtocolReferenceLink,
-	}
-	bed.Id = entry.EntryId
+	bed.Protocol = entry.Protocol
+	bed.Id = entry.Id
 	bed.Url = entry.Url
 	bed.RequestSenderIp = entry.RequestSenderIp
 	bed.Service = entry.Service
@@ -228,6 +225,21 @@ const (
 	BODY  string = "body"
 )
 
+type SectionData struct {
+	Type     string `json:"type"`
+	Title    string `json:"title"`
+	Data     string `json:"data"`
+	Encoding string `json:"encoding,omitempty"`
+	MimeType string `json:"mimeType,omitempty"`
+	Selector string `json:"selector,omitempty"`
+}
+
+type TableData struct {
+	Name     string      `json:"name"`
+	Value    interface{} `json:"value"`
+	Selector string      `json:"selector"`
+}
+
 const (
 	TypeHttpRequest = iota
 	TypeHttpResponse
@@ -275,7 +287,7 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) {
 			RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)},
 		})
 	default:
-		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type))
+		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s", h.Type))
 	}
 }
 

tap/diagnose/diagnose.go

@@ -39,7 +39,7 @@ func StartMemoryProfiler(envDumpPath string, envTimeInterval string) {
 
 			filename := fmt.Sprintf("%s/%s__mem.prof", dumpPath, t.Format("15_04_05"))
 
-			logger.Log.Infof("Writing memory profile to %s\n", filename)
+			logger.Log.Infof("Writing memory profile to %s", filename)
 
 			f, err := os.Create(filename)
 			if err != nil {

tap/extensions/amqp/helpers.go

@@ -131,97 +131,109 @@ func representProperties(properties map[string]interface{}, rep []interface{}) (
 	userId := ""
 	appId := ""
 
-	if properties["ContentType"] != nil {
-		contentType = properties["ContentType"].(string)
+	if properties["contentType"] != nil {
+		contentType = properties["contentType"].(string)
 	}
-	if properties["ContentEncoding"] != nil {
-		contentEncoding = properties["ContentEncoding"].(string)
+	if properties["contentEncoding"] != nil {
+		contentEncoding = properties["contentEncoding"].(string)
 	}
-	if properties["Delivery Mode"] != nil {
-		deliveryMode = fmt.Sprintf("%g", properties["DeliveryMode"].(float64))
+	if properties["deliveryMode"] != nil {
+		deliveryMode = fmt.Sprintf("%g", properties["deliveryMode"].(float64))
 	}
-	if properties["Priority"] != nil {
-		priority = fmt.Sprintf("%g", properties["Priority"].(float64))
+	if properties["priority"] != nil {
+		priority = fmt.Sprintf("%g", properties["priority"].(float64))
 	}
-	if properties["CorrelationId"] != nil {
-		correlationId = properties["CorrelationId"].(string)
+	if properties["correlationId"] != nil {
+		correlationId = properties["correlationId"].(string)
 	}
-	if properties["ReplyTo"] != nil {
-		replyTo = properties["ReplyTo"].(string)
+	if properties["replyTo"] != nil {
+		replyTo = properties["replyTo"].(string)
 	}
-	if properties["Expiration"] != nil {
-		expiration = properties["Expiration"].(string)
+	if properties["expiration"] != nil {
+		expiration = properties["expiration"].(string)
 	}
-	if properties["MessageId"] != nil {
-		messageId = properties["MessageId"].(string)
+	if properties["messageId"] != nil {
+		messageId = properties["messageId"].(string)
 	}
-	if properties["Timestamp"] != nil {
-		timestamp = properties["Timestamp"].(string)
+	if properties["timestamp"] != nil {
+		timestamp = properties["timestamp"].(string)
 	}
-	if properties["Type"] != nil {
-		_type = properties["Type"].(string)
+	if properties["type"] != nil {
+		_type = properties["type"].(string)
 	}
-	if properties["UserId"] != nil {
-		userId = properties["UserId"].(string)
+	if properties["userId"] != nil {
+		userId = properties["userId"].(string)
 	}
-	if properties["AppId"] != nil {
-		appId = properties["AppId"].(string)
+	if properties["appId"] != nil {
+		appId = properties["appId"].(string)
 	}
 
-	props, _ := json.Marshal([]map[string]string{
+	props, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Content Type",
-			"value": contentType,
+			Name:     "Content Type",
+			Value:    contentType,
+			Selector: `request.properties.contentType`,
 		},
 		{
-			"name":  "Content Encoding",
-			"value": contentEncoding,
+			Name:     "Content Encoding",
+			Value:    contentEncoding,
+			Selector: `request.properties.contentEncoding`,
 		},
 		{
-			"name":  "Delivery Mode",
-			"value": deliveryMode,
+			Name:     "Delivery Mode",
+			Value:    deliveryMode,
+			Selector: `request.properties.deliveryMode`,
 		},
 		{
-			"name":  "Priority",
-			"value": priority,
+			Name:     "Priority",
+			Value:    priority,
+			Selector: `request.properties.priority`,
 		},
 		{
-			"name":  "Correlation ID",
-			"value": correlationId,
+			Name:     "Correlation ID",
+			Value:    correlationId,
+			Selector: `request.properties.correlationId`,
 		},
 		{
-			"name":  "Reply To",
-			"value": replyTo,
+			Name:     "Reply To",
+			Value:    replyTo,
+			Selector: `request.properties.replyTo`,
 		},
 		{
-			"name":  "Expiration",
-			"value": expiration,
+			Name:     "Expiration",
+			Value:    expiration,
+			Selector: `request.properties.expiration`,
 		},
 		{
-			"name":  "Message ID",
-			"value": messageId,
+			Name:     "Message ID",
+			Value:    messageId,
+			Selector: `request.properties.messageId`,
 		},
 		{
-			"name":  "Timestamp",
-			"value": timestamp,
+			Name:     "Timestamp",
+			Value:    timestamp,
+			Selector: `request.properties.timestamp`,
 		},
 		{
-			"name":  "Type",
-			"value": _type,
+			Name:     "Type",
+			Value:    _type,
+			Selector: `request.properties.type`,
 		},
 		{
-			"name":  "User ID",
-			"value": userId,
+			Name:     "User ID",
+			Value:    userId,
+			Selector: `request.properties.userId`,
 		},
 		{
-			"name":  "App ID",
-			"value": appId,
+			Name:     "App ID",
+			Value:    appId,
+			Selector: `request.properties.appId`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Properties",
-		"data":  string(props),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Properties",
+		Data:  string(props),
 	})
 
 	return rep, contentType, contentEncoding
@@ -230,56 +242,62 @@ func representProperties(properties map[string]interface{}, rep []interface{}) (
 func representBasicPublish(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "Routing Key",
-			"value": event["RoutingKey"].(string),
+			Name:     "Routing Key",
+			Value:    event["routingKey"].(string),
+			Selector: `request.routingKey`,
 		},
 		{
-			"name":  "Mandatory",
-			"value": strconv.FormatBool(event["Mandatory"].(bool)),
+			Name:     "Mandatory",
+			Value:    strconv.FormatBool(event["mandatory"].(bool)),
+			Selector: `request.mandatory`,
 		},
 		{
-			"name":  "Immediate",
-			"value": strconv.FormatBool(event["Immediate"].(bool)),
+			Name:     "Immediate",
+			Value:    strconv.FormatBool(event["immediate"].(bool)),
+			Selector: `request.immediate`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	properties := event["Properties"].(map[string]interface{})
+	properties := event["properties"].(map[string]interface{})
 	rep, contentType, _ := representProperties(properties, rep)
 
-	if properties["Headers"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range properties["Headers"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if properties["headers"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range properties["headers"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Headers",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Headers",
+			Data:  string(headersMarshaled),
 		})
 	}
 
-	if event["Body"] != nil {
-		rep = append(rep, map[string]string{
-			"type":      api.BODY,
-			"title":     "Body",
-			"encoding":  "base64",
-			"mime_type": contentType,
-			"data":      event["Body"].(string),
+	if event["body"] != nil {
+		rep = append(rep, api.SectionData{
+			Type:     api.BODY,
+			Title:    "Body",
+			Encoding: "base64",
+			MimeType: contentType,
+			Data:     event["body"].(string),
+			Selector: `request.body`,
 		})
 	}
 
@@ -293,70 +311,77 @@ func representBasicDeliver(event map[string]interface{}) []interface{} {
 	deliveryTag := ""
 	redelivered := ""
 
-	if event["ConsumerTag"] != nil {
-		consumerTag = event["ConsumerTag"].(string)
+	if event["consumerTag"] != nil {
+		consumerTag = event["consumerTag"].(string)
 	}
-	if event["DeliveryTag"] != nil {
-		deliveryTag = fmt.Sprintf("%g", event["DeliveryTag"].(float64))
+	if event["deliveryTag"] != nil {
+		deliveryTag = fmt.Sprintf("%g", event["deliveryTag"].(float64))
 	}
-	if event["Redelivered"] != nil {
-		redelivered = strconv.FormatBool(event["Redelivered"].(bool))
+	if event["redelivered"] != nil {
+		redelivered = strconv.FormatBool(event["redelivered"].(bool))
 	}
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Consumer Tag",
-			"value": consumerTag,
+			Name:     "Consumer Tag",
+			Value:    consumerTag,
+			Selector: `request.consumerTag`,
 		},
 		{
-			"name":  "Delivery Tag",
-			"value": deliveryTag,
+			Name:     "Delivery Tag",
+			Value:    deliveryTag,
+			Selector: `request.deliveryTag`,
 		},
 		{
-			"name":  "Redelivered",
-			"value": redelivered,
+			Name:     "Redelivered",
+			Value:    redelivered,
+			Selector: `request.redelivered`,
 		},
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "Routing Key",
-			"value": event["RoutingKey"].(string),
+			Name:     "Routing Key",
+			Value:    event["routingKey"].(string),
+			Selector: `request.routingKey`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	properties := event["Properties"].(map[string]interface{})
+	properties := event["properties"].(map[string]interface{})
 	rep, contentType, _ := representProperties(properties, rep)
 
-	if properties["Headers"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range properties["Headers"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if properties["headers"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range properties["headers"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Headers",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Headers",
+			Data:  string(headersMarshaled),
 		})
 	}
 
-	if event["Body"] != nil {
-		rep = append(rep, map[string]string{
-			"type":      api.BODY,
-			"title":     "Body",
-			"encoding":  "base64",
-			"mime_type": contentType,
-			"data":      event["Body"].(string),
+	if event["body"] != nil {
+		rep = append(rep, api.SectionData{
+			Type:     api.BODY,
+			Title:    "Body",
+			Encoding: "base64",
+			MimeType: contentType,
+			Data:     event["body"].(string),
+			Selector: `request.body`,
 		})
 	}
 
@@ -366,51 +391,58 @@ func representBasicDeliver(event map[string]interface{}) []interface{} {
 func representQueueDeclare(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Queue",
-			"value": event["Queue"].(string),
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Passive",
-			"value": strconv.FormatBool(event["Passive"].(bool)),
+			Name:     "Passive",
+			Value:    strconv.FormatBool(event["passive"].(bool)),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Durable",
-			"value": strconv.FormatBool(event["Durable"].(bool)),
+			Name:     "Durable",
+			Value:    strconv.FormatBool(event["durable"].(bool)),
+			Selector: `request.durable`,
 		},
 		{
-			"name":  "Exclusive",
-			"value": strconv.FormatBool(event["Exclusive"].(bool)),
+			Name:     "Exclusive",
+			Value:    strconv.FormatBool(event["exclusive"].(bool)),
+			Selector: `request.exclusive`,
 		},
 		{
-			"name":  "Auto Delete",
-			"value": strconv.FormatBool(event["AutoDelete"].(bool)),
+			Name:     "Auto Delete",
+			Value:    strconv.FormatBool(event["autoDelete"].(bool)),
+			Selector: `request.autoDelete`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -420,55 +452,63 @@ func representQueueDeclare(event map[string]interface{}) []interface{} {
 func representExchangeDeclare(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "Type",
-			"value": event["Type"].(string),
+			Name:     "Type",
+			Value:    event["type"].(string),
+			Selector: `request.type`,
 		},
 		{
-			"name":  "Passive",
-			"value": strconv.FormatBool(event["Passive"].(bool)),
+			Name:     "Passive",
+			Value:    strconv.FormatBool(event["passive"].(bool)),
+			Selector: `request.passive`,
 		},
 		{
-			"name":  "Durable",
-			"value": strconv.FormatBool(event["Durable"].(bool)),
+			Name:     "Durable",
+			Value:    strconv.FormatBool(event["durable"].(bool)),
+			Selector: `request.durable`,
 		},
 		{
-			"name":  "Auto Delete",
-			"value": strconv.FormatBool(event["AutoDelete"].(bool)),
+			Name:     "Auto Delete",
+			Value:    strconv.FormatBool(event["autoDelete"].(bool)),
+			Selector: `request.autoDelete`,
 		},
 		{
-			"name":  "Internal",
-			"value": strconv.FormatBool(event["Internal"].(bool)),
+			Name:     "Internal",
+			Value:    strconv.FormatBool(event["internal"].(bool)),
+			Selector: `request.internal`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -478,33 +518,37 @@ func representExchangeDeclare(event map[string]interface{}) []interface{} {
 func representConnectionStart(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Version Major",
-			"value": fmt.Sprintf("%g", event["VersionMajor"].(float64)),
+			Name:     "Version Major",
+			Value:    fmt.Sprintf("%g", event["versionMajor"].(float64)),
+			Selector: `request.versionMajor`,
 		},
 		{
-			"name":  "Version Minor",
-			"value": fmt.Sprintf("%g", event["VersionMinor"].(float64)),
+			Name:     "Version Minor",
+			Value:    fmt.Sprintf("%g", event["versionMinor"].(float64)),
+			Selector: `request.versionMinor`,
 		},
 		{
-			"name":  "Mechanisms",
-			"value": event["Mechanisms"].(string),
+			Name:     "Mechanisms",
+			Value:    event["mechanisms"].(string),
+			Selector: `request.mechanisms`,
 		},
 		{
-			"name":  "Locales",
-			"value": event["Locales"].(string),
+			Name:     "Locales",
+			Value:    event["locales"].(string),
+			Selector: `request.locales`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["ServerProperties"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["ServerProperties"].(map[string]interface{}) {
+	if event["serverProperties"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["serverProperties"].(map[string]interface{}) {
 			var outcome string
 			switch value.(type) {
 			case string:
@@ -517,16 +561,17 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 			default:
 				panic("Unknown data type for the server property!")
 			}
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": outcome,
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    outcome,
+				Selector: fmt.Sprintf(`request.serverProperties["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Server Properties",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Server Properties",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -536,28 +581,32 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 func representConnectionClose(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Reply Code",
-			"value": fmt.Sprintf("%g", event["ReplyCode"].(float64)),
+			Name:     "Reply Code",
+			Value:    fmt.Sprintf("%g", event["replyCode"].(float64)),
+			Selector: `request.replyCode`,
 		},
 		{
-			"name":  "Reply Text",
-			"value": event["ReplyText"].(string),
+			Name:     "Reply Text",
+			Value:    event["replyText"].(string),
+			Selector: `request.replyText`,
 		},
 		{
-			"name":  "Class ID",
-			"value": fmt.Sprintf("%g", event["ClassId"].(float64)),
+			Name:     "Class ID",
+			Value:    fmt.Sprintf("%g", event["classId"].(float64)),
+			Selector: `request.classId`,
 		},
 		{
-			"name":  "Method ID",
-			"value": fmt.Sprintf("%g", event["MethodId"].(float64)),
+			Name:     "Method ID",
+			Value:    fmt.Sprintf("%g", event["methodId"].(float64)),
+			Selector: `request.methodId`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
 	return rep
@@ -566,43 +615,48 @@ func representConnectionClose(event map[string]interface{}) []interface{} {
 func representQueueBind(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Queue",
-			"value": event["Queue"].(string),
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "RoutingKey",
-			"value": event["RoutingKey"].(string),
+			Name:     "RoutingKey",
+			Value:    event["routingKey"].(string),
+			Selector: `request.routingKey`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -612,51 +666,58 @@ func representQueueBind(event map[string]interface{}) []interface{} {
 func representBasicConsume(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Queue",
-			"value": event["Queue"].(string),
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Consumer Tag",
-			"value": event["ConsumerTag"].(string),
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `request.consumerTag`,
 		},
 		{
-			"name":  "No Local",
-			"value": strconv.FormatBool(event["NoLocal"].(bool)),
+			Name:     "No Local",
+			Value:    strconv.FormatBool(event["noLocal"].(bool)),
+			Selector: `request.noLocal`,
 		},
 		{
-			"name":  "No Ack",
-			"value": strconv.FormatBool(event["NoAck"].(bool)),
+			Name:     "No Ack",
+			Value:    strconv.FormatBool(event["noAck"].(bool)),
+			Selector: `request.noAck`,
 		},
 		{
-			"name":  "Exclusive",
-			"value": strconv.FormatBool(event["Exclusive"].(bool)),
+			Name:     "Exclusive",
+			Value:    strconv.FormatBool(event["exclusive"].(bool)),
+			Selector: `request.exclusive`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 

tap/extensions/amqp/main.go

@@ -16,6 +16,7 @@ var protocol api.Protocol = api.Protocol{
 	Name:            "amqp",
 	LongName:        "Advanced Message Queuing Protocol 0-9-1",
 	Abbreviation:    "AMQP",
+	Macro:           "amqp",
 	Version:         "0-9-1",
 	BackgroundColor: "#ff6600",
 	ForegroundColor: "#ffffff",
@@ -36,7 +37,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", protocol.Name)
+	log.Printf("pong %s", protocol.Name)
 }
 
 const amqpRequest string = "amqp_request"
@@ -217,12 +218,12 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			}
 
 		default:
-			// log.Printf("unexpected frame: %+v\n", f)
+			// log.Printf("unexpected frame: %+v", f)
 		}
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 	service := "amqp"
@@ -235,75 +236,79 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	summary := ""
 	switch request["method"] {
 	case basicMethodMap[40]:
-		summary = reqDetails["Exchange"].(string)
+		summary = reqDetails["exchange"].(string)
 		break
 	case basicMethodMap[60]:
-		summary = reqDetails["Exchange"].(string)
+		summary = reqDetails["exchange"].(string)
 		break
 	case exchangeMethodMap[10]:
-		summary = reqDetails["Exchange"].(string)
+		summary = reqDetails["exchange"].(string)
 		break
 	case queueMethodMap[10]:
-		summary = reqDetails["Queue"].(string)
+		summary = reqDetails["queue"].(string)
 		break
 	case connectionMethodMap[10]:
 		summary = fmt.Sprintf(
 			"%s.%s",
-			strconv.Itoa(int(reqDetails["VersionMajor"].(float64))),
-			strconv.Itoa(int(reqDetails["VersionMinor"].(float64))),
+			strconv.Itoa(int(reqDetails["versionMajor"].(float64))),
+			strconv.Itoa(int(reqDetails["versionMinor"].(float64))),
 		)
 		break
 	case connectionMethodMap[50]:
-		summary = reqDetails["ReplyText"].(string)
+		summary = reqDetails["replyText"].(string)
 		break
 	case queueMethodMap[20]:
-		summary = reqDetails["Queue"].(string)
+		summary = reqDetails["queue"].(string)
 		break
 	case basicMethodMap[20]:
-		summary = reqDetails["Queue"].(string)
+		summary = reqDetails["queue"].(string)
 		break
 	}
 
 	request["url"] = summary
-	entryBytes, _ := json.Marshal(item.Pair)
+	reqDetails["method"] = request["method"]
 	return &api.MizuEntry{
-		ProtocolName:            protocol.Name,
-		ProtocolLongName:        protocol.LongName,
-		ProtocolAbbreviation:    protocol.Abbreviation,
-		ProtocolVersion:         protocol.Version,
-		ProtocolBackgroundColor: protocol.BackgroundColor,
-		ProtocolForegroundColor: protocol.ForegroundColor,
-		ProtocolFontSize:        protocol.FontSize,
-		ProtocolReferenceLink:   protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, summary),
-		Method:                  request["method"].(string),
-		Status:                  0,
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             0,
-		Path:                    summary,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Url:                 fmt.Sprintf("%s%s", service, summary),
+		Method:              request["method"].(string),
+		Status:              0,
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         0,
+		Summary:             summary,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
 	}
 
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        protocol,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -320,39 +325,35 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	}
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	p = protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	protoOut = protocol
 	bodySize = 0
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
 	var repRequest []interface{}
-	details := request["details"].(map[string]interface{})
 	switch request["method"].(string) {
 	case basicMethodMap[40]:
-		repRequest = representBasicPublish(details)
+		repRequest = representBasicPublish(request)
 		break
 	case basicMethodMap[60]:
-		repRequest = representBasicDeliver(details)
+		repRequest = representBasicDeliver(request)
 		break
 	case queueMethodMap[10]:
-		repRequest = representQueueDeclare(details)
+		repRequest = representQueueDeclare(request)
 		break
 	case exchangeMethodMap[10]:
-		repRequest = representExchangeDeclare(details)
+		repRequest = representExchangeDeclare(request)
 		break
 	case connectionMethodMap[10]:
-		repRequest = representConnectionStart(details)
+		repRequest = representConnectionStart(request)
 		break
 	case connectionMethodMap[50]:
-		repRequest = representConnectionClose(details)
+		repRequest = representConnectionClose(request)
 		break
 	case queueMethodMap[20]:
-		repRequest = representQueueBind(details)
+		repRequest = representQueueBind(request)
 		break
 	case basicMethodMap[20]:
-		repRequest = representBasicConsume(details)
+		repRequest = representBasicConsume(request)
 		break
 	}
 	representation["request"] = repRequest
@@ -360,4 +361,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`amqp`: fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation),
+	}
+}
+
 var Dissector dissecting

tap/extensions/amqp/spec091.go

@@ -71,11 +71,11 @@ func isSoftExceptionCode(code int) bool {
 }
 
 type ConnectionStart struct {
-	VersionMajor     byte
-	VersionMinor     byte
-	ServerProperties Table
-	Mechanisms       string
-	Locales          string
+	VersionMajor     byte   `json:"versionMajor"`
+	VersionMinor     byte   `json:"versionMinor"`
+	ServerProperties Table  `json:"serverProperties"`
+	Mechanisms       string `json:"mechanisms"`
+	Locales          string `json:"locales"`
 }
 
 func (msg *ConnectionStart) id() (uint16, uint16) {
@@ -429,10 +429,10 @@ func (msg *connectionOpenOk) read(r io.Reader) (err error) {
 }
 
 type ConnectionClose struct {
-	ReplyCode uint16
-	ReplyText string
-	ClassId   uint16
-	MethodId  uint16
+	ReplyCode uint16 `json:"relyCode"`
+	ReplyText string `json:"replyText"`
+	ClassId   uint16 `json:"classId"`
+	MethodId  uint16 `json:"methodId"`
 }
 
 func (msg *ConnectionClose) id() (uint16, uint16) {
@@ -767,14 +767,14 @@ func (msg *channelCloseOk) read(r io.Reader) (err error) {
 
 type ExchangeDeclare struct {
 	reserved1  uint16
-	Exchange   string
-	Type       string
-	Passive    bool
-	Durable    bool
-	AutoDelete bool
-	Internal   bool
-	NoWait     bool
-	Arguments  Table
+	Exchange   string `json:"exchange"`
+	Type       string `json:"type"`
+	Passive    bool   `json:"passive"`
+	Durable    bool   `json:"durable"`
+	AutoDelete bool   `json:"autoDelete"`
+	Internal   bool   `json:"internal"`
+	NoWait     bool   `json:"noWait"`
+	Arguments  Table  `json:"arguments"`
 }
 
 func (msg *ExchangeDeclare) id() (uint16, uint16) {
@@ -1163,13 +1163,13 @@ func (msg *exchangeUnbindOk) read(r io.Reader) (err error) {
 
 type QueueDeclare struct {
 	reserved1  uint16
-	Queue      string
-	Passive    bool
-	Durable    bool
-	Exclusive  bool
-	AutoDelete bool
-	NoWait     bool
-	Arguments  Table
+	Queue      string `json:"queue"`
+	Passive    bool   `json:"passive"`
+	Durable    bool   `json:"durable"`
+	Exclusive  bool   `json:"exclusive"`
+	AutoDelete bool   `json:"autoDelete"`
+	NoWait     bool   `json:"noWait"`
+	Arguments  Table  `json:"arguments"`
 }
 
 func (msg *QueueDeclare) id() (uint16, uint16) {
@@ -1297,11 +1297,11 @@ func (msg *QueueDeclareOk) read(r io.Reader) (err error) {
 
 type QueueBind struct {
 	reserved1  uint16
-	Queue      string
-	Exchange   string
-	RoutingKey string
-	NoWait     bool
-	Arguments  Table
+	Queue      string `json:"queue"`
+	Exchange   string `json:"exchange"`
+	RoutingKey string `json:"routingKey"`
+	NoWait     bool   `json:"noWait"`
+	Arguments  Table  `json:"arguments"`
 }
 
 func (msg *QueueBind) id() (uint16, uint16) {
@@ -1737,13 +1737,13 @@ func (msg *basicQosOk) read(r io.Reader) (err error) {
 
 type BasicConsume struct {
 	reserved1   uint16
-	Queue       string
-	ConsumerTag string
-	NoLocal     bool
-	NoAck       bool
-	Exclusive   bool
-	NoWait      bool
-	Arguments   Table
+	Queue       string `json:"queue"`
+	ConsumerTag string `json:"consumerTag"`
+	NoLocal     bool   `json:"noLocal"`
+	NoAck       bool   `json:"noAck"`
+	Exclusive   bool   `json:"exclusive"`
+	NoWait      bool   `json:"noWait"`
+	Arguments   Table  `json:"arguments"`
 }
 
 func (msg *BasicConsume) id() (uint16, uint16) {
@@ -1932,12 +1932,12 @@ func (msg *basicCancelOk) read(r io.Reader) (err error) {
 
 type BasicPublish struct {
 	reserved1  uint16
-	Exchange   string
-	RoutingKey string
-	Mandatory  bool
-	Immediate  bool
-	Properties Properties
-	Body       []byte
+	Exchange   string     `json:"exchange"`
+	RoutingKey string     `json:"routingKey"`
+	Mandatory  bool       `json:"mandatory"`
+	Immediate  bool       `json:"immediate"`
+	Properties Properties `json:"properties"`
+	Body       []byte     `json:"body"`
 }
 
 func (msg *BasicPublish) id() (uint16, uint16) {
@@ -2072,13 +2072,13 @@ func (msg *basicReturn) read(r io.Reader) (err error) {
 }
 
 type BasicDeliver struct {
-	ConsumerTag string
-	DeliveryTag uint64
-	Redelivered bool
-	Exchange    string
-	RoutingKey  string
-	Properties  Properties
-	Body        []byte
+	ConsumerTag string     `json:"consumerTag"`
+	DeliveryTag uint64     `json:"deliveryTag"`
+	Redelivered bool       `json:"redelivered"`
+	Exchange    string     `json:"exchange"`
+	RoutingKey  string     `json:"routingKey"`
+	Properties  Properties `json:"properties"`
+	Body        []byte     `json:"body"`
 }
 
 func (msg *BasicDeliver) id() (uint16, uint16) {

tap/extensions/amqp/types.go

@@ -93,19 +93,19 @@ func (e Error) Error() string {
 
 // Used by header frames to capture routing and header information
 type Properties struct {
-	ContentType     string    // MIME content type
-	ContentEncoding string    // MIME content encoding
-	Headers         Table     // Application or header exchange table
-	DeliveryMode    uint8     // queue implementation use - Transient (1) or Persistent (2)
-	Priority        uint8     // queue implementation use - 0 to 9
-	CorrelationId   string    // application use - correlation identifier
-	ReplyTo         string    // application use - address to to reply to (ex: RPC)
-	Expiration      string    // implementation use - message expiration spec
-	MessageId       string    // application use - message identifier
-	Timestamp       time.Time // application use - message timestamp
-	Type            string    // application use - message type name
-	UserId          string    // application use - creating user id
-	AppId           string    // application use - creating application
+	ContentType     string    `json:"contentType"`     // MIME content type
+	ContentEncoding string    `json:"contentEncoding"` // MIME content encoding
+	Headers         Table     `json:"headers"`         // Application or header exchange table
+	DeliveryMode    uint8     `json:"deliveryMode"`    // queue implementation use - Transient (1) or Persistent (2)
+	Priority        uint8     `json:"priority"`        // queue implementation use - 0 to 9
+	CorrelationId   string    `json:"correlationId"`   // application use - correlation identifier
+	ReplyTo         string    `json:"replyTo"`         // application use - address to to reply to (ex: RPC)
+	Expiration      string    `json:"expiration"`      // implementation use - message expiration spec
+	MessageId       string    `json:"messageId"`       // application use - message identifier
+	Timestamp       time.Time `json:"timestamp"`       // application use - message timestamp
+	Type            string    `json:"type"`            // application use - message type name
+	UserId          string    `json:"userId"`          // application use - creating user id
+	AppId           string    `json:"appId"`           // application use - creating application
 	reserved1       string    // was cluster-id - process for buffer consumption
 }
 

tap/extensions/http/helpers.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) {
+	newMap = make(map[string]interface{})
+	for _, header := range mapSlice {
+		h := header.(map[string]interface{})
+		newMap[h["name"].(string)] = h["value"]
+	}
+
+	return
+}
+
+func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
+	var table []api.TableData
+	for _, header := range mapSlice {
+		h := header.(map[string]interface{})
+		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string))
+		table = append(table, api.TableData{
+			Name:     h["name"].(string),
+			Value:    h["value"],
+			Selector: selector,
+		})
+	}
+
+	obj, _ := json.Marshal(table)
+	representation = string(obj)
+	return
+}

tap/extensions/http/main.go

@@ -17,6 +17,7 @@ var protocol api.Protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
 	Abbreviation:    "HTTP",
+	Macro:           "http",
 	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
@@ -30,6 +31,7 @@ var http2Protocol api.Protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) (gRPC)",
 	Abbreviation:    "HTTP/2",
+	Macro:           "grpc",
 	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
@@ -56,7 +58,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", protocol.Name)
+	log.Printf("pong %s", protocol.Name)
 }
 
 func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
@@ -117,7 +119,7 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	var host, scheme, authority, path, service string
 
 	request := item.Pair.Request.Payload.(map[string]interface{})
@@ -141,14 +143,40 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 		}
 	}
 
+	if resDetails["bodySize"].(float64) < 0 {
+		resDetails["bodySize"] = 0
+	}
+
 	if item.Protocol.Version == "2.0" {
 		service = fmt.Sprintf("%s://%s", scheme, authority)
 	} else {
 		service = fmt.Sprintf("http://%s", host)
-		path = reqDetails["url"].(string)
+		u, err := url.Parse(reqDetails["url"].(string))
+		if err != nil {
+			path = reqDetails["url"].(string)
+		} else {
+			path = u.Path
+		}
 	}
 
-	request["url"] = path
+	request["url"] = reqDetails["url"].(string)
+	reqDetails["path"] = path
+	reqDetails["summary"] = path
+
+	// Rearrange the maps for the querying
+	reqDetails["_headers"] = reqDetails["headers"]
+	reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{}))
+	resDetails["_headers"] = resDetails["headers"]
+	resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{}))
+
+	reqDetails["_cookies"] = reqDetails["cookies"]
+	reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{}))
+	resDetails["_cookies"] = resDetails["cookies"]
+	resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{}))
+
+	reqDetails["_queryString"] = reqDetails["queryString"]
+	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryString"].([]interface{}))
+
 	if resolvedDestination != "" {
 		service = SetHostname(service, resolvedDestination)
 	} else if resolvedSource != "" {
@@ -156,51 +184,62 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	}
 
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	entryBytes, _ := json.Marshal(item.Pair)
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
+	httpPair, _ := json.Marshal(item.Pair)
+	_protocol := protocol
+	_protocol.Version = item.Protocol.Version
 	return &api.MizuEntry{
-		ProtocolName:            protocol.Name,
-		ProtocolLongName:        protocol.LongName,
-		ProtocolAbbreviation:    protocol.Abbreviation,
-		ProtocolVersion:         item.Protocol.Version,
-		ProtocolBackgroundColor: protocol.BackgroundColor,
-		ProtocolForegroundColor: protocol.ForegroundColor,
-		ProtocolFontSize:        protocol.FontSize,
-		ProtocolReferenceLink:   protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, path),
-		Method:                  reqDetails["method"].(string),
-		Status:                  int(resDetails["status"].(float64)),
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             elapsedTime,
-		Path:                    path,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: _protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Response:            resDetails,
+		Url:                 fmt.Sprintf("%s%s", service, path),
+		Method:              reqDetails["method"].(string),
+		Status:              int(resDetails["status"].(float64)),
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         elapsedTime,
+		Summary:             path,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
+		HTTPPair:            string(httpPair),
 	}
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	var p api.Protocol
-	if entry.ProtocolVersion == "2.0" {
+	if entry.Protocol.Version == "2.0" {
 		p = http2Protocol
 	} else {
 		p = protocol
 	}
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        p,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
 		Path:            entry.Path,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -218,45 +257,50 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 }
 
 func representRequest(request map[string]interface{}) (repRequest []interface{}) {
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Method",
-			"value": request["method"].(string),
+			Name:     "Method",
+			Value:    request["method"].(string),
+			Selector: `request.method`,
 		},
 		{
-			"name":  "URL",
-			"value": request["url"].(string),
+			Name:     "URL",
+			Value:    request["url"].(string),
+			Selector: `request.url`,
 		},
 		{
-			"name":  "Body Size",
-			"value": fmt.Sprintf("%g bytes", request["bodySize"].(float64)),
+			Name:     "Path",
+			Value:    request["path"].(string),
+			Selector: `request.path`,
+		},
+		{
+			Name:     "Body Size (bytes)",
+			Value:    int64(request["bodySize"].(float64)),
+			Selector: `request.bodySize`,
 		},
 	})
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	headers, _ := json.Marshal(request["headers"].([]interface{}))
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Headers",
-		"data":  string(headers),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Headers",
+		Data:  representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`),
 	})
 
-	cookies, _ := json.Marshal(request["cookies"].([]interface{}))
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Cookies",
-		"data":  string(cookies),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Cookies",
+		Data:  representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`),
 	})
 
-	queryString, _ := json.Marshal(request["queryString"].([]interface{}))
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Query String",
-		"data":  string(queryString),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Query String",
+		Data:  representMapSliceAsTable(request["_queryString"].([]interface{}), `request.queryString`),
 	})
 
 	postData, _ := request["postData"].(map[string]interface{})
@@ -266,12 +310,12 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 	}
 	text, _ := postData["text"]
 	if text != nil {
-		repRequest = append(repRequest, map[string]string{
-			"type":      api.BODY,
-			"title":     "POST Data (text/plain)",
-			"encoding":  "",
-			"mime_type": mimeType.(string),
-			"data":      text.(string),
+		repRequest = append(repRequest, api.SectionData{
+			Type:     api.BODY,
+			Title:    "POST Data (text/plain)",
+			MimeType: mimeType.(string),
+			Data:     text.(string),
+			Selector: `request.postData.text`,
 		})
 	}
 
@@ -285,16 +329,16 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 						"value": string(params),
 					},
 				})
-				repRequest = append(repRequest, map[string]string{
-					"type":  api.TABLE,
-					"title": "POST Data (multipart/form-data)",
-					"data":  string(multipart),
+				repRequest = append(repRequest, api.SectionData{
+					Type:  api.TABLE,
+					Title: "POST Data (multipart/form-data)",
+					Data:  string(multipart),
 				})
 			} else {
-				repRequest = append(repRequest, map[string]string{
-					"type":  api.TABLE,
-					"title": "POST Data (application/x-www-form-urlencoded)",
-					"data":  string(params),
+				repRequest = append(repRequest, api.SectionData{
+					Type:  api.TABLE,
+					Title: "POST Data (application/x-www-form-urlencoded)",
+					Data:  representMapSliceAsTable(postData["params"].([]interface{}), `request.postData.params`),
 				})
 			}
 		}
@@ -308,38 +352,39 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 
 	bodySize = int64(response["bodySize"].(float64))
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Status",
-			"value": fmt.Sprintf("%g", response["status"].(float64)),
+			Name:     "Status",
+			Value:    int64(response["status"].(float64)),
+			Selector: `response.status`,
 		},
 		{
-			"name":  "Status Text",
-			"value": response["statusText"].(string),
+			Name:     "Status Text",
+			Value:    response["statusText"].(string),
+			Selector: `response.statusText`,
 		},
 		{
-			"name":  "Body Size",
-			"value": fmt.Sprintf("%d bytes", bodySize),
+			Name:     "Body Size (bytes)",
+			Value:    bodySize,
+			Selector: `response.bodySize`,
 		},
 	})
-	repResponse = append(repResponse, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	repResponse = append(repResponse, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	headers, _ := json.Marshal(response["headers"].([]interface{}))
-	repResponse = append(repResponse, map[string]string{
-		"type":  api.TABLE,
-		"title": "Headers",
-		"data":  string(headers),
+	repResponse = append(repResponse, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Headers",
+		Data:  representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`),
 	})
 
-	cookies, _ := json.Marshal(response["cookies"].([]interface{}))
-	repResponse = append(repResponse, map[string]string{
-		"type":  api.TABLE,
-		"title": "Cookies",
-		"data":  string(cookies),
+	repResponse = append(repResponse, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Cookies",
+		Data:  representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`),
 	})
 
 	content, _ := response["content"].(map[string]interface{})
@@ -350,37 +395,40 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 	encoding, _ := content["encoding"]
 	text, _ := content["text"]
 	if text != nil {
-		repResponse = append(repResponse, map[string]string{
-			"type":      api.BODY,
-			"title":     "Body",
-			"encoding":  encoding.(string),
-			"mime_type": mimeType.(string),
-			"data":      text.(string),
+		repResponse = append(repResponse, api.SectionData{
+			Type:     api.BODY,
+			Title:    "Body",
+			Encoding: encoding.(string),
+			MimeType: mimeType.(string),
+			Data:     text.(string),
+			Selector: `response.content.text`,
 		})
 	}
 
 	return
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	if entry.ProtocolVersion == "2.0" {
-		p = http2Protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	if protoIn.Version == "2.0" {
+		protoOut = http2Protocol
 	} else {
-		p = protocol
+		protoOut = protocol
 	}
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
-	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
-	reqDetails := request["details"].(map[string]interface{})
-	resDetails := response["details"].(map[string]interface{})
-	repRequest := representRequest(reqDetails)
-	repResponse, bodySize := representResponse(resDetails)
+	repRequest := representRequest(request)
+	repResponse, bodySize := representResponse(response)
 	representation["request"] = repRequest
 	representation["response"] = repResponse
 	object, err = json.Marshal(representation)
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`http`:  fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, protocol.Version),
+		`grpc`:  fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version),
+		`http2`: fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version),
+	}
+}
+
 var Dissector dissecting

tap/extensions/http/sensitive_data_cleaner.go

@@ -16,6 +16,7 @@ import (
 )
 
 const maskedFieldPlaceholderValue = "[REDACTED]"
+const userAgent = "user-agent"
 
 //these values MUST be all lower case and contain no `-` or `_` characters
 var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
@@ -32,7 +33,7 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 	request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
 
 	for headerKey, headerValues := range request.Header {
-		if strings.ToLower(headerKey) == "user-agent" {
+		if strings.ToLower(headerKey) == userAgent {
 			for _, userAgent := range options.IgnoredUserAgents {
 				for _, headerValue := range headerValues {
 					if strings.Contains(strings.ToLower(headerValue), strings.ToLower(userAgent)) {
@@ -89,6 +90,10 @@ func filterResponseBody(response *http.Response, options *api.TrafficFilteringOp
 
 func filterHeaders(headers *http.Header) {
 	for key, _ := range *headers {
+		if strings.ToLower(key) == userAgent {
+			continue
+		}
+
 		if strings.ToLower(key) == "cookie" {
 			headers.Del(key)
 		} else if isFieldNameSensitive(key) {

tap/extensions/kafka/helpers.go

@@ -27,48 +27,54 @@ type KafkaWrapper struct {
 }
 
 func representRequestHeader(data map[string]interface{}, rep []interface{}) []interface{} {
-	requestHeader, _ := json.Marshal([]map[string]string{
+	requestHeader, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "ApiKey",
-			"value": apiNames[int(data["ApiKey"].(float64))],
+			Name:     "ApiKey",
+			Value:    apiNames[int(data["apiKey"].(float64))],
+			Selector: `request.apiKey`,
 		},
 		{
-			"name":  "ApiVersion",
-			"value": fmt.Sprintf("%d", int(data["ApiVersion"].(float64))),
+			Name:     "ApiVersion",
+			Value:    fmt.Sprintf("%d", int(data["apiVersion"].(float64))),
+			Selector: `request.apiVersion`,
 		},
 		{
-			"name":  "Client ID",
-			"value": data["ClientID"].(string),
+			Name:     "Client ID",
+			Value:    data["clientID"].(string),
+			Selector: `request.clientID`,
 		},
 		{
-			"name":  "Correlation ID",
-			"value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))),
+			Name:     "Correlation ID",
+			Value:    fmt.Sprintf("%d", int(data["correlationID"].(float64))),
+			Selector: `request.correlationID`,
 		},
 		{
-			"name":  "Size",
-			"value": fmt.Sprintf("%d", int(data["Size"].(float64))),
+			Name:     "Size",
+			Value:    fmt.Sprintf("%d", int(data["size"].(float64))),
+			Selector: `request.size`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Request Header",
-		"data":  string(requestHeader),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Request Header",
+		Data:  string(requestHeader),
 	})
 
 	return rep
 }
 
 func representResponseHeader(data map[string]interface{}, rep []interface{}) []interface{} {
-	requestHeader, _ := json.Marshal([]map[string]string{
+	requestHeader, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Correlation ID",
-			"value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))),
+			Name:     "Correlation ID",
+			Value:    fmt.Sprintf("%d", int(data["correlationID"].(float64))),
+			Selector: `response.correlationID`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Response Header",
-		"data":  string(requestHeader),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Response Header",
+		Data:  string(requestHeader),
 	})
 
 	return rep
@@ -79,46 +85,50 @@ func representMetadataRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
 	allowAutoTopicCreation := ""
 	includeClusterAuthorizedOperations := ""
 	includeTopicAuthorizedOperations := ""
-	if payload["Topics"] != nil {
-		x, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		x, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(x)
 	}
-	if payload["AllowAutoTopicCreation"] != nil {
-		allowAutoTopicCreation = strconv.FormatBool(payload["AllowAutoTopicCreation"].(bool))
+	if payload["allowAutoTopicCreation"] != nil {
+		allowAutoTopicCreation = strconv.FormatBool(payload["allowAutoTopicCreation"].(bool))
 	}
-	if payload["IncludeClusterAuthorizedOperations"] != nil {
-		includeClusterAuthorizedOperations = strconv.FormatBool(payload["IncludeClusterAuthorizedOperations"].(bool))
+	if payload["includeClusterAuthorizedOperations"] != nil {
+		includeClusterAuthorizedOperations = strconv.FormatBool(payload["includeClusterAuthorizedOperations"].(bool))
 	}
-	if payload["IncludeTopicAuthorizedOperations"] != nil {
-		includeTopicAuthorizedOperations = strconv.FormatBool(payload["IncludeTopicAuthorizedOperations"].(bool))
+	if payload["includeTopicAuthorizedOperations"] != nil {
+		includeTopicAuthorizedOperations = strconv.FormatBool(payload["includeTopicAuthorizedOperations"].(bool))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Allow Auto Topic Creation",
-			"value": allowAutoTopicCreation,
+			Name:     "Allow Auto Topic Creation",
+			Value:    allowAutoTopicCreation,
+			Selector: `request.payload.allowAutoTopicCreation`,
 		},
 		{
-			"name":  "Include Cluster Authorized Operations",
-			"value": includeClusterAuthorizedOperations,
+			Name:     "Include Cluster Authorized Operations",
+			Value:    includeClusterAuthorizedOperations,
+			Selector: `request.payload.includeClusterAuthorizedOperations`,
 		},
 		{
-			"name":  "Include Topic Authorized Operations",
-			"value": includeTopicAuthorizedOperations,
+			Name:     "Include Topic Authorized Operations",
+			Value:    includeTopicAuthorizedOperations,
+			Selector: `request.payload.includeTopicAuthorizedOperations`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -129,63 +139,69 @@ func representMetadataResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		_topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		_topics, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(_topics)
 	}
 	brokers := ""
-	if payload["Brokers"] != nil {
-		_brokers, _ := json.Marshal(payload["Brokers"].([]interface{}))
+	if payload["brokers"] != nil {
+		_brokers, _ := json.Marshal(payload["brokers"].([]interface{}))
 		brokers = string(_brokers)
 	}
 	controllerID := ""
 	clusterID := ""
 	throttleTimeMs := ""
 	clusterAuthorizedOperations := ""
-	if payload["ControllerID"] != nil {
-		controllerID = fmt.Sprintf("%d", int(payload["ControllerID"].(float64)))
+	if payload["controllerID"] != nil {
+		controllerID = fmt.Sprintf("%d", int(payload["controllerID"].(float64)))
 	}
-	if payload["ClusterID"] != nil {
-		clusterID = payload["ClusterID"].(string)
+	if payload["clusterID"] != nil {
+		clusterID = payload["clusterID"].(string)
 	}
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	if payload["ClusterAuthorizedOperations"] != nil {
-		clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["ClusterAuthorizedOperations"].(float64)))
+	if payload["clusterAuthorizedOperations"] != nil {
+		clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["clusterAuthorizedOperations"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Brokers",
-			"value": brokers,
+			Name:     "Brokers",
+			Value:    brokers,
+			Selector: `response.payload.brokers`,
 		},
 		{
-			"name":  "Cluster ID",
-			"value": clusterID,
+			Name:     "Cluster ID",
+			Value:    clusterID,
+			Selector: `response.payload.clusterID`,
 		},
 		{
-			"name":  "Controller ID",
-			"value": controllerID,
+			Name:     "Controller ID",
+			Value:    controllerID,
+			Selector: `response.payload.controllerID`,
 		},
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `response.payload.topics`,
 		},
 		{
-			"name":  "Cluster Authorized Operations",
-			"value": clusterAuthorizedOperations,
+			Name:     "Cluster Authorized Operations",
+			Value:    clusterAuthorizedOperations,
+			Selector: `response.payload.clusterAuthorizedOperations`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -196,29 +212,31 @@ func representApiVersionsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	clientSoftwareName := ""
 	clientSoftwareVersion := ""
-	if payload["ClientSoftwareName"] != nil {
-		clientSoftwareName = payload["ClientSoftwareName"].(string)
+	if payload["clientSoftwareName"] != nil {
+		clientSoftwareName = payload["clientSoftwareName"].(string)
 	}
-	if payload["ClientSoftwareVersion"] != nil {
-		clientSoftwareVersion = payload["ClientSoftwareVersion"].(string)
+	if payload["clientSoftwareVersion"] != nil {
+		clientSoftwareVersion = payload["clientSoftwareVersion"].(string)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Client Software Name",
-			"value": clientSoftwareName,
+			Name:     "Client Software Name",
+			Value:    clientSoftwareName,
+			Selector: `request.payload.clientSoftwareName`,
 		},
 		{
-			"name":  "Client Software Version",
-			"value": clientSoftwareVersion,
+			Name:     "Client Software Version",
+			Value:    clientSoftwareVersion,
+			Selector: `request.payload.clientSoftwareVersion`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -229,34 +247,37 @@ func representApiVersionsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	apiKeys := ""
-	if payload["TopicNames"] != nil {
-		x, _ := json.Marshal(payload["ApiKeys"].([]interface{}))
+	if payload["apiKeys"] != nil {
+		x, _ := json.Marshal(payload["apiKeys"].([]interface{}))
 		apiKeys = string(x)
 	}
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Error Code",
-			"value": fmt.Sprintf("%d", int(payload["ErrorCode"].(float64))),
+			Name:     "Error Code",
+			Value:    fmt.Sprintf("%d", int(payload["errorCode"].(float64))),
+			Selector: `response.payload.errorCode`,
 		},
 		{
-			"name":  "ApiKeys",
-			"value": apiKeys,
+			Name:     "ApiKeys",
+			Value:    apiKeys,
+			Selector: `response.payload.apiKeys`,
 		},
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -267,39 +288,43 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topicData := ""
-	_topicData := payload["TopicData"]
+	_topicData := payload["topicData"]
 	if _topicData != nil {
 		x, _ := json.Marshal(_topicData.([]interface{}))
 		topicData = string(x)
 	}
 	transactionalID := ""
-	if payload["TransactionalID"] != nil {
-		transactionalID = payload["TransactionalID"].(string)
+	if payload["transactionalID"] != nil {
+		transactionalID = payload["transactionalID"].(string)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Transactional ID",
-			"value": transactionalID,
+			Name:     "Transactional ID",
+			Value:    transactionalID,
+			Selector: `request.payload.transactionalID`,
 		},
 		{
-			"name":  "Required Acknowledgements",
-			"value": fmt.Sprintf("%d", int(payload["RequiredAcks"].(float64))),
+			Name:     "Required Acknowledgements",
+			Value:    fmt.Sprintf("%d", int(payload["requiredAcks"].(float64))),
+			Selector: `request.payload.requiredAcks`,
 		},
 		{
-			"name":  "Timeout",
-			"value": fmt.Sprintf("%d", int(payload["Timeout"].(float64))),
+			Name:     "Timeout",
+			Value:    fmt.Sprintf("%d", int(payload["timeout"].(float64))),
+			Selector: `request.payload.timeout`,
 		},
 		{
-			"name":  "Topic Data",
-			"value": topicData,
+			Name:     "Topic Data",
+			Value:    topicData,
+			Selector: `request.payload.topicData`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -310,30 +335,32 @@ func representProduceResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	responses := ""
-	if payload["Responses"] != nil {
-		_responses, _ := json.Marshal(payload["Responses"].([]interface{}))
+	if payload["responses"] != nil {
+		_responses, _ := json.Marshal(payload["responses"].([]interface{}))
 		responses = string(_responses)
 	}
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Responses",
-			"value": string(responses),
+			Name:     "Responses",
+			Value:    string(responses),
+			Selector: `response.payload.responses`,
 		},
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -344,87 +371,97 @@ func representFetchRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		_topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		_topics, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(_topics)
 	}
 	replicaId := ""
-	if payload["ReplicaId"] != nil {
-		replicaId = fmt.Sprintf("%d", int(payload["ReplicaId"].(float64)))
+	if payload["replicaId"] != nil {
+		replicaId = fmt.Sprintf("%d", int(payload["replicaId"].(float64)))
 	}
 	maxBytes := ""
-	if payload["MaxBytes"] != nil {
-		maxBytes = fmt.Sprintf("%d", int(payload["MaxBytes"].(float64)))
+	if payload["maxBytes"] != nil {
+		maxBytes = fmt.Sprintf("%d", int(payload["maxBytes"].(float64)))
 	}
 	isolationLevel := ""
-	if payload["IsolationLevel"] != nil {
-		isolationLevel = fmt.Sprintf("%d", int(payload["IsolationLevel"].(float64)))
+	if payload["isolationLevel"] != nil {
+		isolationLevel = fmt.Sprintf("%d", int(payload["isolationLevel"].(float64)))
 	}
 	sessionId := ""
-	if payload["SessionId"] != nil {
-		sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64)))
+	if payload["sessionId"] != nil {
+		sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64)))
 	}
 	sessionEpoch := ""
-	if payload["SessionEpoch"] != nil {
-		sessionEpoch = fmt.Sprintf("%d", int(payload["SessionEpoch"].(float64)))
+	if payload["sessionEpoch"] != nil {
+		sessionEpoch = fmt.Sprintf("%d", int(payload["sessionEpoch"].(float64)))
 	}
 	forgottenTopicsData := ""
-	if payload["ForgottenTopicsData"] != nil {
-		x, _ := json.Marshal(payload["ForgottenTopicsData"].(map[string]interface{}))
+	if payload["forgottenTopicsData"] != nil {
+		x, _ := json.Marshal(payload["forgottenTopicsData"].(map[string]interface{}))
 		forgottenTopicsData = string(x)
 	}
 	rackId := ""
-	if payload["RackId"] != nil {
-		rackId = payload["RackId"].(string)
+	if payload["rackId"] != nil {
+		rackId = payload["rackId"].(string)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Replica ID",
-			"value": replicaId,
+			Name:     "Replica ID",
+			Value:    replicaId,
+			Selector: `request.payload.replicaId`,
 		},
 		{
-			"name":  "Maximum Wait (ms)",
-			"value": fmt.Sprintf("%d", int(payload["MaxWaitMs"].(float64))),
+			Name:     "Maximum Wait (ms)",
+			Value:    fmt.Sprintf("%d", int(payload["maxWaitMs"].(float64))),
+			Selector: `request.payload.maxWaitMs`,
 		},
 		{
-			"name":  "Minimum Bytes",
-			"value": fmt.Sprintf("%d", int(payload["MinBytes"].(float64))),
+			Name:     "Minimum Bytes",
+			Value:    fmt.Sprintf("%d", int(payload["minBytes"].(float64))),
+			Selector: `request.payload.minBytes`,
 		},
 		{
-			"name":  "Maximum Bytes",
-			"value": maxBytes,
+			Name:     "Maximum Bytes",
+			Value:    maxBytes,
+			Selector: `request.payload.maxBytes`,
 		},
 		{
-			"name":  "Isolation Level",
-			"value": isolationLevel,
+			Name:     "Isolation Level",
+			Value:    isolationLevel,
+			Selector: `request.payload.isolationLevel`,
 		},
 		{
-			"name":  "Session ID",
-			"value": sessionId,
+			Name:     "Session ID",
+			Value:    sessionId,
+			Selector: `request.payload.sessionId`,
 		},
 		{
-			"name":  "Session Epoch",
-			"value": sessionEpoch,
+			Name:     "Session Epoch",
+			Value:    sessionEpoch,
+			Selector: `request.payload.sessionEpoch`,
 		},
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Forgotten Topics Data",
-			"value": forgottenTopicsData,
+			Name:     "Forgotten Topics Data",
+			Value:    forgottenTopicsData,
+			Selector: `request.payload.forgottenTopicsData`,
 		},
 		{
-			"name":  "Rack ID",
-			"value": rackId,
+			Name:     "Rack ID",
+			Value:    rackId,
+			Selector: `request.payload.rackId`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -435,46 +472,50 @@ func representFetchResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	responses := ""
-	if payload["Responses"] != nil {
-		_responses, _ := json.Marshal(payload["Responses"].([]interface{}))
+	if payload["responses"] != nil {
+		_responses, _ := json.Marshal(payload["responses"].([]interface{}))
 		responses = string(_responses)
 	}
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
 	errorCode := ""
-	if payload["ErrorCode"] != nil {
-		errorCode = fmt.Sprintf("%d", int(payload["ErrorCode"].(float64)))
+	if payload["errorCode"] != nil {
+		errorCode = fmt.Sprintf("%d", int(payload["errorCode"].(float64)))
 	}
 	sessionId := ""
-	if payload["SessionId"] != nil {
-		sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64)))
+	if payload["sessionId"] != nil {
+		sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Error Code",
-			"value": errorCode,
+			Name:     "Error Code",
+			Value:    errorCode,
+			Selector: `response.payload.errorCode`,
 		},
 		{
-			"name":  "Session ID",
-			"value": sessionId,
+			Name:     "Session ID",
+			Value:    sessionId,
+			Selector: `response.payload.sessionId`,
 		},
 		{
-			"name":  "Responses",
-			"value": responses,
+			Name:     "Responses",
+			Value:    responses,
+			Selector: `response.payload.responses`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -485,26 +526,28 @@ func representListOffsetsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		_topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		_topics, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(_topics)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Replica ID",
-			"value": fmt.Sprintf("%d", int(payload["ReplicaId"].(float64))),
+			Name:     "Replica ID",
+			Value:    fmt.Sprintf("%d", int(payload["replicaId"].(float64))),
+			Selector: `request.payload.replicaId`,
 		},
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `request.payload.topics`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -515,26 +558,28 @@ func representListOffsetsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `response.payload.topics`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -545,30 +590,33 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	validateOnly := ""
-	if payload["ValidateOnly"] != nil {
-		validateOnly = strconv.FormatBool(payload["ValidateOnly"].(bool))
+	if payload["validateOnly"] != nil {
+		validateOnly = strconv.FormatBool(payload["validateOnly"].(bool))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Timeout (ms)",
-			"value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))),
+			Name:     "Timeout (ms)",
+			Value:    fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))),
+			Selector: `request.payload.timeoutMs`,
 		},
 		{
-			"name":  "Validate Only",
-			"value": validateOnly,
+			Name:     "Validate Only",
+			Value:    validateOnly,
+			Selector: `request.payload.validateOnly`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -579,26 +627,28 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `response.payload.topics`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -609,35 +659,38 @@ func representDeleteTopicsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		x, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		x, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(x)
 	}
 	topicNames := ""
-	if payload["TopicNames"] != nil {
-		x, _ := json.Marshal(payload["TopicNames"].([]interface{}))
+	if payload["topicNames"] != nil {
+		x, _ := json.Marshal(payload["topicNames"].([]interface{}))
 		topicNames = string(x)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "TopicNames",
-			"value": string(topicNames),
+			Name:     "TopicNames",
+			Value:    string(topicNames),
+			Selector: `request.payload.topicNames`,
 		},
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Timeout (ms)",
-			"value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))),
+			Name:     "Timeout (ms)",
+			Value:    fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))),
+			Selector: `request.payload.timeoutMs`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -648,26 +701,28 @@ func representDeleteTopicsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	responses, _ := json.Marshal(payload["Responses"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	responses, _ := json.Marshal(payload["responses"].([]interface{}))
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Responses",
-			"value": string(responses),
+			Name:     "Responses",
+			Value:    string(responses),
+			Selector: `response.payload.responses`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep

tap/extensions/kafka/main.go

@@ -15,6 +15,7 @@ var _protocol api.Protocol = api.Protocol{
 	Name:            "kafka",
 	LongName:        "Apache Kafka Protocol",
 	Abbreviation:    "KAFKA",
+	Macro:           "kafka",
 	Version:         "12",
 	BackgroundColor: "#000000",
 	ForegroundColor: "#ffffff",
@@ -36,7 +37,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", _protocol.Name)
+	log.Printf("pong %s", _protocol.Name)
 }
 
 func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
@@ -61,7 +62,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 	service := "kafka"
@@ -70,76 +71,76 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	} else if resolvedSource != "" {
 		service = resolvedSource
 	}
-	apiKey := ApiKey(reqDetails["ApiKey"].(float64))
+	apiKey := ApiKey(reqDetails["apiKey"].(float64))
 
 	summary := ""
 	switch apiKey {
 	case Metadata:
-		_topics := reqDetails["Payload"].(map[string]interface{})["Topics"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case ApiVersions:
-		summary = reqDetails["ClientID"].(string)
+		summary = reqDetails["clientID"].(string)
 		break
 	case Produce:
-		_topics := reqDetails["Payload"].(map[string]interface{})["TopicData"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topicData"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case Fetch:
-		_topics := reqDetails["Payload"].(map[string]interface{})["Topics"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case ListOffsets:
-		_topics := reqDetails["Payload"].(map[string]interface{})["Topics"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case CreateTopics:
-		topics := reqDetails["Payload"].(map[string]interface{})["Topics"].([]interface{})
+		topics := reqDetails["payload"].(map[string]interface{})["topics"].([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case DeleteTopics:
-		topicNames := reqDetails["TopicNames"].([]string)
+		topicNames := reqDetails["topicNames"].([]string)
 		for _, name := range topicNames {
 			summary += fmt.Sprintf("%s, ", name)
 		}
@@ -148,44 +149,51 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 
 	request["url"] = summary
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	entryBytes, _ := json.Marshal(item.Pair)
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
 	return &api.MizuEntry{
-		ProtocolName:            _protocol.Name,
-		ProtocolLongName:        _protocol.LongName,
-		ProtocolAbbreviation:    _protocol.Abbreviation,
-		ProtocolVersion:         _protocol.Version,
-		ProtocolBackgroundColor: _protocol.BackgroundColor,
-		ProtocolForegroundColor: _protocol.ForegroundColor,
-		ProtocolFontSize:        _protocol.FontSize,
-		ProtocolReferenceLink:   _protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, summary),
-		Method:                  apiNames[apiKey],
-		Status:                  0,
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             elapsedTime,
-		Path:                    summary,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: _protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Response:            item.Pair.Response.Payload.(map[string]interface{})["details"].(map[string]interface{}),
+		Url:                 fmt.Sprintf("%s%s", service, summary),
+		Method:              apiNames[apiKey],
+		Status:              0,
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         elapsedTime,
+		Summary:             summary,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
 	}
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        _protocol,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -202,49 +210,43 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	}
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	p = _protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	protoOut = _protocol
 	bodySize = 0
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
-	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
-	reqDetails := request["details"].(map[string]interface{})
-	resDetails := response["details"].(map[string]interface{})
 
-	apiKey := ApiKey(reqDetails["ApiKey"].(float64))
+	apiKey := ApiKey(request["apiKey"].(float64))
 
 	var repRequest []interface{}
 	var repResponse []interface{}
 	switch apiKey {
 	case Metadata:
-		repRequest = representMetadataRequest(reqDetails)
-		repResponse = representMetadataResponse(resDetails)
+		repRequest = representMetadataRequest(request)
+		repResponse = representMetadataResponse(response)
 		break
 	case ApiVersions:
-		repRequest = representApiVersionsRequest(reqDetails)
-		repResponse = representApiVersionsResponse(resDetails)
+		repRequest = representApiVersionsRequest(request)
+		repResponse = representApiVersionsResponse(response)
 		break
 	case Produce:
-		repRequest = representProduceRequest(reqDetails)
-		repResponse = representProduceResponse(resDetails)
+		repRequest = representProduceRequest(request)
+		repResponse = representProduceResponse(response)
 		break
 	case Fetch:
-		repRequest = representFetchRequest(reqDetails)
-		repResponse = representFetchResponse(resDetails)
+		repRequest = representFetchRequest(request)
+		repResponse = representFetchResponse(response)
 		break
 	case ListOffsets:
-		repRequest = representListOffsetsRequest(reqDetails)
-		repResponse = representListOffsetsResponse(resDetails)
+		repRequest = representListOffsetsRequest(request)
+		repResponse = representListOffsetsResponse(response)
 		break
 	case CreateTopics:
-		repRequest = representCreateTopicsRequest(reqDetails)
-		repResponse = representCreateTopicsResponse(resDetails)
+		repRequest = representCreateTopicsRequest(request)
+		repResponse = representCreateTopicsResponse(response)
 		break
 	case DeleteTopics:
-		repRequest = representDeleteTopicsRequest(reqDetails)
-		repResponse = representDeleteTopicsResponse(resDetails)
+		repRequest = representDeleteTopicsRequest(request)
+		repResponse = representDeleteTopicsResponse(response)
 		break
 	}
 
@@ -254,4 +256,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`kafka`: fmt.Sprintf(`proto.abbr == "%s"`, _protocol.Abbreviation),
+	}
+}
+
 var Dissector dissecting

tap/extensions/kafka/request.go

@@ -10,13 +10,13 @@ import (
 )
 
 type Request struct {
-	Size          int32
-	ApiKey        ApiKey
-	ApiVersion    int16
-	CorrelationID int32
-	ClientID      string
-	Payload       interface{}
-	CaptureTime   time.Time
+	Size          int32       `json:"size"`
+	ApiKey        ApiKey      `json:"apiKey"`
+	ApiVersion    int16       `json:"apiVersion"`
+	CorrelationID int32       `json:"correlationID"`
+	ClientID      string      `json:"clientID"`
+	Payload       interface{} `json:"payload"`
+	CaptureTime   time.Time   `json:"captureTime"`
 }
 
 func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (apiKey ApiKey, apiVersion int16, err error) {

tap/extensions/kafka/response.go

@@ -10,10 +10,10 @@ import (
 )
 
 type Response struct {
-	Size          int32
-	CorrelationID int32
-	Payload       interface{}
-	CaptureTime   time.Time
+	Size          int32       `json:"size"`
+	CorrelationID int32       `json:"correlationID"`
+	Payload       interface{} `json:"payload"`
+	CaptureTime   time.Time   `json:"captureTime"`
 }
 
 func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter) (err error) {

tap/extensions/redis/helpers.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 
 	"github.com/up9inc/mizu/tap/api"
 )
@@ -24,33 +25,38 @@ type RedisWrapper struct {
 	Details interface{} `json:"details"`
 }
 
-func representGeneric(generic map[string]interface{}) (representation []interface{}) {
-	details, _ := json.Marshal([]map[string]string{
+func representGeneric(generic map[string]interface{}, selectorPrefix string) (representation []interface{}) {
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Type",
-			"value": generic["type"].(string),
+			Name:     "Type",
+			Value:    generic["type"].(string),
+			Selector: fmt.Sprintf("%stype", selectorPrefix),
 		},
 		{
-			"name":  "Command",
-			"value": generic["command"].(string),
+			Name:     "Command",
+			Value:    generic["command"].(string),
+			Selector: fmt.Sprintf("%scommand", selectorPrefix),
 		},
 		{
-			"name":  "Key",
-			"value": generic["key"].(string),
+			Name:     "Key",
+			Value:    generic["key"].(string),
+			Selector: fmt.Sprintf("%skey", selectorPrefix),
 		},
 		{
-			"name":  "Value",
-			"value": generic["value"].(string),
+			Name:     "Value",
+			Value:    generic["value"].(string),
+			Selector: fmt.Sprintf("%svalue", selectorPrefix),
 		},
 		{
-			"name":  "Keyword",
-			"value": generic["keyword"].(string),
+			Name:     "Keyword",
+			Value:    generic["keyword"].(string),
+			Selector: fmt.Sprintf("%skeyword", selectorPrefix),
 		},
 	})
-	representation = append(representation, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	representation = append(representation, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
 	return

tap/extensions/redis/main.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/up9inc/mizu/tap/api"
 )
@@ -13,6 +14,7 @@ var protocol api.Protocol = api.Protocol{
 	Name:            "redis",
 	LongName:        "Redis Serialization Protocol",
 	Abbreviation:    "REDIS",
+	Macro:           "redis",
 	Version:         "3.x",
 	BackgroundColor: "#a41e11",
 	ForegroundColor: "#ffffff",
@@ -34,7 +36,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", protocol.Name)
+	log.Printf("pong %s", protocol.Name)
 }
 
 func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
@@ -57,9 +59,12 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
+	response := item.Pair.Response.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
+	resDetails := response["details"].(map[string]interface{})
+
 	service := "redis"
 	if resolvedDestination != "" {
 		service = resolvedDestination
@@ -78,45 +83,53 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	}
 
 	request["url"] = summary
-	entryBytes, _ := json.Marshal(item.Pair)
+	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
 	return &api.MizuEntry{
-		ProtocolName:            protocol.Name,
-		ProtocolLongName:        protocol.LongName,
-		ProtocolAbbreviation:    protocol.Abbreviation,
-		ProtocolVersion:         protocol.Version,
-		ProtocolBackgroundColor: protocol.BackgroundColor,
-		ProtocolForegroundColor: protocol.ForegroundColor,
-		ProtocolFontSize:        protocol.FontSize,
-		ProtocolReferenceLink:   protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, summary),
-		Method:                  method,
-		Status:                  0,
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             0,
-		Path:                    summary,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Response:            resDetails,
+		Url:                 fmt.Sprintf("%s%s", service, summary),
+		Method:              method,
+		Status:              0,
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         elapsedTime,
+		Summary:             summary,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
 	}
 
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        protocol,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -133,22 +146,22 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	}
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	p = protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	protoOut = protocol
 	bodySize = 0
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
-	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
-	reqDetails := request["details"].(map[string]interface{})
-	resDetails := response["details"].(map[string]interface{})
-	repRequest := representGeneric(reqDetails)
-	repResponse := representGeneric(resDetails)
+	repRequest := representGeneric(request, `request.`)
+	repResponse := representGeneric(response, `response.`)
 	representation["request"] = repRequest
 	representation["response"] = repResponse
 	object, err = json.Marshal(representation)
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`redis`: fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation),
+	}
+}
+
 var Dissector dissecting

tap/extensions/redis/read.go

@@ -313,7 +313,7 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
 				packet.Value = fmt.Sprintf("%s]", packet.Value)
 			}
 		default:
-			msg := fmt.Sprintf("Unrecognized element in Redis array: %v\n", reflect.TypeOf(array[0]))
+			msg := fmt.Sprintf("Unrecognized element in Redis array: %v", reflect.TypeOf(array[0]))
 			err = errors.New(msg)
 			return
 		}
@@ -333,7 +333,7 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
 	case int64:
 		packet.Value = fmt.Sprintf("%d", x.(int64))
 	default:
-		msg := fmt.Sprintf("Unrecognized Redis data type: %v\n", reflect.TypeOf(x))
+		msg := fmt.Sprintf("Unrecognized Redis data type: %v", reflect.TypeOf(x))
 		err = errors.New(msg)
 		return
 	}

tap/go.mod

@@ -9,8 +9,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
-	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect
-	golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f
 )
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ./api

tap/go.sum

@@ -1,34 +1,686 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
+github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
 github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-errors/errors v1.4.1 h1:IvVlgbzSsaUNudsw5dcXSzF3EWyXTi5XrAdngnuhRyg=
 github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
+github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
+github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
+github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
+github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
+github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
+github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
+github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
+github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
+github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
+github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
+github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
+github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
+github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
+github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
+k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
+k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI=
+k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
+k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
+k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
+k8s.io/component-helpers v0.21.2/go.mod h1:DbyFt/A0p6Cv+R5+QOGSJ5f5t4xDfI8Yb89a57DgJlQ=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
+k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=
+k8s.io/metrics v0.21.2/go.mod h1:wzlOINZMCtWq8dR9gHlyaOemmYlOpAoldEIXE82gAhI=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY=
+sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIMgYrEDfwOeFdMK0=
+sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo=
+sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

tap/passive_tapper.go

@@ -40,6 +40,7 @@ var verbose = flag.Bool("verbose", false, "Be verbose")
 var debug = flag.Bool("debug", false, "Display debug information")
 var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
 var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex")
+var procfs = flag.String("procfs", "/proc", "The procfs directory, used when mapping host volumes into a container")
 
 // capture
 var iface = flag.String("i", "en0", "Interface to read packets from")
@@ -48,6 +49,7 @@ var snaplen = flag.Int("s", 65536, "Snap length (number of bytes max to read per
 var tstype = flag.String("timestamp_type", "", "Type of timestamps to use")
 var promisc = flag.Bool("promisc", true, "Set promiscuous mode")
 var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to keep connections which don't transmit data")
+var pids = flag.String("pids", "", "A comma separated list of PIDs to capture their network namespaces")
 
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
@@ -129,6 +131,24 @@ func printPeriodicStats(cleaner *Cleaner) {
 	}
 }
 
+func initializePacketSources() (*source.PacketSourceManager, error) {
+	var bpffilter string
+	if len(flag.Args()) > 0 {
+		bpffilter = strings.Join(flag.Args(), " ")
+	}
+
+	behaviour := source.TcpPacketSourceBehaviour{
+		SnapLength:  *snaplen,
+		Promisc:     *promisc,
+		Tstype:      *tstype,
+		DecoderName: *decoder,
+		Lazy:        *lazy,
+		BpfFilter:   bpffilter,
+	}
+
+	return source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, behaviour)
+}
+
 func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	streamsMap := NewTcpStreamMap()
 	go streamsMap.closeTimedoutTcpStreamChannels()
@@ -136,25 +156,13 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	diagnose.InitializeErrorsMap(*debug, *verbose, *quiet)
 	diagnose.InitializeTapperInternalStats()
 
-	var bpffilter string
-	if len(flag.Args()) > 0 {
-		bpffilter = strings.Join(flag.Args(), " ")
-	}
-
-	packetSource, err := source.NewTcpPacketSource(*fname, *iface, source.TcpPacketSourceBehaviour{
-		SnapLength:  *snaplen,
-		Promisc:     *promisc,
-		Tstype:      *tstype,
-		DecoderName: *decoder,
-		Lazy:        *lazy,
-		BpfFilter:   bpffilter,
-	})
+	sources, err := initializePacketSources()
 
 	if err != nil {
 		logger.Log.Fatal(err)
 	}
 
-	defer packetSource.Close()
+	defer sources.Close()
 
 	if err != nil {
 		logger.Log.Fatal(err)
@@ -163,10 +171,9 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	packets := make(chan source.TcpPacketInfo)
 	assembler := NewTcpAssembler(outputItems, streamsMap)
 
-	logger.Log.Info("Starting to read packets")
 	diagnose.AppStats.SetStartTime(time.Now())
 
-	go packetSource.ReadPackets(!*nodefrag, packets)
+	sources.ReadPackets(!*nodefrag, packets)
 
 	staleConnectionTimeout := time.Second * time.Duration(*staleTimeoutSeconds)
 	cleaner := Cleaner{
@@ -186,7 +193,7 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	}
 
 	if err := diagnose.DumpMemoryProfile(*memprofile); err != nil {
-		logger.Log.Errorf("Error dumping memory profile %v\n", err)
+		logger.Log.Errorf("Error dumping memory profile %v", err)
 	}
 
 	assembler.waitAndDump()

tap/source/packet_source_manager.go

@@ -0,0 +1,154 @@
+package source
+
+import (
+	"fmt"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/vishvananda/netns"
+)
+
+type PacketSourceManager struct {
+	sources []*tcpPacketSource
+}
+
+func NewPacketSourceManager(procfs string, pids string, filename string, interfaceName string,
+	behaviour TcpPacketSourceBehaviour) (*PacketSourceManager, error) {
+	sources := make([]*tcpPacketSource, 0)
+	hostSource, err := newHostPacketSource(filename, interfaceName, behaviour)
+
+	if err != nil {
+		return nil, err
+	}
+
+	sources = append(sources, hostSource)
+
+	if pids != "" {
+		netnsSources := newNetnsPacketSources(procfs, pids, interfaceName, behaviour)
+		sources = append(sources, netnsSources...)
+	}
+
+	return &PacketSourceManager{
+		sources: sources,
+	}, nil
+}
+
+func newHostPacketSource(filename string, interfaceName string,
+	behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) {
+	var name string
+
+	if filename == "" {
+		name = fmt.Sprintf("host-%v", interfaceName)
+	} else {
+		name = fmt.Sprintf("file-%v", filename)
+	}
+
+	source, err := newTcpPacketSource(name, filename, interfaceName, behaviour)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return source, nil
+}
+
+func newNetnsPacketSources(procfs string, pids string, interfaceName string,
+	behaviour TcpPacketSourceBehaviour) []*tcpPacketSource {
+	result := make([]*tcpPacketSource, 0)
+
+	for _, pidstr := range strings.Split(pids, ",") {
+		pid, err := strconv.Atoi(pidstr)
+
+		if err != nil {
+			logger.Log.Errorf("Invalid PID: %v - %v", pid, err)
+			continue
+		}
+
+		nsh, err := netns.GetFromPath(fmt.Sprintf("%v/%v/ns/net", procfs, pid))
+
+		if err != nil {
+			logger.Log.Errorf("Unable to get netns of pid %v - %v", pid, err)
+			continue
+		}
+
+		src, err := newNetnsPacketSource(pid, nsh, interfaceName, behaviour)
+
+		if err != nil {
+			logger.Log.Errorf("Error starting netns packet source for %v - %v", pid, err)
+			continue
+		}
+
+		result = append(result, src)
+	}
+
+	return result
+}
+
+func newNetnsPacketSource(pid int, nsh netns.NsHandle, interfaceName string,
+	behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) {
+
+	done := make(chan *tcpPacketSource)
+	errors := make(chan error)
+
+	go func(done chan<- *tcpPacketSource) {
+		// Setting a netns should be done from a dedicated OS thread.
+		//
+		// goroutines are not really OS threads, we try to mimic the issue by
+		//	locking the OS thread to this goroutine
+		//
+		runtime.LockOSThread()
+		defer runtime.UnlockOSThread()
+		
+		oldnetns, err := netns.Get()
+		
+		if err != nil {
+			logger.Log.Errorf("Unable to get netns of current thread %v", err)
+			errors <- err
+			return
+		}
+
+		if err := netns.Set(nsh); err != nil {
+			logger.Log.Errorf("Unable to set netns of pid %v - %v", pid, err)
+			errors <- err
+			return
+		}
+
+		name := fmt.Sprintf("netns-%v-%v", pid, interfaceName)
+		src, err := newTcpPacketSource(name, "", interfaceName, behaviour)
+
+		if err != nil {
+			logger.Log.Errorf("Error listening to PID %v - %v", pid, err)
+			errors <- err
+			return
+		}
+
+		if err := netns.Set(oldnetns); err != nil {
+			logger.Log.Errorf("Unable to set back netns of current thread %v", err)
+			errors <- err
+			return
+		}
+
+		done <- src
+	}(done)
+
+	select {
+	case err := <-errors:
+		return nil, err
+	case source := <-done:
+		return source, nil
+	}
+}
+
+func (m *PacketSourceManager) ReadPackets(ipdefrag bool, packets chan<- TcpPacketInfo) {
+	for _, src := range m.sources {
+		go src.readPackets(ipdefrag, packets)
+	}
+}
+
+func (m *PacketSourceManager) Close() {
+	for _, src := range m.sources {
+		src.close()
+	}
+}

tap/source/tcp_packet_source.go

@@ -13,11 +13,12 @@ import (
 	"github.com/up9inc/mizu/tap/diagnose"
 )
 
-type TcpPacketSource struct {
+type tcpPacketSource struct {
 	source    *gopacket.PacketSource
 	handle    *pcap.Handle
 	defragger *ip4defrag.IPv4Defragmenter
 	Behaviour *TcpPacketSourceBehaviour
+	name      string
 }
 
 type TcpPacketSourceBehaviour struct {
@@ -31,14 +32,15 @@ type TcpPacketSourceBehaviour struct {
 
 type TcpPacketInfo struct {
 	Packet gopacket.Packet
-	Source *TcpPacketSource
+	Source *tcpPacketSource
 }
 
-func NewTcpPacketSource(filename string, interfaceName string,
-	behaviour TcpPacketSourceBehaviour) (*TcpPacketSource, error) {
+func newTcpPacketSource(name, filename string, interfaceName string,
+	behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) {
 	var err error
 
-	result := &TcpPacketSource{
+	result := &tcpPacketSource{
+		name:      name,
 		defragger: ip4defrag.NewIPv4Defragmenter(),
 		Behaviour: &behaviour,
 	}
@@ -96,21 +98,24 @@ func NewTcpPacketSource(filename string, interfaceName string,
 	return result, nil
 }
 
-func (source *TcpPacketSource) Close() {
+func (source *tcpPacketSource) close() {
 	if source.handle != nil {
 		source.handle.Close()
 	}
 }
 
-func (source *TcpPacketSource) ReadPackets(ipdefrag bool, packets chan<- TcpPacketInfo) error {
+func (source *tcpPacketSource) readPackets(ipdefrag bool, packets chan<- TcpPacketInfo) {
+	logger.Log.Infof("Start reading packets from %v", source.name)
+
 	for {
 		packet, err := source.source.NextPacket()
 
 		if err == io.EOF {
-			return err
+			logger.Log.Infof("Got EOF while reading packets from %v", source.name)
+			return
 		} else if err != nil {
 			if err.Error() != "Timeout Expired" {
-				logger.Log.Debugf("Error: %T", err)
+				logger.Log.Debugf("Error while reading from %v - %v", source.name, err)
 			}
 			continue
 		}

tap/tcp_assembler.go

@@ -78,7 +78,7 @@ func (a *tcpAssembler) processPackets(dumpPacket bool, packets <-chan source.Tcp
 			if *checksum {
 				err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
 				if err != nil {
-					logger.Log.Fatalf("Failed to set network layer for checksum: %s\n", err)
+					logger.Log.Fatalf("Failed to set network layer for checksum: %s", err)
 				}
 			}
 			c := context{

tap/tcp_reader.go

@@ -66,7 +66,7 @@ func (h *tcpReader) Read(p []byte) (int, error) {
 			clientHello := tlsx.ClientHello{}
 			err := clientHello.Unmarshall(msg.bytes)
 			if err == nil {
-				logger.Log.Debugf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
+				logger.Log.Debugf("Detected TLS client hello with SNI %s", clientHello.SNI)
 				// TODO: Throws `panic: runtime error: invalid memory address or nil pointer dereference` error.
 				// numericPort, _ := strconv.Atoi(h.tcpID.DstPort)
 				// h.outboundLinkWriter.WriteOutboundLink(h.tcpID.SrcIP, h.tcpID.DstIP, numericPort, clientHello.SNI, TLSProtocol)

tap/tcp_streams_map.go

@@ -46,7 +46,7 @@ func (streamMap *tcpStreamMap) closeTimedoutTcpStreamChannels() {
 				if !stream.isClosed && time.Now().After(streamWrapper.createdAt.Add(tcpStreamChannelTimeout)) {
 					stream.Close()
 					diagnose.AppStats.IncDroppedTcpStreams()
-					logger.Log.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d\n",
+					logger.Log.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d",
 						diagnose.AppStats.DroppedTcpStreams, runtime.NumGoroutine(), tcpStreamChannelTimeout/1000000)
 				}
 			} else {

tap/tester/tester.go

@@ -33,7 +33,7 @@ func loadExtensions() ([]*tapApi.Extension, error) {
 			continue
 		}
 
-		logger.Log.Infof("Loading extension: %s\n", filename)
+		logger.Log.Infof("Loading extension: %s", filename)
 
 		extension := &tapApi.Extension{
 			Path: path.Join(extensionsDir, filename),
@@ -55,7 +55,7 @@ func loadExtensions() ([]*tapApi.Extension, error) {
 		dissector, ok := symDissector.(tapApi.Dissector)
 
 		if !ok {
-			return nil, errors.Errorf("Symbol Dissector type error: %v %T\n", file, symDissector)
+			return nil, errors.Errorf("Symbol Dissector type error: %v %T", file, symDissector)
 		}
 
 		dissector.Register(extension)
@@ -68,7 +68,7 @@ func loadExtensions() ([]*tapApi.Extension, error) {
 	})
 
 	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v\n", extension)
+		logger.Log.Infof("Extension Properties: %+v", extension)
 	}
 
 	return extensions, nil
@@ -92,7 +92,7 @@ func internalRun() error {
 
 	tap.StartPassiveTapper(&opts, outputItems, extenssions, &tapOpts)
 
-	logger.Log.Infof("Tapping, press enter to exit...\n")
+	logger.Log.Infof("Tapping, press enter to exit...")
 	reader := bufio.NewReader(os.Stdin)
 	reader.ReadLine()
 	return nil
@@ -104,9 +104,9 @@ func main() {
 	if err != nil {
 		switch err := err.(type) {
 		case *errors.Error:
-			logger.Log.Errorf("Error: %v\n", err.ErrorStack())
+			logger.Log.Errorf("Error: %v", err.ErrorStack())
 		default:
-			logger.Log.Errorf("Error: %v\n", err)
+			logger.Log.Errorf("Error: %v", err)
 		}
 
 		os.Exit(1)

ui/.snyk

@@ -133,3 +133,6 @@ ignore:
   SNYK-JS-WS-1296835:
     - '*':
         reason: Non given
+  SNYK-JS-JSONSCHEMA-1920922:
+    - '*':
+        reason: Non given

ui/package-lock.json

@@ -1769,6 +1769,33 @@
         }
       }
     },
+    "@mapbox/rehype-prism": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/@mapbox/rehype-prism/-/rehype-prism-0.7.0.tgz",
+      "integrity": "sha512-zSG46selA6v+3THhCatTyOt9DuTzxTIVTxTbcj15kFmxPDtjzZ5VoFVCLZfjWFouYa9PiXxcbMLLhJoVzCxh9w==",
+      "requires": {
+        "hast-util-to-string": "^1.0.4",
+        "refractor": "^3.4.0",
+        "unist-util-visit": "^2.0.3"
+      },
+      "dependencies": {
+        "prismjs": {
+          "version": "1.24.1",
+          "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.24.1.tgz",
+          "integrity": "sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow=="
+        },
+        "refractor": {
+          "version": "3.4.0",
+          "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.4.0.tgz",
+          "integrity": "sha512-dBeD02lC5eytm9Gld2Mx0cMcnR+zhSnsTfPpWqFaMgUMJfC9A6bcN3Br/NaXrnBJcuxnLFR90k1jrkaSyV8umg==",
+          "requires": {
+            "hastscript": "^6.0.0",
+            "parse-entities": "^2.0.0",
+            "prismjs": "~1.24.0"
+          }
+        }
+      }
+    },
     "@material-ui/core": {
       "version": "4.12.2",
       "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.2.tgz",
@@ -1788,6 +1815,14 @@
         "react-transition-group": "^4.4.0"
       }
     },
+    "@material-ui/icons": {
+      "version": "4.11.2",
+      "resolved": "https://registry.npmjs.org/@material-ui/icons/-/icons-4.11.2.tgz",
+      "integrity": "sha512-fQNsKX2TxBmqIGJCSi3tGTO/gZ+eJgWmMJkgDiOfyNaunNaxcklJQFaFogYcFl0qFuaEz1qaXYXboa/bUXVSOQ==",
+      "requires": {
+        "@babel/runtime": "^7.4.4"
+      }
+    },
     "@material-ui/lab": {
       "version": "4.0.0-alpha.60",
       "resolved": "https://registry.npmjs.org/@material-ui/lab/-/lab-4.0.0-alpha.60.tgz",
@@ -2406,6 +2441,11 @@
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
       "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA=="
     },
+    "@types/parse5": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@types/parse5/-/parse5-6.0.1.tgz",
+      "integrity": "sha512-ARATsLdrGPUnaBvxLhUlnltcMgn7pQG312S8ccdYlnyijabrX9RN/KN/iGj9Am96CoW8e/K9628BA7Bv4XHdrA=="
+    },
     "@types/prettier": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.2.3.tgz",
@@ -2629,6 +2669,26 @@
         "eslint-visitor-keys": "^2.0.0"
       }
     },
+    "@uiw/react-textarea-code-editor": {
+      "version": "1.4.12",
+      "resolved": "https://registry.npmjs.org/@uiw/react-textarea-code-editor/-/react-textarea-code-editor-1.4.12.tgz",
+      "integrity": "sha512-op0aIRxX8hLi8OLwm/23dQ2X4o2xHUoK2pLr1DWBFgNbIh4L+RM5zByNFjV8mHfi/NvE6mQLB6LtMd1qaor5MQ==",
+      "requires": {
+        "@babel/runtime": "7.15.4",
+        "@mapbox/rehype-prism": "0.7.0",
+        "rehype": "12.0.0"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.15.4",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.15.4.tgz",
+          "integrity": "sha512-99catp6bHCaxr4sJ/DbTGgHS4+Rs2RVd2g7iOap6SLGPDknRK9ztKNsE/Fg6QhSeh1FGE5f6gHGQmvvn3I3xhw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
+      }
+    },
     "@webassemblyjs/ast": {
       "version": "1.9.0",
       "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz",
@@ -3689,6 +3749,11 @@
       "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz",
       "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ=="
     },
+    "bail": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.1.tgz",
+      "integrity": "sha512-d5FoTAr2S5DSUPKl85WNm2yUwsINN8eidIdIwsOge2t33DaOfOdSmmsI11jMN3GmALCXaw+Y6HMVHDzePshFAA=="
+    },
     "balanced-match": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -4179,6 +4244,11 @@
       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
+    "ccount": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.0.tgz",
+      "integrity": "sha512-VOR0NWFYX65n9gELQdcpqsie5L5ihBXuZGAgaPEp/U7IOSjnPMEH6geE+2f6lcekaNEfWzAHS45mPvSo5bqsUA=="
+    },
     "chalk": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -4199,6 +4269,11 @@
       "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz",
       "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw=="
     },
+    "character-entities-html4": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.0.0.tgz",
+      "integrity": "sha512-dwT2xh5ZhUAjyP96k57ilMKoTQyASaw9IAMR9U5c1lCu2RUni6O6jxfpUEdO2RcPT6TJFvr8pqsbami4Jk+2oA=="
+    },
     "character-entities-legacy": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz",
@@ -7534,11 +7609,121 @@
         "minimalistic-assert": "^1.0.1"
       }
     },
+    "hast-util-from-parse5": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-7.1.0.tgz",
+      "integrity": "sha512-m8yhANIAccpU4K6+121KpPP55sSl9/samzQSQGpb0mTExcNh2WlvjtMwSWFhg6uqD4Rr6Nfa8N6TMypQM51rzQ==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "@types/parse5": "^6.0.0",
+        "@types/unist": "^2.0.0",
+        "hastscript": "^7.0.0",
+        "property-information": "^6.0.0",
+        "vfile": "^5.0.0",
+        "vfile-location": "^4.0.0",
+        "web-namespaces": "^2.0.0"
+      },
+      "dependencies": {
+        "comma-separated-tokens": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz",
+          "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg=="
+        },
+        "hast-util-parse-selector": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.0.tgz",
+          "integrity": "sha512-AyjlI2pTAZEOeu7GeBPZhROx0RHBnydkQIXlhnFzDi0qfXTmGUWoCYZtomHbrdrheV4VFUlPcfJ6LMF5T6sQzg==",
+          "requires": {
+            "@types/hast": "^2.0.0"
+          }
+        },
+        "hastscript": {
+          "version": "7.0.2",
+          "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-7.0.2.tgz",
+          "integrity": "sha512-uA8ooUY4ipaBvKcMuPehTAB/YfFLSSzCwFSwT6ltJbocFUKH/GDHLN+tflq7lSRf9H86uOuxOFkh1KgIy3Gg2g==",
+          "requires": {
+            "@types/hast": "^2.0.0",
+            "comma-separated-tokens": "^2.0.0",
+            "hast-util-parse-selector": "^3.0.0",
+            "property-information": "^6.0.0",
+            "space-separated-tokens": "^2.0.0"
+          }
+        },
+        "property-information": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz",
+          "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg=="
+        },
+        "space-separated-tokens": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz",
+          "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw=="
+        }
+      }
+    },
+    "hast-util-is-element": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-2.1.1.tgz",
+      "integrity": "sha512-ag0fiZfRWsPiR1udvnSbaazJLGv8qd8E+/e3rW8rUZhbKG4HNJmFL4QkEceN+22BgE+uozXY30z/s+2dL6Z++g==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "@types/unist": "^2.0.0"
+      }
+    },
     "hast-util-parse-selector": {
       "version": "2.2.5",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz",
       "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ=="
     },
+    "hast-util-to-html": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-8.0.2.tgz",
+      "integrity": "sha512-ipLhUTMyyJi9F/LXaNDG9BrRdshP6obCfmUZYbE/+T639IdzqAOkKN4DyrEyID0gbb+rsC3PKf0XlviZwzomhw==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "ccount": "^2.0.0",
+        "comma-separated-tokens": "^2.0.0",
+        "hast-util-is-element": "^2.0.0",
+        "hast-util-whitespace": "^2.0.0",
+        "html-void-elements": "^2.0.0",
+        "property-information": "^6.0.0",
+        "space-separated-tokens": "^2.0.0",
+        "stringify-entities": "^4.0.0",
+        "unist-util-is": "^5.0.0"
+      },
+      "dependencies": {
+        "comma-separated-tokens": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz",
+          "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg=="
+        },
+        "property-information": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz",
+          "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg=="
+        },
+        "space-separated-tokens": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz",
+          "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw=="
+        },
+        "unist-util-is": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.1.1.tgz",
+          "integrity": "sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ=="
+        }
+      }
+    },
+    "hast-util-to-string": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-1.0.4.tgz",
+      "integrity": "sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w=="
+    },
+    "hast-util-whitespace": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.0.tgz",
+      "integrity": "sha512-Pkw+xBHuV6xFeJprJe2BBEoDV+AvQySaz3pPDRUs5PNZEMQjpXJJueqrpcHIXxnWTcAGi/UOCgVShlkY6kLoqg=="
+    },
     "hastscript": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz",
@@ -7671,6 +7856,11 @@
         "terser": "^4.6.3"
       }
     },
+    "html-void-elements": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-2.0.0.tgz",
+      "integrity": "sha512-4OYzQQsBt0G9bJ/nM9/DDsjm4+fVdzAaPJJcWk5QwA3GIAPxQEeOR0rsI8HbDHQz5Gta8pVvGnnTNSbZVEVvkQ=="
+    },
     "html-webpack-plugin": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-4.5.0.tgz",
@@ -13454,9 +13644,9 @@
       }
     },
     "react-scrollable-feed-virtualized": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.3.tgz",
-      "integrity": "sha512-M9WgJKr57jCyWKNCksc3oi+xhtO0YbL9d7Ll8Sdc5ZWOIstNvdNbNX0k4Nq6kXUVaHCJ9qE8omdSI/CxT3MLAQ=="
+      "version": "1.4.8",
+      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.8.tgz",
+      "integrity": "sha512-zsSO/9QB+4V6HEk39lxeMEUA6JFSZjfV4stw7RF17+vZdlVhyATsTBCzsj8hZywY4F29cBfH+3/GKrMhwmhAsw=="
     },
     "react-syntax-highlighter": {
       "version": "15.4.3",
@@ -13470,6 +13660,14 @@
         "refractor": "^3.2.0"
       }
     },
+    "react-toastify": {
+      "version": "8.0.3",
+      "resolved": "https://registry.npmjs.org/react-toastify/-/react-toastify-8.0.3.tgz",
+      "integrity": "sha512-rv3koC7f9lKKSkdpYgo/TGzgWlrB/aaiUInF1DyV7BpiM4kyTs+uhu6/r8XDMtBY2FOIHK+FlK3Iv7OzpA/tCA==",
+      "requires": {
+        "clsx": "^1.1.1"
+      }
+    },
     "react-transition-group": {
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz",
@@ -13693,6 +13891,38 @@
         }
       }
     },
+    "rehype": {
+      "version": "12.0.0",
+      "resolved": "https://registry.npmjs.org/rehype/-/rehype-12.0.0.tgz",
+      "integrity": "sha512-gZcttmf9R5IYHb8AlI1rlmWqXS1yX0rSB/S5ZGJs8atfYZy2DobvH3Ic/gSzB+HL/+oOHPtBguw1TprfhxXBgQ==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "rehype-parse": "^8.0.0",
+        "rehype-stringify": "^9.0.0",
+        "unified": "^10.0.0"
+      }
+    },
+    "rehype-parse": {
+      "version": "8.0.3",
+      "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-8.0.3.tgz",
+      "integrity": "sha512-RGw0CVt+0S6KdvpE8bbP2Db9WXclQcIX7A0ufM3QFqAhTo/ddJMQrrI2j3cijlRPZlGK8R3pRgC8U5HyV76IDw==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "hast-util-from-parse5": "^7.0.0",
+        "parse5": "^6.0.0",
+        "unified": "^10.0.0"
+      }
+    },
+    "rehype-stringify": {
+      "version": "9.0.2",
+      "resolved": "https://registry.npmjs.org/rehype-stringify/-/rehype-stringify-9.0.2.tgz",
+      "integrity": "sha512-BuVA6lAEYtOpXO2xuHLohAzz8UNoQAxAqYRqh4QEEtU39Co+P1JBZhw6wXA9hMWp+JLcmrxWH8+UKcNSr443Fw==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "hast-util-to-html": "^8.0.0",
+        "unified": "^10.0.0"
+      }
+    },
     "relateurl": {
       "version": "0.2.7",
       "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz",
@@ -15342,6 +15572,22 @@
         }
       }
     },
+    "stringify-entities": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.1.tgz",
+      "integrity": "sha512-gmMQxKXPWIO3NXNSPyWNhlYcBNGpPA/487D+9dLPnU4xBnIrnHdr8cv5rGJOS/1BRxEXRb7uKwg7BA36IWV7xg==",
+      "requires": {
+        "character-entities-html4": "^2.0.0",
+        "character-entities-legacy": "^2.0.0"
+      },
+      "dependencies": {
+        "character-entities-legacy": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-2.0.0.tgz",
+          "integrity": "sha512-YwaEtEvWLpFa6Wh3uVLrvirA/ahr9fki/NUd/Bd4OR6EdJ8D22hovYQEOUCBfQfcqnC4IAMGMsHXY1eXgL4ZZA=="
+        }
+      }
+    },
     "stringify-object": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz",
@@ -15877,6 +16123,11 @@
       "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
       "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM="
     },
+    "trough": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/trough/-/trough-2.0.2.tgz",
+      "integrity": "sha512-FnHq5sTMxC0sk957wHDzRnemFnNBvt/gSY99HzK8F7UP5WAbvP70yX5bd7CjEQkN+TjdxwI7g7lJ6podqrG2/w=="
+    },
     "true-case-path": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz",
@@ -16037,6 +16288,32 @@
       "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz",
       "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg=="
     },
+    "unified": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.0.tgz",
+      "integrity": "sha512-4U3ru/BRXYYhKbwXV6lU6bufLikoAavTwev89H5UxY8enDFaAT2VXmIXYNm6hb5oHPng/EXr77PVyDFcptbk5g==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "bail": "^2.0.0",
+        "extend": "^3.0.0",
+        "is-buffer": "^2.0.0",
+        "is-plain-obj": "^4.0.0",
+        "trough": "^2.0.0",
+        "vfile": "^5.0.0"
+      },
+      "dependencies": {
+        "is-buffer": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
+          "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ=="
+        },
+        "is-plain-obj": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.0.0.tgz",
+          "integrity": "sha512-NXRbBtUdBioI73y/HmOhogw/U5msYPC9DAtGkJXeFcFWSFZw0mCUsPxk/snTuJHzNKA8kLBK4rH97RMB1BfCXw=="
+        }
+      }
+    },
     "union-value": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
@@ -16082,6 +16359,38 @@
         "crypto-random-string": "^1.0.0"
       }
     },
+    "unist-util-is": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz",
+      "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg=="
+    },
+    "unist-util-stringify-position": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.0.tgz",
+      "integrity": "sha512-SdfAl8fsDclywZpfMDTVDxA2V7LjtRDTOFd44wUJamgl6OlVngsqWjxvermMYf60elWHbxhuRCZml7AnuXCaSA==",
+      "requires": {
+        "@types/unist": "^2.0.0"
+      }
+    },
+    "unist-util-visit": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz",
+      "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "unist-util-is": "^4.0.0",
+        "unist-util-visit-parents": "^3.0.0"
+      }
+    },
+    "unist-util-visit-parents": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz",
+      "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "unist-util-is": "^4.0.0"
+      }
+    },
     "universalify": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
@@ -16306,6 +16615,42 @@
         "extsprintf": "^1.2.0"
       }
     },
+    "vfile": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.1.0.tgz",
+      "integrity": "sha512-4o7/DJjEaFPYSh0ckv5kcYkJTHQgCKdL8ozMM1jLAxO9ox95IzveDPXCZp08HamdWq8JXTkClDvfAKaeLQeKtg==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "is-buffer": "^2.0.0",
+        "unist-util-stringify-position": "^3.0.0",
+        "vfile-message": "^3.0.0"
+      },
+      "dependencies": {
+        "is-buffer": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
+          "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ=="
+        }
+      }
+    },
+    "vfile-location": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-4.0.1.tgz",
+      "integrity": "sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "vfile": "^5.0.0"
+      }
+    },
+    "vfile-message": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-3.0.2.tgz",
+      "integrity": "sha512-UUjZYIOg9lDRwwiBAuezLIsu9KlXntdxwG+nXnjuQAHvBpcX3x0eN8h+I7TkY5nkCXj+cWVp4ZqebtGBvok8ww==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "unist-util-stringify-position": "^3.0.0"
+      }
+    },
     "vm-browserify": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz",
@@ -16590,6 +16935,11 @@
         "minimalistic-assert": "^1.0.0"
       }
     },
+    "web-namespaces": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.0.tgz",
+      "integrity": "sha512-dE7ELZRVWh0ceQsRgkjLgsAvwTuv3kcjSY/hLjqL0llleUlQBDjE9JkB9FCBY5F2mnFEwiyJoowl8+NVGHe8dw=="
+    },
     "web-vitals": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-1.1.1.tgz",

ui/package.json

@@ -4,6 +4,7 @@
   "private": true,
   "dependencies": {
     "@material-ui/core": "^4.11.3",
+    "@material-ui/icons": "^4.11.2",
     "@material-ui/lab": "^4.0.0-alpha.60",
     "@testing-library/jest-dom": "^5.11.10",
     "@testing-library/react": "^11.2.6",
@@ -12,6 +13,7 @@
     "@types/node": "^12.20.10",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.3",
+    "@uiw/react-textarea-code-editor": "^1.4.12",
     "axios": "^0.21.1",
     "jsonpath": "^1.1.1",
     "node-sass": "^5.0.0",
@@ -21,8 +23,9 @@
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
-    "react-scrollable-feed-virtualized": "^1.4.3",
+    "react-scrollable-feed-virtualized": "^1.4.8",
     "react-syntax-highlighter": "^15.4.3",
+    "react-toastify": "^8.0.3",
     "typescript": "^4.2.4",
     "web-vitals": "^1.1.1"
   },

ui/src/components/EntriesList.tsx

@@ -1,138 +1,44 @@
-import {EntryItem} from "./EntryListItem/EntryListItem";
-import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
+import React, {useRef} from "react";
 import styles from './style/EntriesList.module.sass';
-import spinner from './assets/spinner.svg';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
-import {StatusType} from "./Filters";
-import Api from "../helpers/api";
 import down from "./assets/downImg.svg";
 
 interface EntriesListProps {
     entries: any[];
-    setEntries: (entries: any[]) => void;
-    focusedEntryId: string;
-    setFocusedEntryId: (id: string) => void;
-    connectionOpen: boolean;
-    noMoreDataTop: boolean;
-    setNoMoreDataTop: (flag: boolean) => void;
-    noMoreDataBottom: boolean;
-    setNoMoreDataBottom: (flag: boolean) => void;
-    methodsFilter: Array<string>;
-    statusFilter: Array<string>;
-    pathFilter: string;
-    serviceFilter: string;
     listEntryREF: any;
-    onScrollEvent: (isAtBottom:boolean) => void;
-    scrollableList: boolean;
+    onSnapBrokenEvent: () => void;
+    isSnappedToBottom: boolean;
+    setIsSnappedToBottom: any;
+    queriedCurrent: number;
+    queriedTotal: number;
+    startTime: number;
 }
 
-enum FetchOperator {
-    LT = "lt",
-    GT = "gt"
-}
-
-const api = new Api();
-
-export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter, serviceFilter, listEntryREF, onScrollEvent, scrollableList}) => {
-
-    const [loadMoreTop, setLoadMoreTop] = useState(false);
-    const [isLoadingTop, setIsLoadingTop] = useState(false);
+export const EntriesList: React.FC<EntriesListProps> = ({entries, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, queriedTotal, startTime}) => {
 
     const scrollableRef = useRef(null);
 
-    useEffect(() => {
-        const list = document.getElementById('list').firstElementChild;
-        list.addEventListener('scroll', (e) => {
-            const el: any = e.target;
-            if(el.scrollTop === 0) {
-                setLoadMoreTop(true);
-            } else {
-                setLoadMoreTop(false);
-            }
-        });
-    }, []);
-
-    const filterEntries = useCallback((entry) => {
-        if(methodsFilter.length > 0 && !methodsFilter.includes(entry.method.toLowerCase())) return;
-        if(pathFilter && entry.path?.toLowerCase()?.indexOf(pathFilter) === -1) return;
-        if(serviceFilter && entry.service?.toLowerCase()?.indexOf(serviceFilter) === -1) return;
-        if(statusFilter.includes(StatusType.SUCCESS) && entry.statusCode >= 400) return;
-        if(statusFilter.includes(StatusType.ERROR) && entry.statusCode < 400) return;
-        return entry;
-    },[methodsFilter, pathFilter, statusFilter, serviceFilter])
-
-    const filteredEntries = useMemo(() => {
-        return entries.filter(filterEntries);
-    },[entries, filterEntries])
-
-    const getOldEntries = useCallback(async () => {
-        setIsLoadingTop(true);
-        const data = await api.fetchEntries(FetchOperator.LT, entries[0].timestamp);
-        setLoadMoreTop(false);
-
-        let scrollTo;
-        if(data.length === 0) {
-            setNoMoreDataTop(true);
-            scrollTo = document.getElementById("noMoreDataTop");
-        } else {
-            scrollTo = document.getElementById(filteredEntries?.[0]?.id);
-        }
-        setIsLoadingTop(false);
-        const newEntries = [...data, ...entries];
-        setEntries(newEntries);
-
-        if(scrollTo) {
-            scrollTo.scrollIntoView();
-        }
-    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, filteredEntries, setNoMoreDataTop])
-
-    useEffect(() => {
-        if(!loadMoreTop || connectionOpen || noMoreDataTop) return;
-        getOldEntries();
-    }, [loadMoreTop, connectionOpen, noMoreDataTop, getOldEntries]);
-
-    const getNewEntries = async () => {
-        const data = await api.fetchEntries(FetchOperator.GT, entries[entries.length - 1].timestamp);
-        let scrollTo;
-        if(data.length === 0) {
-            setNoMoreDataBottom(true);
-        }
-        scrollTo = document.getElementById(filteredEntries?.[filteredEntries.length -1]?.id);
-        let newEntries = [...entries, ...data];
-        setEntries(newEntries);
-        if(scrollTo) {
-            scrollTo.scrollIntoView({behavior: "smooth"});
-        }
-    }
-
     return <>
             <div className={styles.list}>
                 <div id="list" ref={listEntryREF} className={styles.list}>
-                    {isLoadingTop && <div className={styles.spinnerContainer}>
-                        <img alt="spinner" src={spinner} style={{height: 25}}/>
-                    </div>}
-                    <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onScroll={(isAtBottom) => onScrollEvent(isAtBottom)}>
-                        {noMoreDataTop && !connectionOpen && <div id="noMoreDataTop" className={styles.noMoreDataAvailable}>No more data available</div>}
-                        {filteredEntries.map(entry => <EntryItem key={entry.id}
-                                                        entry={entry}
-                                                        setFocusedEntryId={setFocusedEntryId}
-                                                        isSelected={focusedEntryId === entry.id}
-                                                        style={{}}/>)}
+                    <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onSnapBroken={onSnapBrokenEvent}>
+                        {false /* TODO: why there is a need for something here (not necessarily false)? */}
+                        {entries}
                     </ScrollableFeedVirtualized>
-                    {!connectionOpen && !noMoreDataBottom && <div className={styles.fetchButtonContainer}>
-                        <div className={styles.styledButton} onClick={() => getNewEntries()}>Fetch more entries</div>
-                    </div>}
                     <button type="button"
-                        className={`${styles.btnLive} ${scrollableList ? styles.showButton : styles.hideButton}`}
-                        onClick={(_) => scrollableRef.current.jumpToBottom()}>
+                        className={`${styles.btnLive} ${isSnappedToBottom ? styles.hideButton : styles.showButton}`}
+                        onClick={(_) => {
+                            scrollableRef.current.jumpToBottom();
+                            setIsSnappedToBottom(true);
+                        }}>
                         <img alt="down" src={down} />
                     </button>
                 </div>
 
-                {entries?.length > 0 && <div className={styles.footer}>
-                    <div><b>{filteredEntries?.length !== entries.length && `${filteredEntries?.length} / `} {entries?.length}</b> requests</div>
-                    <div>Started listening at <span style={{marginRight: 5, fontWeight: 600, fontSize: 13}}>{new Date(+entries[0].timestamp)?.toLocaleString()}</span></div>
-                </div>}
+                <div className={styles.footer}>
+                    <div>Displaying <b>{entries?.length}</b> results (queried <b>{queriedCurrent}</b>/<b>{queriedTotal}</b>)</div>
+                    {startTime !== 0 && <div>Started listening at <span style={{marginRight: 5, fontWeight: 600, fontSize: 13}}>{new Date(startTime).toLocaleString()}</span></div>}
+                </div>
             </div>
     </>;
 };

ui/src/components/EntryDetailed.tsx

@@ -3,7 +3,7 @@ import EntryViewer from "./EntryDetailed/EntryViewer";
 import {makeStyles} from "@material-ui/core";
 import Protocol from "./UI/Protocol"
 import StatusCode from "./UI/StatusCode";
-import {EndpointPath} from "./UI/EndpointPath";
+import {Summary} from "./UI/Summary";
 
 const useStyles = makeStyles(() => ({
     entryTitle: {
@@ -28,50 +28,79 @@ const useStyles = makeStyles(() => ({
 
 interface EntryDetailedProps {
     entryData: any
+    updateQuery: any
 }
 
 export const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`;
 
-const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime}) => {
+const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updateQuery}) => {
     const classes = useStyles();
-    const {response} = JSON.parse(data.entry);
+    const response = data.response;
 
 
     return <div className={classes.entryTitle}>
-        <Protocol protocol={protocol} horizontal={true}/>
+        <Protocol protocol={protocol} horizontal={true} updateQuery={null}/>
         <div style={{right: "30px", position: "absolute", display: "flex"}}>
-            {response.payload && <div style={{margin: "0 18px", opacity: 0.5}}>{formatSize(bodySize)}</div>}
-            {response.payload && <div style={{marginRight: 18, opacity: 0.5}}>{Math.round(elapsedTime)}ms</div>}
+            {response && <div
+                className="queryable"
+                style={{margin: "0 18px", opacity: 0.5}}
+                onClick={() => {
+                    updateQuery(`response.bodySize == ${bodySize}`)
+                }}
+            >
+                {formatSize(bodySize)}
+            </div>}
+            {response && <div
+                className="queryable"
+                style={{marginRight: 18, opacity: 0.5}}
+                onClick={() => {
+                    updateQuery(`elapsedTime >= ${elapsedTime}`)
+                }}
+            >
+                {Math.round(elapsedTime)}ms
+            </div>}
         </div>
     </div>;
 };
 
-const EntrySummary: React.FC<any> = ({data}) => {
+const EntrySummary: React.FC<any> = ({data, updateQuery}) => {
     const classes = useStyles();
 
-    const {response, request} = JSON.parse(data.entry);
+    const response = data.response;
 
     return <div className={classes.entrySummary}>
-        {response?.payload && response.payload?.details && "status" in response.payload.details && <div style={{marginRight: 8}}>
-            <StatusCode statusCode={response.payload.details.status}/>
+        {response && "status" in response && <div style={{marginRight: 8}}>
+            <StatusCode statusCode={response.status} updateQuery={updateQuery}/>
         </div>}
         <div style={{flexGrow: 1, overflow: 'hidden'}}>
-            <EndpointPath method={request?.payload.method} path={request?.payload.url}/>
+            <Summary method={data.method} summary={data.summary} updateQuery={updateQuery}/>
         </div>
     </div>;
 };
 
-export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData}) => {
+export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData, updateQuery}) => {
     return <>
         <EntryTitle
             protocol={entryData.protocol}
             data={entryData.data}
             bodySize={entryData.bodySize}
             elapsedTime={entryData.data.elapsedTime}
+            updateQuery={updateQuery}
         />
-        {entryData.data && <EntrySummary data={entryData.data}/>}
+        {entryData.data && <EntrySummary data={entryData.data} updateQuery={updateQuery}/>}
         <>
-            {entryData.data && <EntryViewer representation={entryData.representation} isRulesEnabled={entryData.isRulesEnabled} rulesMatched={entryData.rulesMatched} contractStatus={entryData.data.contractStatus} requestReason={entryData.data.contractRequestReason} responseReason={entryData.data.contractResponseReason} contractContent={entryData.data.contractContent} elapsedTime={entryData.data.elapsedTime} color={entryData.protocol.backgroundColor}/>}
+            {entryData.data && <EntryViewer
+                representation={entryData.representation}
+                isRulesEnabled={entryData.isRulesEnabled}
+                rulesMatched={entryData.rulesMatched}
+                contractStatus={entryData.data.contractStatus}
+                requestReason={entryData.data.contractRequestReason}
+                responseReason={entryData.data.contractResponseReason}
+                contractContent={entryData.data.contractContent}
+                elapsedTime={entryData.data.elapsedTime}
+                color={entryData.protocol.backgroundColor}
+                updateQuery={updateQuery}
+            />}
         </>
     </>
 };

ui/src/components/EntryDetailed/EntrySections.tsx

@@ -9,11 +9,29 @@ import ProtobufDecoder from "protobuf-decoder";
 interface EntryViewLineProps {
     label: string;
     value: number | string;
+    updateQuery: any;
+    selector: string;
+    overrideQueryValue?: string;
 }
 
-const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value}) => {
+const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value, updateQuery, selector, overrideQueryValue}) => {
     return (label && value && <tr className={styles.dataLine}>
-                <td className={styles.dataKey}>{label}</td>
+                <td
+                    className={`queryable ${styles.dataKey}`}
+                    onClick={() => {
+                        if (!selector) {
+                            return
+                        } else if (overrideQueryValue) {
+                            updateQuery(`${selector} == ${overrideQueryValue}`)
+                        } else if (typeof(value) === "string") {
+                            updateQuery(`${selector} == "${JSON.stringify(value).slice(1, -1)}"`)
+                        } else {
+                            updateQuery(`${selector} == ${value}`)
+                        }
+                    }}
+                >
+                    {label}
+                </td>
                 <td>
                     <FancyTextDisplay
                         className={styles.dataValue}
@@ -62,15 +80,19 @@ export const EntrySectionContainer: React.FC<EntrySectionContainerProps> = ({tit
 interface EntryBodySectionProps {
     content: any,
     color: string,
+    updateQuery: any,
     encoding?: string,
     contentType?: string,
+    selector?: string,
 }
 
 export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
     color,
+    updateQuery,
     content,
     encoding,
     contentType,
+    selector,
 }) => {
     const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes
     const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...]
@@ -107,8 +129,8 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
         {content && content?.length > 0 && <EntrySectionContainer title='Body' color={color}>
             <table>
                 <tbody>
-                    <EntryViewLine label={'Mime type'} value={contentType}/>
-                    <EntryViewLine label={'Encoding'} value={encoding}/>
+                    <EntryViewLine label={'Mime type'} value={contentType} updateQuery={updateQuery} selector={selector} overrideQueryValue={`r".*"`}/>
+                    <EntryViewLine label={'Encoding'} value={encoding} updateQuery={updateQuery} selector={selector} overrideQueryValue={`r".*"`}/>
                 </tbody>
             </table>
 
@@ -132,17 +154,23 @@ interface EntrySectionProps {
     title: string,
     color: string,
     arrayToIterate: any[],
+    updateQuery: any,
 }
 
-export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, arrayToIterate}) => {
+export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, arrayToIterate, updateQuery}) => {
     return <React.Fragment>
         {
             arrayToIterate && arrayToIterate.length > 0 ?
                 <EntrySectionContainer title={title} color={color}>
                     <table>
                         <tbody>
-                            {arrayToIterate.map(({name, value}, index) => <EntryViewLine key={index} label={name}
-                                                                                            value={value}/>)}
+                            {arrayToIterate.map(({name, value, selector}, index) => <EntryViewLine
+                                key={index}
+                                label={name}
+                                value={value}
+                                updateQuery={updateQuery}
+                                selector={selector}
+                            />)}
                         </tbody>
                     </table>
                 </EntrySectionContainer> : <span/>

ui/src/components/EntryDetailed/EntryViewer.tsx

@@ -8,7 +8,7 @@ enum SectionTypes {
     SectionBody = "body",
 }
 
-const SectionsRepresentation: React.FC<any> = ({data, color}) => {
+const SectionsRepresentation: React.FC<any> = ({data, color, updateQuery}) => {
     const sections = []
 
     if (data) {
@@ -16,12 +16,12 @@ const SectionsRepresentation: React.FC<any> = ({data, color}) => {
             switch (row.type) {
                 case SectionTypes.SectionTable:
                     sections.push(
-                        <EntryTableSection key={i} title={row.title} color={color} arrayToIterate={JSON.parse(row.data)}/>
+                        <EntryTableSection key={i} title={row.title} color={color} arrayToIterate={JSON.parse(row.data)} updateQuery={updateQuery}/>
                     )
                     break;
                 case SectionTypes.SectionBody:
                     sections.push(
-                        <EntryBodySection key={i} color={color} content={row.data} encoding={row.encoding} contentType={row.mime_type}/>
+                        <EntryBodySection key={i} color={color} content={row.data} updateQuery={updateQuery} encoding={row.encoding} contentType={row.mimeType} selector={row.selector}/>
                     )
                     break;
                 default:
@@ -33,7 +33,7 @@ const SectionsRepresentation: React.FC<any> = ({data, color}) => {
     return <>{sections}</>;
 }
 
-const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => {
+const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => {
     var TABS = [
         {
             tab: 'Request'
@@ -85,10 +85,10 @@ const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rule
                 <Tabs tabs={TABS} currentTab={currentTab} color={color} onChange={setCurrentTab} leftAligned/>
             </div>
             {currentTab === TABS[0].tab && <React.Fragment>
-                <SectionsRepresentation data={request} color={color}/>
+                <SectionsRepresentation data={request} color={color} updateQuery={updateQuery}/>
             </React.Fragment>}
             {response && currentTab === TABS[responseTabIndex].tab && <React.Fragment>
-                <SectionsRepresentation data={response} color={color}/>
+                <SectionsRepresentation data={response} color={color} updateQuery={updateQuery}/>
             </React.Fragment>}
             {isRulesEnabled && currentTab === TABS[rulesTabIndex].tab && <React.Fragment>
                 <EntryTablePolicySection title={'Rule'} color={color} latency={elapsedTime} arrayToIterate={rulesMatched ? rulesMatched : []}/>
@@ -110,9 +110,10 @@ interface Props {
     contractContent: string;
     color: string;
     elapsedTime: number;
+    updateQuery: any;
 }
 
-const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => {
+const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => {
     return <AutoRepresentation
         representation={representation}
         isRulesEnabled={isRulesEnabled}
@@ -123,6 +124,7 @@ const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatc
         contractContent={contractContent}
         elapsedTime={elapsedTime}
         color={color}
+        updateQuery={updateQuery}
     />
 };
 

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -1,8 +1,8 @@
-import React from "react";
+import React, {useState} from "react";
 import styles from './EntryListItem.module.sass';
 import StatusCode, {getClassification, StatusCodeClassification} from "../UI/StatusCode";
 import Protocol, {ProtocolInterface} from "../UI/Protocol"
-import {EndpointPath} from "../UI/EndpointPath";
+import {Summary} from "../UI/Summary";
 import ingoingIconSuccess from "../assets/ingoing-traffic-success.svg"
 import ingoingIconFailure from "../assets/ingoing-traffic-failure.svg"
 import ingoingIconNeutral from "../assets/ingoing-traffic-neutral.svg"
@@ -15,7 +15,7 @@ interface Entry {
     method?: string,
     summary: string,
     service: string,
-    id: string,
+    id: number,
     statusCode?: number;
     url?: string;
     timestamp: Date;
@@ -38,11 +38,14 @@ interface Rules {
 interface EntryProps {
     entry: Entry;
     setFocusedEntryId: (id: string) => void;
-    isSelected?: boolean;
     style: object;
+    updateQuery: any;
 }
 
-export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSelected, style}) => {
+export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, style, updateQuery}) => {
+
+    const [isSelected, setIsSelected] = useState(false);
+
     const classification = getClassification(entry.statusCode)
     const numberOfRules = entry.rules.numberOfRules
     let ingoingIcon;
@@ -115,10 +118,13 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
 
     return <>
         <div
-            id={entry.id}
+            id={entry.id.toString()}
             className={`${styles.row}
             ${isSelected && !rule && !contractEnabled ? styles.rowSelected : additionalRulesProperties}`}
-            onClick={() => setFocusedEntryId(entry.id)}
+            onClick={() => {
+                setIsSelected(!isSelected);
+                setFocusedEntryId(entry.id.toString());
+            }}
             style={{
                 border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid",
                 position: "absolute",
@@ -127,14 +133,26 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
                 width: "calc(100% - 25px)",
             }}
         >
-            <Protocol protocol={entry.protocol} horizontal={false}/>
+            <Protocol
+                protocol={entry.protocol}
+                horizontal={false}
+                updateQuery={updateQuery}
+            />
             {((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0) && <div>
-                <StatusCode statusCode={entry.statusCode}/>
+                <StatusCode statusCode={entry.statusCode} updateQuery={updateQuery}/>
             </div>}
             <div className={styles.endpointServiceContainer}>
-                <EndpointPath method={entry.method} path={entry.summary}/>
+                <Summary method={entry.method} summary={entry.summary} updateQuery={updateQuery}/>
                 <div className={styles.service}>
-                    <span title="Service Name">{entry.service}</span>
+                    <span
+                        title="Service Name"
+                        className="queryable"
+                        onClick={() => {
+                            updateQuery(`service == "${entry.service}"`)
+                        }}
+                    >
+                        {entry.service}
+                    </span>
                 </div>
             </div>
             {
@@ -152,17 +170,53 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
                 : ""
             }
             <div className={styles.separatorRight}>
-                <span className={styles.port} title="Source Port">{entry.sourcePort}</span>
+                <span
+                    className={`queryable ${styles.port}`}
+                    title="Source Port"
+                    onClick={() => {
+                        updateQuery(`src.port == "${entry.sourcePort}"`)
+                    }}
+                >
+                    {entry.sourcePort}
+                </span>
                 {entry.isOutgoing ?
-                    <img src={outgoingIcon} alt="Ingoing traffic" title="Ingoing"/>
+                    <img
+                        src={outgoingIcon}
+                        alt="Ingoing traffic"
+                        title="Ingoing"
+                        onClick={() => {
+                            updateQuery(`outgoing == true`)
+                        }}
+                    />
                     :
-                    <img src={ingoingIcon} alt="Outgoing traffic" title="Outgoing"/>
+                    <img
+                        src={ingoingIcon}
+                        alt="Outgoing traffic"
+                        title="Outgoing"
+                        onClick={() => {
+                            updateQuery(`outgoing == false`)
+                        }}
+                    />
                 }
-                <span className={styles.port} title="Destination Port">{entry.destinationPort}</span>
+                <span
+                    className={`queryable ${styles.port}`}
+                    title="Destination Port"
+                    onClick={() => {
+                        updateQuery(`dst.port == "${entry.destinationPort}"`)
+                    }}
+                >
+                    {entry.destinationPort}
+                </span>
             </div>
             <div className={styles.timestamp}>
-                <span title="Timestamp">
-                    {new Date(+entry.timestamp)?.toLocaleString()}
+                <span
+                    title="Timestamp"
+                    className="queryable"
+                    onClick={() => {
+                        updateQuery(`timestamp >= datetime("${new Date(+entry.timestamp)?.toLocaleString("en-US", {timeZone: 'UTC' })}")`)
+                    }}
+                >
+                    {new Date(+entry.timestamp)?.toLocaleString("en-US")}
                 </span>
             </div>
         </div>

ui/src/components/Filters.tsx

@@ -1,137 +1,326 @@
-import React from "react";
+import React, {useRef, useState} from "react";
 import styles from './style/Filters.module.sass';
-import {FilterSelect} from "./UI/FilterSelect";
-import {TextField} from "@material-ui/core";
-import {ALL_KEY} from "./UI/Select";
+import {Button, Grid, Modal, Box, Typography, Backdrop, Fade, Divider} from "@material-ui/core";
+import CodeEditor from '@uiw/react-textarea-code-editor';
+import MenuBookIcon from '@material-ui/icons/MenuBook';
+import {SyntaxHighlighter} from "./UI/SyntaxHighlighter/index";
+import filterUIExample1 from "./assets/filter-ui-example-1.png"
+import filterUIExample2 from "./assets/filter-ui-example-2.png"
+import variables from '../variables.module.scss';
 
 interface FiltersProps {
-    methodsFilter: Array<string>;
-    setMethodsFilter: (methods: Array<string>) => void;
-    statusFilter: Array<string>;
-    setStatusFilter: (methods: Array<string>) => void;
-    pathFilter: string
-    setPathFilter: (val: string) => void;
-    serviceFilter: string
-    setServiceFilter: (val: string) => void;
+    query: string
+    setQuery: any
+    backgroundColor: string
+    ws: any
+    openWebSocket: (query: string) => void;
 }
 
-export const Filters: React.FC<FiltersProps> = ({methodsFilter, setMethodsFilter, statusFilter, setStatusFilter, pathFilter, setPathFilter, serviceFilter, setServiceFilter}) => {
-
+export const Filters: React.FC<FiltersProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
     return <div className={styles.container}>
-        <MethodFilter methodsFilter={methodsFilter} setMethodsFilter={setMethodsFilter}/>
-        <StatusTypesFilter statusFilter={statusFilter} setStatusFilter={setStatusFilter}/>
-        <ServiceFilter serviceFilter={serviceFilter} setServiceFilter={setServiceFilter}/>
-        <PathFilter pathFilter={pathFilter} setPathFilter={setPathFilter}/>
+        <QueryForm
+            query={query}
+            setQuery={setQuery}
+            backgroundColor={backgroundColor}
+            ws={ws}
+            openWebSocket={openWebSocket}
+        />
     </div>;
 };
 
-const _toUpperCase = v => v.toUpperCase();
+interface QueryFormProps {
+    query: string
+    setQuery: any
+    backgroundColor: string
+    ws: any
+    openWebSocket: (query: string) => void;
+}
 
-const FilterContainer: React.FC = ({children}) => {
-    return <div className={styles.filterContainer}>
-        {children}
-    </div>;
+const style = {
+    position: 'absolute',
+    top: '10%',
+    left: '50%',
+    transform: 'translate(-50%, 0%)',
+    width: '80vw',
+    bgcolor: 'background.paper',
+    borderRadius: '5px',
+    boxShadow: 24,
+    p: 4,
+    color: '#000',
 };
 
-enum HTTPMethod {
-    GET = "get",
-    PUT = "put",
-    POST = "post",
-    DELETE = "delete",
-    OPTIONS="options",
-    PATCH = "patch"
-}
+export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
 
-interface MethodFilterProps {
-    methodsFilter: Array<string>;
-    setMethodsFilter: (methods: Array<string>) => void;
-}
+    const formRef = useRef<HTMLFormElement>(null);
 
-const MethodFilter: React.FC<MethodFilterProps> = ({methodsFilter, setMethodsFilter}) => {
+    const [openModal, setOpenModal] = useState(false);
 
-    const methodClicked = (val) => {
-        if(val === ALL_KEY) {
-            setMethodsFilter([]);
-            return;
-        }
-        if(methodsFilter.includes(val)) {
-            setMethodsFilter(methodsFilter.filter(method => method !== val))
-        } else {
-            setMethodsFilter([...methodsFilter, val]);
-        }
+    const handleOpenModal = () => setOpenModal(true);
+    const handleCloseModal = () => setOpenModal(false);
+
+    const handleChange = async (e) => {
+        setQuery(e.target.value);
     }
 
-    return <FilterContainer>
-        <FilterSelect
-            items={Object.values(HTTPMethod)}
-            allowMultiple={true}
-            value={methodsFilter}
-            onChange={(val) => methodClicked(val)}
-            transformDisplay={_toUpperCase}
-            label={"Methods"}
-        />
-    </FilterContainer>;
-};
-
-export enum StatusType {
-    SUCCESS = "success",
-    ERROR = "error"
-}
-
-interface StatusTypesFilterProps {
-    statusFilter: Array<string>;
-    setStatusFilter: (methods: Array<string>) => void;
-}
-
-const StatusTypesFilter: React.FC<StatusTypesFilterProps> = ({statusFilter, setStatusFilter}) => {
-
-    const statusClicked = (val) => {
-        if(val === ALL_KEY) {
-            setStatusFilter([]);
-            return;
-        }
-        setStatusFilter([val]);
+    const handleSubmit = (e) => {
+        ws.close()
+        openWebSocket(query)
+        e.preventDefault();
     }
 
-    return <FilterContainer>
-        <FilterSelect
-            items={Object.values(StatusType)}
-            allowMultiple={true}
-            value={statusFilter}
-            onChange={(val) => statusClicked(val)}
-            transformDisplay={_toUpperCase}
-            label="Status"
-        />
-    </FilterContainer>;
-};
+    return <>
+        <form
+            ref={formRef}
+            onSubmit={handleSubmit}
+            style={{
+                width: '100%',
+            }}
+        >
+            <Grid container spacing={2}>
+                <Grid
+                    item
+                    xs={8}
+                    style={{
+                        maxHeight: '25vh',
+                        overflowY: 'auto',
+                    }}
+                >
+                    <label>
+                        <CodeEditor
+                            value={query}
+                            language="py"
+                            placeholder="Mizu Filter Syntax"
+                            onChange={handleChange}
+                            padding={8}
+                            style={{
+                                fontSize: 14,
+                                backgroundColor: `${backgroundColor}`,
+                                fontFamily: 'ui-monospace,SFMono-Regular,SF Mono,Consolas,Liberation Mono,Menlo,monospace',
+                            }}
+                        />
+                    </label>
+                </Grid>
+                <Grid item xs={4}>
+                    <Button
+                        type="submit"
+                        variant="contained"
+                        style={{
+                            margin: "2px 0px 0px 0px",
+                            backgroundColor: variables.blueColor,
+                            fontWeight: 600,
+                            borderRadius: "4px",
+                            color: "#fff",
+                            textTransform: "none",
+                        }}
+                    >
+                        Apply
+                    </Button>
+                    <Button
+                        title="Open Filtering Guide (Cheatsheet)"
+                        variant="contained"
+                        color="primary"
+                        style={{
+                            margin: "2px 0px 0px 10px",
+                            minWidth: "26px",
+                            backgroundColor: variables.blueColor,
+                            fontWeight: 600,
+                            borderRadius: "4px",
+                            color: "#fff",
+                            textTransform: "none",
+                        }}
+                        onClick={handleOpenModal}
+                    >
+                        <MenuBookIcon fontSize="inherit"></MenuBookIcon>
+                    </Button>
+                </Grid>
+            </Grid>
+        </form>
 
-interface PathFilterProps {
-    pathFilter: string;
-    setPathFilter: (val: string) => void;
+        <Modal
+            aria-labelledby="transition-modal-title"
+            aria-describedby="transition-modal-description"
+            open={openModal}
+            onClose={handleCloseModal}
+            closeAfterTransition
+            BackdropComponent={Backdrop}
+            BackdropProps={{
+                timeout: 500,
+            }}
+            style={{overflow: 'auto'}}
+        >
+            <Fade in={openModal}>
+                <Box sx={style}>
+                    <Typography id="modal-modal-title" variant="h5" component="h2" style={{textAlign: 'center'}}>
+                        Filtering Guide (Cheatsheet)
+                    </Typography>
+                    <Typography id="modal-modal-description">
+                        <p>Mizu has a rich filtering syntax that let's you query the results both flexibly and efficiently.</p>
+                        <p>Here are some examples that you can try;</p>
+                    </Typography>
+                    <Grid container>
+                        <Grid item xs style={{margin: "10px"}}>
+                            <Typography id="modal-modal-description">
+                                This is a simple query that matches to HTTP packets with request path "/catalogue":
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and request.path == "/catalogue"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The same query can be negated for HTTP path and written like this:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and request.path != "/catalogue"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The syntax supports regular expressions. Here is a query that matches the HTTP requests that send JSON to a server:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and request.headers["Accept"] == r"application/json.*"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                Here is another query that matches HTTP responses with status code 4xx:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and response.status == r"4.*"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The same exact query can be as integer comparison:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and response.status >= 400`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The results can be queried based on their timestamps:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`timestamp < datetime("10/28/2021, 9:13:02 PM")`}
+                                language="python"
+                            />
+                        </Grid>
+                        <Divider className={styles.divider1} orientation="vertical" flexItem />
+                        <Grid item xs style={{margin: "10px"}}>
+                            <Typography id="modal-modal-description">
+                                Since Mizu supports various protocols like gRPC, AMQP, Kafka and Redis. It's possible to write complex queries that match multiple protocols like this:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`(http and request.method == "PUT") or (amqp and request.queue.startsWith("test"))\n or (kafka and response.payload.errorCode == 2) or (redis and request.key == "example")\n or (grpc and request.headers[":path"] == r".*foo.*")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                By clicking the UI elements in both left-pane and right-pane, you can automatically select a field and update the query:
+                            </Typography>
+                            <img
+                                src={filterUIExample1}
+                                width={600}
+                                alt="Clicking to UI elements (left-pane)"
+                                title="Clicking to UI elements (left-pane)"
+                            />
+                            <Typography id="modal-modal-description">
+                                Such that; clicking this in left-pane, would append the query below:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`and service == "http://carts.sock-shop"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                Another queriable UI element example, this time from the right-pane:
+                            </Typography>
+                            <img
+                                src={filterUIExample2}
+                                width={300}
+                                alt="Clicking to UI elements (right-pane)"
+                                title="Clicking to UI elements (right-pane)"
+                            />
+                            <Typography id="modal-modal-description">
+                                A query that compares one selector to another is also a valid query:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and (request.query["x"] == response.headers["y"]\n or response.content.text.contains(request.query["x"]))`}
+                                language="python"
+                            />
+                        </Grid>
+                        <Divider className={styles.divider2} orientation="vertical" flexItem />
+                        <Grid item xs style={{margin: "10px"}}>
+                            <Typography id="modal-modal-description">
+                                There are a few helper methods included the in the filter language* to help building queries more easily.
+                            </Typography>
+                            <br></br>
+                            <Typography id="modal-modal-description">
+                                true if the given selector's value starts with the string:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`request.path.startsWith("something")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                true if the given selector's value ends with the string:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`request.path.endsWith("something")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                true if the given selector's value contains the string:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`request.path.contains("something")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                returns the UNIX timestamp which is the equivalent of the time that's provided by the string. Invalid input evaluates to false:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`timestamp >= datetime("10/19/2021, 6:29:02 PM")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                limits the number of records that are streamed back as a result of a query. Always evaluates to true:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`and limit(100)`}
+                                language="python"
+                            />
+                        </Grid>
+                    </Grid>
+                    <br></br>
+                    <Typography id="modal-modal-description" style={{fontSize: 12, fontStyle: 'italic'}}>
+                        *The filtering functionality is provided through <b>Basenine</b> database server. Please refer to <a href="https://github.com/up9inc/basenine/wiki/BFL-Syntax-Reference"><b>BFL Syntax Reference</b></a> for more information.
+                    </Typography>
+                </Box>
+            </Fade>
+        </Modal>
+    </>
 }
-
-const PathFilter: React.FC<PathFilterProps> = ({pathFilter, setPathFilter}) => {
-
-    return <FilterContainer>
-        <div className={styles.filterLabel}>Path</div>
-        <div>
-            <TextField value={pathFilter} variant="outlined" className={styles.filterText} style={{minWidth: '150px'}} onChange={(e: any) => setPathFilter(e.target.value)}/>
-        </div>
-    </FilterContainer>;
-};
-
-interface ServiceFilterProps {
-    serviceFilter: string;
-    setServiceFilter: (val: string) => void;
-}
-
-const ServiceFilter: React.FC<ServiceFilterProps> = ({serviceFilter, setServiceFilter}) => {
-
-    return <FilterContainer>
-        <div className={styles.filterLabel}>Service</div>
-        <div>
-            <TextField value={serviceFilter} variant="outlined" className={styles.filterText} style={{minWidth: '150px'}} onChange={(e: any) => setServiceFilter(e.target.value)}/>
-        </div>
-    </FilterContainer>;
-};
-

ui/src/components/TrafficPage.tsx

@@ -1,6 +1,7 @@
 import React, {useEffect, useRef, useState} from "react";
 import {Filters} from "./Filters";
 import {EntriesList} from "./EntriesList";
+import {EntryItem} from "./EntryListItem/EntryListItem";
 import {makeStyles} from "@material-ui/core";
 import "./style/TrafficPage.sass";
 import styles from './style/EntriesList.module.sass';
@@ -10,6 +11,8 @@ import pauseIcon from './assets/pause.svg';
 import variables from '../variables.module.scss';
 import {StatusBar} from "./UI/StatusBar";
 import Api, {MizuWebsocketURL} from "../helpers/api";
+import { ToastContainer, toast } from 'react-toastify';
+import 'react-toastify/dist/ReactToastify.css';
 
 const useLayoutStyles = makeStyles(() => ({
     details: {
@@ -34,7 +37,6 @@ const useLayoutStyles = makeStyles(() => ({
 enum ConnectionStatus {
     Closed,
     Connected,
-    Paused
 }
 
 interface TrafficPageProps {
@@ -49,28 +51,63 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     const classes = useLayoutStyles();
 
     const [entries, setEntries] = useState([] as any);
+    const [entriesBuffer, setEntriesBuffer] = useState([] as any);
     const [focusedEntryId, setFocusedEntryId] = useState(null);
     const [selectedEntryData, setSelectedEntryData] = useState(null);
     const [connection, setConnection] = useState(ConnectionStatus.Closed);
-    const [noMoreDataTop, setNoMoreDataTop] = useState(false);
-    const [noMoreDataBottom, setNoMoreDataBottom] = useState(false);
-
-    const [methodsFilter, setMethodsFilter] = useState([]);
-    const [statusFilter, setStatusFilter] = useState([]);
-    const [pathFilter, setPathFilter] = useState("");
-    const [serviceFilter, setServiceFilter] = useState("");
 
     const [tappingStatus, setTappingStatus] = useState(null);
 
-    const [disableScrollList, setDisableScrollList] = useState(false);
+    const [isSnappedToBottom, setIsSnappedToBottom] = useState(true);
+
+    const [query, setQuery] = useState("");
+    const [queryBackgroundColor, setQueryBackgroundColor] = useState("#f5f5f5");
+    const [addition, updateQuery] = useState("");
+
+    const [queriedCurrent, setQueriedCurrent] = useState(0);
+    const [queriedTotal, setQueriedTotal] = useState(0);
+
+    const [startTime, setStartTime] = useState(0);
+
+    useEffect(() => {
+        (async function() {
+            if (!query) {
+                setQueryBackgroundColor("#f5f5f5")
+            } else {
+                const data = await api.validateQuery(query);
+                if (!data) {
+                    return;
+                }
+                if (data.valid) {
+                    setQueryBackgroundColor("#d2fad2");
+                } else {
+                    setQueryBackgroundColor("#fad6dc");
+                }
+            }
+        })();
+    }, [query]);
+
+    useEffect(() => {
+        if (query) {
+            setQuery(`${query} and ${addition}`);
+        } else {
+            setQuery(addition);
+        }
+        // eslint-disable-next-line
+    }, [addition]);
 
     const ws = useRef(null);
 
     const listEntry = useRef(null);
 
-    const openWebSocket = () => {
+    const openWebSocket = (query) => {
+        setEntries([]);
+        setEntriesBuffer([]);
         ws.current = new WebSocket(MizuWebsocketURL);
-        ws.current.onopen = () => setConnection(ConnectionStatus.Connected);
+        ws.current.onopen = () => {
+            ws.current.send(query)
+            setConnection(ConnectionStatus.Connected);
+        }
         ws.current.onclose = () => setConnection(ConnectionStatus.Closed);
     }
 
@@ -80,19 +117,18 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
             const message = JSON.parse(e.data);
             switch (message.messageType) {
                 case "entry":
-                    const entry = message.data
-                    if (connection === ConnectionStatus.Paused) {
-                        setNoMoreDataBottom(false)
-                        return;
-                    }
-                    if (!focusedEntryId) setFocusedEntryId(entry.id)
-                    let newEntries = [...entries];
-                    setEntries([...newEntries, entry])
-                    if(listEntry.current) {
-                        if(isScrollable(listEntry.current.firstChild)) {
-                            setDisableScrollList(true)
-                        }
-                    }
+                    const entry = message.data;
+                    if (!focusedEntryId) setFocusedEntryId(entry.id.toString());
+                    setEntriesBuffer([
+                        ...entriesBuffer,
+                        <EntryItem
+                            key={entry.id}
+                            entry={entry}
+                            setFocusedEntryId={setFocusedEntryId}
+                            style={{}}
+                            updateQuery={updateQuery}
+                        />
+                    ]);
                     break
                 case "status":
                     setTappingStatus(message.tappingStatus);
@@ -103,6 +139,26 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                 case "outboundLink":
                     onTLSDetected(message.Data.DstIP);
                     break;
+                case "toast":
+                    toast[message.data.type](message.data.text, {
+                        position: "bottom-right",
+                        theme: "colored",
+                        autoClose: message.data.autoClose,
+                        hideProgressBar: false,
+                        closeOnClick: true,
+                        pauseOnHover: true,
+                        draggable: true,
+                        progress: undefined,
+                    });
+                    break;
+                case "queryMetadata":
+                    setQueriedCurrent(message.data.current);
+                    setQueriedTotal(message.data.total);
+                    setEntries(entriesBuffer);
+                    break;
+                case "startTime":
+                    setStartTime(message.data);
+                    break;
                 default:
                     console.error(`unsupported websocket message type, Got: ${message.messageType}`)
             }
@@ -111,7 +167,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     useEffect(() => {
         (async () => {
-            openWebSocket();
+            openWebSocket("rlimit(100)");
             try{
                 const tapStatusResponse = await api.tapStatus();
                 setTappingStatus(tapStatusResponse);
@@ -139,14 +195,17 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     }, [focusedEntryId])
 
     const toggleConnection = () => {
-        setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected);
+        if (connection === ConnectionStatus.Connected) {
+            ws.current.close();
+        } else {
+            openWebSocket(query);
+            setConnection(ConnectionStatus.Connected);
+        }
     }
 
     const getConnectionStatusClass = (isContainer) => {
         const container = isContainer ? "Container" : "";
         switch (connection) {
-            case ConnectionStatus.Paused:
-                return "orangeIndicator" + container;
             case ConnectionStatus.Connected:
                 return "greenIndicator" + container;
             default:
@@ -156,8 +215,6 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     const getConnectionTitle = () => {
         switch (connection) {
-            case ConnectionStatus.Paused:
-                return "traffic paused";
             case ConnectionStatus.Connected:
                 return "connected, waiting for traffic"
             default:
@@ -165,19 +222,17 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
         }
     }
 
-    const onScrollEvent = (isAtBottom) => {
-        isAtBottom ? setDisableScrollList(false) : setDisableScrollList(true)
+    const onSnapBrokenEvent = () => {
+        setIsSnappedToBottom(false)
     }
 
-    const isScrollable = (element) => {
-        return element.scrollHeight > element.clientHeight;
-    };
-
     return (
         <div className="TrafficPage">
             <div className="TrafficPageHeader">
-                {connection !== ConnectionStatus.Closed && <img style={{cursor: "pointer", marginRight: 15, height: 30}} alt="pause"
-                     src={connection === ConnectionStatus.Connected ? pauseIcon : playIcon} onClick={toggleConnection}/>}
+                <img className="playPauseIcon" style={{visibility: connection === ConnectionStatus.Connected ? "visible" : "hidden"}} alt="pause"
+                    src={pauseIcon} onClick={toggleConnection}/>
+                <img className="playPauseIcon" style={{position: "absolute", visibility: connection === ConnectionStatus.Connected ? "hidden" : "visible"}} alt="play"
+                    src={playIcon} onClick={toggleConnection}/>
                 <div className="connectionText">
                     {getConnectionTitle()}
                     <div className={"indicatorContainer " + getConnectionStatusClass(true)}>
@@ -185,42 +240,44 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     </div>
                 </div>
             </div>
-            {entries.length > 0 && <div className="TrafficPage-Container">
+            {<div className="TrafficPage-Container">
                 <div className="TrafficPage-ListContainer">
-                    <Filters methodsFilter={methodsFilter}
-                                setMethodsFilter={setMethodsFilter}
-                                statusFilter={statusFilter}
-                                setStatusFilter={setStatusFilter}
-                                pathFilter={pathFilter}
-                                setPathFilter={setPathFilter}
-                                serviceFilter={serviceFilter}
-                                setServiceFilter={setServiceFilter}
+                    <Filters
+                        query={query}
+                        setQuery={setQuery}
+                        backgroundColor={queryBackgroundColor}
+                        ws={ws.current}
+                        openWebSocket={openWebSocket}
                     />
                     <div className={styles.container}>
-                        <EntriesList entries={entries}
-                                        setEntries={setEntries}
-                                        focusedEntryId={focusedEntryId}
-                                        setFocusedEntryId={setFocusedEntryId}
-                                        connectionOpen={connection === ConnectionStatus.Connected}
-                                        noMoreDataBottom={noMoreDataBottom}
-                                        setNoMoreDataBottom={setNoMoreDataBottom}
-                                        noMoreDataTop={noMoreDataTop}
-                                        setNoMoreDataTop={setNoMoreDataTop}
-                                        methodsFilter={methodsFilter}
-                                        statusFilter={statusFilter}
-                                        pathFilter={pathFilter}
-                                        serviceFilter={serviceFilter}
-                                        listEntryREF={listEntry}
-                                        onScrollEvent={onScrollEvent}
-                                        scrollableList={disableScrollList}
+                        <EntriesList
+                            entries={entries}
+                            listEntryREF={listEntry}
+                            onSnapBrokenEvent={onSnapBrokenEvent}
+                            isSnappedToBottom={isSnappedToBottom}
+                            setIsSnappedToBottom={setIsSnappedToBottom}
+                            queriedCurrent={queriedCurrent}
+                            queriedTotal={queriedTotal}
+                            startTime={startTime}
                         />
                     </div>
                 </div>
                 <div className={classes.details}>
-                    {selectedEntryData && <EntryDetailed entryData={selectedEntryData}/>}
+                    {selectedEntryData && <EntryDetailed entryData={selectedEntryData} updateQuery={updateQuery}/>}
                 </div>
             </div>}
             {tappingStatus?.pods != null && <StatusBar tappingStatus={tappingStatus}/>}
+            <ToastContainer
+                position="bottom-right"
+                autoClose={5000}
+                hideProgressBar={false}
+                newestOnTop={false}
+                closeOnClick
+                rtl={false}
+                pauseOnFocusLoss
+                draggable
+                pauseOnHover
+            />
         </div>
     )
 };

ui/src/components/UI/EndpointPath.tsx

@@ -1,15 +0,0 @@
-import miscStyles from "./style/misc.module.sass";
-import React from "react";
-import styles from './style/EndpointPath.module.sass';
-
-interface EndpointPathProps {
-    method: string,
-    path: string
-}
-
-export const EndpointPath: React.FC<EndpointPathProps> = ({method, path}) => {
-    return <div className={styles.container}>
-        {method && <span title="Method" className={`${miscStyles.protocol} ${miscStyles.method}`}>{method}</span>}
-        {path && <div title="Summary" className={styles.path}>{path}</div>}
-    </div>
-};

ui/src/components/UI/FilterSelect.tsx

@@ -1,28 +0,0 @@
-import React from "react";
-import { MenuItem } from '@material-ui/core';
-import style from './style/FilterSelect.module.sass';
-import { Select, SelectProps } from "./Select";
-
-interface FilterSelectProps extends SelectProps {
-    items: string[];
-    value: string | string[];
-    onChange: (string) => void;
-    label?: string;
-    allowMultiple?: boolean;
-    transformDisplay?: (string) => string;
-}
-
-export const FilterSelect: React.FC<FilterSelectProps> = ({items, value, onChange, label, allowMultiple= false, transformDisplay}) => {
-    return <Select
-        value={value}
-        multiple={allowMultiple}
-        label={label}
-        onChange={onChange}
-        transformDisplay={transformDisplay}
-        labelOnTop={true}
-        labelClassName={style.SelectLabel}
-        trimItemsWhenMultiple={true}
-    >
-        {items?.map(item => <MenuItem key={item} value={item}><span className='uppercase'>{item}</span></MenuItem>)}
-    </Select>
-};

ui/src/components/UI/Protocol.tsx

@@ -4,7 +4,8 @@ import styles from './style/Protocol.module.sass';
 export interface ProtocolInterface {
     name: string
     longName: string
-    abbreviation: string
+    abbr: string
+    macro: string
     backgroundColor: string
     foregroundColor: string
     fontSize: number
@@ -16,9 +17,10 @@ export interface ProtocolInterface {
 interface ProtocolProps {
     protocol: ProtocolInterface
     horizontal: boolean
+    updateQuery: any
 }
 
-const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal}) => {
+const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal, updateQuery}) => {
     if (horizontal) {
         return <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
             <span
@@ -28,25 +30,26 @@ const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal}) => {
                     color: protocol.foregroundColor,
                     fontSize: 13,
                 }}
-                title={protocol.abbreviation}
+                title={protocol.abbr}
             >
                 {protocol.longName}
             </span>
         </a>
     } else {
-        return <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
-            <span
-                className={`${styles.base} ${styles.vertical}`}
-                style={{
-                    backgroundColor: protocol.backgroundColor,
-                    color: protocol.foregroundColor,
-                    fontSize: protocol.fontSize,
-                }}
-                title={protocol.longName}
-            >
-                {protocol.abbreviation}
-            </span>
-        </a>
+        return <span
+            className={`${styles.base} ${styles.vertical}`}
+            style={{
+                backgroundColor: protocol.backgroundColor,
+                color: protocol.foregroundColor,
+                fontSize: protocol.fontSize,
+            }}
+            title={protocol.longName}
+            onClick={() => {
+                updateQuery(protocol.macro)
+            }}
+        >
+            {protocol.abbr}
+        </span>
     }
 };
 

ui/src/components/UI/StatusCode.tsx

@@ -9,16 +9,21 @@ export enum StatusCodeClassification {
 
 interface EntryProps {
     statusCode: number
+    updateQuery: any
 }
 
-const StatusCode: React.FC<EntryProps> = ({statusCode}) => {
+const StatusCode: React.FC<EntryProps> = ({statusCode, updateQuery}) => {
 
     const classification = getClassification(statusCode)
 
     return <span
         title="Status Code"
-        className={`${styles[classification]} ${styles.base}`}>
-            {statusCode}
+        className={`queryable ${styles[classification]} ${styles.base}`}
+        onClick={() => {
+            updateQuery(`response.status == ${statusCode}`)
+        }}
+    >
+        {statusCode}
     </span>
 };
 

ui/src/components/UI/Summary.tsx

@@ -0,0 +1,32 @@
+import miscStyles from "./style/misc.module.sass";
+import React from "react";
+import styles from './style/Summary.module.sass';
+
+interface SummaryProps {
+    method: string
+    summary: string
+    updateQuery: any
+}
+
+export const Summary: React.FC<SummaryProps> = ({method, summary, updateQuery}) => {
+    return <div className={styles.container}>
+        {method && <span
+            title="Method"
+            className={`queryable ${miscStyles.protocol} ${miscStyles.method}`}
+            onClick={() => {
+                updateQuery(`method == "${method}"`)
+            }}
+        >
+            {method}
+        </span>}
+        {summary && <div
+            title="Summary"
+            className={`queryable ${styles.summary}`}
+            onClick={() => {
+                updateQuery(`summary == "${summary}"`)
+            }}
+        >
+            {summary}
+        </div>}
+    </div>
+};

ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts

@@ -112,7 +112,7 @@ export const highlighterStyle = {
         "color": "#C6C5FE"
     },
     "operator": {
-        "color": "#EDEDED"
+        "color": "#A1A1A1"
     },
     "entity": {
         "color": "#fdab2b",

ui/src/components/UI/style/Protocol.module.sass

@@ -6,7 +6,6 @@
     background-color: #000
     color: #fff
     margin-left: -8px
-    margin-bottom: -4px
 
 .vertical
     line-height: 22px

ui/src/components/UI/style/Summary.module.sass

@@ -2,7 +2,7 @@
     display: flex
     align-items: center
 
-.path
+.summary
     text-overflow: ellipsis
     overflow: hidden
-    white-space: nowrap
+    white-space: nowrap

ui/src/components/style/EntriesList.module.sass

@@ -38,23 +38,6 @@
   border: 1px solid #627ef7
   background-color: rgba(255, 255, 255, 0.06)
 
-.spinnerContainer
-  display: flex
-  justify-content: center
-  margin-bottom: 10px
-
-.noMoreDataAvailable
-  text-align: center
-  font-weight: 600
-  color: $secondary-font-color
-.fetchButtonContainer
-  width: 100%
-  display: flex
-  justify-content: center
-  margin-top: 12px
-  font-weight: 600
-  color: rgba(255,255,255,0.75)
-
 .btnLive
   position: absolute
   bottom: 10px

ui/src/components/style/Filters.module.sass

@@ -4,9 +4,6 @@
     display: flex
     flex-direction: row
     align-items: center
-    min-height: 3rem
-    overflow-y: hidden
-    overflow-x: auto
     padding: .5rem 0
     border-bottom: 1px solid #BCC6DD
     margin-right: 20px
@@ -29,8 +26,24 @@
   input
     padding: 4px 12px
     background: $main-background-color
-    border-radius: 12px
-    font-size: 12px
+    border-radius: 4px
+    font-size: 14px
     border: 1px solid #BCC6DD
   fieldset
     border: none
+
+$divider-breakpoint-1: 1474px
+$divider-breakpoint-2: 1366px
+$divider-breakpoint-3: 1980px
+
+@media (max-width: $divider-breakpoint-1)
+  .divider1
+    display: none
+
+@media (max-width: $divider-breakpoint-2)
+  .divider2
+    display: none
+
+@media (min-width: $divider-breakpoint-1) and (max-width: $divider-breakpoint-3)
+  .divider2
+    display: none

ui/src/components/style/TrafficPage.sass

@@ -110,3 +110,8 @@
     align-items: center
     height: 17px
     font-size: 16px
+
+.playPauseIcon
+    cursor: pointer
+    margin-right: 15px
+    height: 30px

ui/src/helpers/api.js

@@ -1,16 +1,16 @@
 import * as axios from "axios";
 
-const mizuAPIPathPrefix = "/mizu";
-
 // When working locally cp `cp .env.example .env`
-export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : `ws://${window.location.host}${mizuAPIPathPrefix}/ws`;
+export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : `ws://${window.location.host}/ws`;
+
+const CancelToken = axios.CancelToken;
 
 export default class Api {
 
     constructor() {
 
         // When working locally cp `cp .env.example .env`
-        const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}${mizuAPIPathPrefix}/`;
+        const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`;
 
         this.client = axios.create({
             baseURL: apiURL,
@@ -19,6 +19,8 @@ export default class Api {
                 Accept: "application/json",
             }
         });
+
+        this.source = null;
     }
 
     tapStatus = async () => {
@@ -31,13 +33,8 @@ export default class Api {
         return response.data;
     }
 
-    getEntry = async (entryId) => {
-        const response = await this.client.get(`/entries/${entryId}`);
-        return response.data;
-    }
-
-    fetchEntries = async (operator, timestamp) => {
-        const response = await this.client.get(`/entries?limit=50&operator=${operator}&timestamp=${timestamp}`);
+    getEntry = async (id) => {
+        const response = await this.client.get(`/entries/${id}`);
         return response.data;
     }
 
@@ -50,4 +47,27 @@ export default class Api {
         const response = await this.client.get("/status/auth");
         return response.data;
     }
+
+    validateQuery = async (query) => {
+        if (this.source) {
+            this.source.cancel();
+        }
+        this.source = CancelToken.source();
+
+        const form = new FormData();
+        form.append('query', query)
+        const response = await this.client.post(`/query/validate`, form, {
+            cancelToken: this.source.token
+        }).catch(function (thrown) {
+            if (!axios.isCancel(thrown)) {
+                console.error('Validate error', thrown.message);
+            }
+        });
+
+        if (!response) {
+            return null;
+        }
+
+        return response.data;
+    }
 }

ui/src/index.sass

@@ -22,6 +22,11 @@ code
 .uppercase
   text-transform: uppercase
 
+.queryable
+  cursor: pointer
+  &:hover
+    text-decoration: underline
+
 /****
  * Button
  ***/