Improving daemon documentation (#457)

* discover envoy pids using cluster ips

* add istio flag to cli + rename mtls flag to istio

* add istio.md to docs

* Fixing typos

* Fix minor typos and grammer in docs

Co-authored-by: Nimrod Gilboa Markevich <nimrod@up9.com>

.github/workflows/inactive-issues-close.yaml

@@ -0,0 +1,22 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v3
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -15,7 +15,6 @@
 # vendor/
 .idea/
 build
-*.db
 
 # Mac OS
 .DS_Store
@@ -29,3 +28,7 @@ build
 
 # pprof
 pprof/*
+
+# Database Files
+*.bin
+*.gob

Dockerfile

@@ -13,7 +13,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash
+RUN apk add libpcap-dev gcc g++ make bash perl-utils
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -24,7 +24,7 @@ COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
+RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 ARG COMMIT_HASH
 ARG GIT_BRANCH
@@ -41,16 +41,24 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
+# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.10/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.10/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+RUN shasum -a 256 -c basenine_linux_amd64.sha256
+RUN chmod +x ./basenine_linux_amd64
+
 COPY devops/build_extensions.sh ..
 RUN cd .. && /bin/bash build_extensions.sh
 
 FROM alpine:3.14
 
-RUN apk add bash libpcap-dev tcpdump
+RUN apk add bash libpcap-dev
+
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 RUN mkdir /app/data/

README.md

@@ -11,7 +11,12 @@ Think TCPDump and Chrome Dev Tools combined.
 ## Features
 
 - Simple and powerful CLI
-- Real-time view of all HTTP requests, REST and gRPC API calls
+- Monitoring network traffic in real-time. Supported protocols:
+  - [HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc2616)
+  - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
+  - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
+  - [Apache Kafka](https://kafka.apache.org/protocol)
+  - [Redis](https://redis.io/topics/protocol)
 - No installation or code instrumentation
 - Works completely on premises
 
@@ -44,15 +49,6 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
 ### Development (unstable) Build
 Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
 
-## Kubeconfig & Permissions
-While `mizu`most often works out of the box, you can influence its behavior:
-
-1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
-2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
-
-For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
-
-
 ## How to Run
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
@@ -83,7 +79,7 @@ To tap all pods in current namespace -
 ```
 
 
-To tap specific pod - 
+### To tap specific pod
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -96,7 +92,7 @@ To tap specific pod -
  ^C
 ```
 
-To tap multiple pods using regex - 
+### To tap multiple pods using regex
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -114,21 +110,22 @@ To tap multiple pods using regex -
 
 ## Configuration
 
-Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
-In case no config file found, defaults will be used <br />
+Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=<PATH>`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml 
 In case of partial configuration defined, all other fields will be used with defaults <br />
 You can always override the defaults or config file with CLI flags
 
 To get the default config params run `mizu config` <br />
 To generate a new config file with default values use `mizu config -r`
 
-### Telemetry
-
-By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file
-
 
 ## Advanced Usage
 
+### Kubeconfig
+
+It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag
+with `--set kube-config-path=<PATH>`. </br >
+If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config`
+
 ### Namespace-Restricted Mode
 
 Some users have permission to only manage resources in one particular namespace assigned to them
@@ -142,6 +139,8 @@ using the `--namespace` flag or by setting `tap.namespaces` in the config file
 
 Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
 
+For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
+
 ### User agent filtering
 
 User-agent filtering (like health checks) - can be configured using command-line options:
@@ -182,14 +181,10 @@ and when changed it will support accessing by IP
 
 ### Run in daemon mode
 
-Mizu can be ran detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run indefinitely in the cluster.
-
-Please note that daemon mode requires you to have RBAC creation permissions, see the [permissions](docs/PERMISSIONS.md) doc for more details.
-
-In order to access a daemon mizu you will have to run `mizu view` after running the `tap --daemon` command.
-
-To stop the detached mizu instance and clean all cluster side resources, run `mizu clean`
+Mizu can be run detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run
+indefinitely in the cluster.
 
+For more information please refer to [DAEMON MODE](docs/DAEMON_MODE.md)
 
 ## How to Run local UI
 

acceptanceTests/Makefile

@@ -1,2 +1,2 @@
 test: ## Run acceptance tests.
-	@go test ./... -timeout 1h
+	@go test ./... -timeout 1h -v

acceptanceTests/config_test.go

@@ -2,11 +2,12 @@ package acceptanceTests
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"testing"
+
+	"gopkg.in/yaml.v3"
 )
 
 type tapConfig struct {

acceptanceTests/go.mod

@@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tests
 go 1.16
 
 require (
+	github.com/gorilla/websocket v1.4.2
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

acceptanceTests/go.sum

@@ -211,6 +211,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -303,6 +304,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=

acceptanceTests/tap_test.go

@@ -66,21 +66,18 @@ func TestTap(t *testing.T) {
 			entriesCheckFunc := func() error {
 				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-				entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
-				requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-				if requestErr != nil {
-					return fmt.Errorf("failed to get entries, err: %v", requestErr)
+				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
+				if err != nil {
+					return err
 				}
-
-				entries := requestResult.([]interface{})
-				if len(entries) == 0 {
-					return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+				err = checkEntriesAtLeast(entries, 1)
+				if err != nil {
+					return err
 				}
-
-				entry :=  entries[0].(map[string]interface{})
+				entry := entries[0]
 
 				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
-				requestResult, requestErr = executeHttpGetRequest(entryUrl)
+				requestResult, requestErr := executeHttpGetRequest(entryUrl)
 				if requestErr != nil {
 					return fmt.Errorf("failed to get entry, err: %v", requestErr)
 				}
@@ -150,10 +147,7 @@ func TestTapAllNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
@@ -202,19 +196,7 @@ func TestTapAllNamespaces(t *testing.T) {
 	}
 
 	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
 			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
 			return
 		}
@@ -226,10 +208,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
 		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
@@ -288,19 +267,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 	}
 
 	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
 			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
 			return
 		}
@@ -313,10 +280,7 @@ func TestTapRegex(t *testing.T) {
 	}
 
 	regexPodName := "httpbin2"
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: regexPodName, Namespace: "mizu-tests"},
 	}
 
@@ -371,19 +335,7 @@ func TestTapRegex(t *testing.T) {
 	}
 
 	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
 			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
 			return
 		}
@@ -431,7 +383,7 @@ func TestTapDryRun(t *testing.T) {
 		resultChannel <- "fail"
 	}()
 
-	testResult := <- resultChannel
+	testResult := <-resultChannel
 	if testResult != "success" {
 		t.Errorf("unexpected result - dry run cmd not done")
 	}
@@ -475,9 +427,10 @@ func TestTapRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
@@ -486,51 +439,39 @@ func TestTapRedact(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
+		firstEntry := entries[0]
 
 		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		requestResult, requestErr := executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
+		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		request := entry["request"].(map[string]interface{})
 
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		headers :=  entryDetails["headers"].([]interface{})
+		headers := request["_headers"].([]interface{})
 		for _, headerInterface := range headers {
 			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Agent" {
+			if header["name"].(string) != "User-Header" {
 				continue
 			}
 
-			userAgent := header["value"].(string)
-			if userAgent != "[REDACTED]" {
+			userHeader := header["value"].(string)
+			if userHeader != "[REDACTED]" {
 				return fmt.Errorf("unexpected result - user agent is not redacted")
 			}
 		}
 
-		postData := entryDetails["postData"].(map[string]interface{})
+		postData := request["postData"].(map[string]interface{})
 		textDataStr := postData["text"].(string)
 
 		var textData map[string]string
@@ -590,9 +531,10 @@ func TestTapNoRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
@@ -601,51 +543,39 @@ func TestTapNoRedact(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
+		firstEntry := entries[0]
 
 		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		requestResult, requestErr := executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
+		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		request := entry["request"].(map[string]interface{})
 
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		headers :=  entryDetails["headers"].([]interface{})
+		headers := request["_headers"].([]interface{})
 		for _, headerInterface := range headers {
 			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Agent" {
+			if header["name"].(string) != "User-Header" {
 				continue
 			}
 
-			userAgent := header["value"].(string)
-			if userAgent == "[REDACTED]" {
+			userHeader := header["value"].(string)
+			if userHeader == "[REDACTED]" {
 				return fmt.Errorf("unexpected result - user agent is redacted")
 			}
 		}
 
-		postData := entryDetails["postData"].(map[string]interface{})
+		postData := request["postData"].(map[string]interface{})
 		textDataStr := postData["text"].(string)
 
 		var textData map[string]string
@@ -716,38 +646,26 @@ func TestTapRegexMasking(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
+		firstEntry := entries[0]
 
 		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		requestResult, requestErr := executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
+		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		request := entry["request"].(map[string]interface{})
 
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		postData := entryDetails["postData"].(map[string]interface{})
+		postData := request["postData"].(map[string]interface{})
 		textData := postData["text"].(string)
 
 		if textData != "[REDACTED]" {
@@ -805,7 +723,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 
 	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
-	headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
+	headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
 	for i := 0; i < defaultEntriesCount; i++ {
 		if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
@@ -823,38 +741,27 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	ignoredUserAgentsCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
 
 		for _, entryInterface := range entries {
-			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
-			requestResult, requestErr = executeHttpGetRequest(entryUrl)
+			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"])
+			requestResult, requestErr := executeHttpGetRequest(entryUrl)
 			if requestErr != nil {
 				return fmt.Errorf("failed to get entry, err: %v", requestErr)
 			}
 
-			data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-			entryJson := data["entry"].(string)
+			entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+			request := entry["request"].(map[string]interface{})
 
-			var entry map[string]interface{}
-			if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-				return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-			}
-
-			entryRequest := entry["request"].(map[string]interface{})
-			entryPayload := entryRequest["payload"].(map[string]interface{})
-			entryDetails := entryPayload["details"].(map[string]interface{})
-
-			entryHeaders :=  entryDetails["headers"].([]interface{})
-			for _, headerInterface := range entryHeaders {
+			headers := request["_headers"].([]interface{})
+			for _, headerInterface := range headers {
 				header := headerInterface.(map[string]interface{})
 				if header["name"].(string) != ignoredUserAgentCustomHeader {
 					continue
@@ -922,21 +829,21 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 
-	var dumpsLogsPath string
+	var dumpLogsPath string
 	for _, file := range files {
 		fileName := file.Name()
 		if strings.Contains(fileName, "mizu_logs") {
-			dumpsLogsPath = path.Join(mizuFolderPath, fileName)
+			dumpLogsPath = path.Join(mizuFolderPath, fileName)
 			break
 		}
 	}
 
-	if dumpsLogsPath == "" {
+	if dumpLogsPath == "" {
 		t.Errorf("dump logs file not found")
 		return
 	}
 
-	zipReader, zipError := zip.OpenReader(dumpsLogsPath)
+	zipReader, zipError := zip.OpenReader(dumpLogsPath)
 	if zipError != nil {
 		t.Errorf("failed to get zip reader, err: %v", zipError)
 		return
@@ -973,3 +880,251 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 }
+
+func TestDaemonSeeTraffic(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []int{50}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			cliPath, cliPathErr := getCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapDaemonCmdArgs := getDefaultTapCommandArgsWithDaemonMode()
+
+			tapNamespace := getDefaultTapNamespace()
+			tapDaemonCmdArgs = append(tapDaemonCmdArgs, tapNamespace...)
+
+			tapCmd := exec.Command(cliPath, tapDaemonCmdArgs...)
+
+			viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...)
+
+			t.Cleanup(func() {
+				daemonCleanup(t, viewCmd)
+			})
+
+			t.Logf("running command: %v", tapCmd.String())
+			if err := tapCmd.Run(); err != nil {
+				t.Errorf("error occured while running the tap command, err: %v", err)
+				return
+			}
+
+			t.Logf("running command: %v", viewCmd.String())
+			if err := viewCmd.Start(); err != nil {
+				t.Errorf("error occured while running the view command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := getApiServerUrl(defaultApiServerPort)
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+
+			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+			for i := 0; i < entriesCount; i++ {
+				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			entriesCheckFunc := func() error {
+				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
+				if err != nil {
+					return err
+				}
+				err = checkEntriesAtLeast(entries, 1)
+				if err != nil {
+					return err
+				}
+				entry := entries[0]
+
+				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
+				requestResult, requestErr := executeHttpGetRequest(entryUrl)
+				if requestErr != nil {
+					return fmt.Errorf("failed to get entry, err: %v", requestErr)
+				}
+
+				if requestResult == nil {
+					return fmt.Errorf("unexpected nil entry result")
+				}
+
+				return nil
+			}
+			if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
+				t.Errorf("%v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestDaemonMultipleNamespacesSeePods(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode()
+	var namespacesCmd []string
+	for _, expectedPod := range expectedPods {
+		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
+	}
+	tapCmdArgs = append(tapCmdArgs, namespacesCmd...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+
+	viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...)
+
+	t.Cleanup(func() {
+		daemonCleanup(t, viewCmd)
+	})
+
+	t.Logf("running command: %v", tapCmd.String())
+	if err := tapCmd.Run(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	t.Logf("running command: %v", viewCmd.String())
+	if err := viewCmd.Start(); err != nil {
+		t.Errorf("error occured while running the view command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
+}
+
+func TestDaemonSingleNamespaceSeePods(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
+	}
+	unexpectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode()
+	var namespacesCmd []string
+	for _, expectedPod := range expectedPods {
+		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
+	}
+	tapCmdArgs = append(tapCmdArgs, namespacesCmd...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+
+	viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...)
+
+	t.Cleanup(func() {
+		daemonCleanup(t, viewCmd)
+	})
+
+	t.Logf("running command: %v", tapCmd.String())
+	if err := tapCmd.Run(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	t.Logf("running command: %v", viewCmd.String())
+	if err := viewCmd.Start(); err != nil {
+		t.Errorf("error occured while running the view command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	for _, unexpectedPod := range unexpectedPods {
+		if isPodDescriptorInPodArray(pods, unexpectedPod) {
+			t.Errorf("unexpected result - unexpected pod found, pod namespace: %v, pod name: %v", unexpectedPod.Namespace, unexpectedPod.Name)
+			return
+		}
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		if !isPodDescriptorInPodArray(pods, expectedPod) {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
+}

acceptanceTests/testsUtils.go

@@ -3,6 +3,7 @@ package acceptanceTests
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -10,9 +11,12 @@ import (
 	"os/exec"
 	"path"
 	"strings"
+	"sync"
 	"syscall"
+	"testing"
 	"time"
 
+	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/shared"
 )
 
@@ -24,8 +28,26 @@ const (
 	defaultServiceName    = "httpbin"
 	defaultEntriesCount   = 50
 	waitAfterTapPodsReady = 3 * time.Second
+	cleanCommandTimeout   = 1 * time.Minute
 )
 
+type PodDescriptor struct {
+	Name      string
+	Namespace string
+}
+
+func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
+	for _, pod := range pods {
+		podNamespace := pod["namespace"].(string)
+		podName := pod["name"].(string)
+
+		if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
+			return true
+		}
+	}
+	return false
+}
+
 func getCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -59,7 +81,11 @@ func getProxyUrl(namespace string, service string) string {
 }
 
 func getApiServerUrl(port uint16) string {
-	return fmt.Sprintf("http://localhost:%v/mizu", port)
+	return fmt.Sprintf("http://localhost:%v", port)
+}
+
+func getWebSocketUrl(port uint16) string {
+	return fmt.Sprintf("ws://localhost:%v/ws", port)
 }
 
 func getDefaultCommandArgs() []string {
@@ -67,8 +93,9 @@ func getDefaultCommandArgs() []string {
 	telemetry := "telemetry=false"
 	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
 	imagePullPolicy := "image-pull-policy=Never"
+	headless := "headless=true"
 
-	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy}
+	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
 func getDefaultTapCommandArgs() []string {
@@ -78,6 +105,10 @@ func getDefaultTapCommandArgs() []string {
 	return append([]string{tapCommand}, defaultCmdArgs...)
 }
 
+func getDefaultTapCommandArgsWithDaemonMode() []string {
+	return append(getDefaultTapCommandArgs(), "--daemon")
+}
+
 func getDefaultTapCommandArgsWithRegex(regex string) []string {
 	tapCommand := "tap"
 	defaultCmdArgs := getDefaultCommandArgs()
@@ -103,6 +134,20 @@ func getDefaultConfigCommandArgs() []string {
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
+func getDefaultCleanCommandArgs() []string {
+	cleanCommand := "clean"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{cleanCommand}, defaultCmdArgs...)
+}
+
+func getDefaultViewCommandArgs() []string {
+	viewCommand := "view"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{viewCommand}, defaultCmdArgs...)
+}
+
 func retriesExecute(retriesCount int, executeFunc func() error) error {
 	var lastError interface{}
 
@@ -195,16 +240,57 @@ func executeHttpGetRequest(url string) (interface{}, error) {
 	return executeHttpRequest(response, requestErr)
 }
 
-func executeHttpPostRequest(url string, body interface{}) (interface{}, error) {
+func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
 	requestBody, jsonErr := json.Marshal(body)
 	if jsonErr != nil {
 		return nil, jsonErr
 	}
 
-	response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody))
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Add("Content-Type", "application/json")
+	for headerKey, headerValue := range headers {
+		request.Header.Add(headerKey, headerValue)
+	}
+
+	client := &http.Client{}
+	response, requestErr := client.Do(request)
 	return executeHttpRequest(response, requestErr)
 }
 
+func runMizuClean() error {
+	cliPath, err := getCliPath()
+	if err != nil {
+		return err
+	}
+
+	cleanCmdArgs := getDefaultCleanCommandArgs()
+
+	cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
+
+	commandDone := make(chan error)
+	go func() {
+		if err := cleanCmd.Run(); err != nil {
+			commandDone <- err
+		}
+		commandDone <- nil
+	}()
+
+	select {
+	case err = <-commandDone:
+		if err != nil {
+			return err
+		}
+	case <-time.After(cleanCommandTimeout):
+		return errors.New("clean command timed out")
+	}
+
+	return nil
+}
+
 func cleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
@@ -239,6 +325,87 @@ func getLogsPath() (string, error) {
 	return logsPath, nil
 }
 
+func daemonCleanup(t *testing.T, viewCmd *exec.Cmd) {
+	if err := runMizuClean(); err != nil {
+		t.Logf("error running mizu clean: %v", err)
+	}
+
+	if err := cleanupCommand(viewCmd); err != nil {
+		t.Logf("failed to cleanup view command, err: %v", err)
+	}
+}
+
+// waitTimeout waits for the waitgroup for the specified max timeout.
+// Returns true if waiting timed out.
+func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
+	channel := make(chan struct{})
+	go func() {
+		defer close(channel)
+		wg.Wait()
+	}()
+	select {
+	case <-channel:
+		return false // completed normally
+	case <-time.After(timeout):
+		return true // timed out
+	}
+}
+
+// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
+func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
+	if len(entries) < n {
+		return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
+	}
+	return nil
+}
+
+// getDBEntries retrieves the entries from the database before the given timestamp.
+// Also limits the results according to the limit parameter.
+// Timeout for the WebSocket connection is defined by the timeout parameter.
+func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
+	query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
+	webSocketUrl := getWebSocketUrl(defaultApiServerPort)
+
+	var connection *websocket.Conn
+	connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
+	if err != nil {
+		return
+	}
+	defer connection.Close()
+
+	handleWSConnection := func(wg *sync.WaitGroup) {
+		defer wg.Done()
+		for {
+			_, message, err := connection.ReadMessage()
+			if err != nil {
+				return
+			}
+
+			var data map[string]interface{}
+			if err = json.Unmarshal([]byte(message), &data); err != nil {
+				return
+			}
+
+			if data["messageType"] == "entry" {
+				entries = append(entries, data)
+			}
+		}
+	}
+
+	err = connection.WriteMessage(websocket.TextMessage, []byte(query))
+	if err != nil {
+		return
+	}
+
+	var wg sync.WaitGroup
+	go handleWSConnection(&wg)
+	wg.Add(1)
+
+	waitTimeout(&wg, timeout)
+
+	return
+}
+
 func Contains(slice []string, containsValue string) bool {
 	for _, sliceValue := range slice {
 		if sliceValue == containsValue {

agent/go.mod

@@ -3,8 +3,8 @@ module mizuserver
 go 1.16
 
 require (
+	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
 	github.com/djherbis/atime v1.0.0
-	github.com/fsnotify/fsnotify v1.4.9
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.2
@@ -16,13 +16,12 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/up9inc/basenine/client/go v0.0.0-20211118123155-7ed075f85c73
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	go.mongodb.org/mongo-driver v1.7.1
-	gorm.io/driver/sqlite v1.1.4
-	gorm.io/gorm v1.21.8
+	golang.org/x/text v0.3.5 // indirect
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
 	k8s.io/client-go v0.21.2

agent/go.sum

@@ -52,6 +52,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b h1:8m+eVxVVDDyJFidv7Ck1OwqnDaQR6pTSRGlCC2Dnw0A=
+github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b/go.mod h1:+tQQjzrp2501Nd6JXrb9s/XsNvFK3ZbxOnCdQl/vDRo=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -94,7 +96,6 @@ github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
 github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -111,7 +112,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY=
@@ -195,31 +195,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
-github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
-github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
-github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
-github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
-github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
-github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
-github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
-github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
-github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
-github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
-github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
-github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
-github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
-github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
-github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -251,7 +227,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -321,12 +296,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
-github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -336,14 +305,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
-github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -364,16 +329,12 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
-github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
-github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -395,7 +356,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
@@ -419,7 +379,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -447,8 +406,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -458,8 +415,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -468,7 +423,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -485,8 +439,9 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -495,26 +450,22 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/up9inc/basenine/client/go v0.0.0-20211118123155-7ed075f85c73 h1:FJUM7w7E0jRGFPcSMa7cVy+jr5zcpbyT6qA30dEtGGI=
+github.com/up9inc/basenine/client/go v0.0.0-20211118123155-7ed075f85c73/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
-github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA=
-go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -527,13 +478,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
@@ -611,7 +560,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -625,13 +573,10 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -685,13 +630,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -805,11 +746,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
-gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
-gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v1.21.8 h1:2CEwZSzogdhsKPlJ9OvBKTdlWIpELXb6HbfLfMNhSYI=
-gorm.io/gorm v1.21.8/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

agent/main.go

@@ -6,13 +6,10 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/up9inc/mizu/shared/kubernetes"
 	"io/ioutil"
-	v1 "k8s.io/api/core/v1"
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
 	"mizuserver/pkg/routes"
@@ -20,6 +17,7 @@ import (
 	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
@@ -28,10 +26,15 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/up9inc/mizu/shared/kubernetes"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/antelman107/net-wait-go/wait"
 	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
@@ -49,10 +52,12 @@ var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 var extensions []*tapApi.Extension             // global
 var extensionsMap map[string]*tapApi.Extension // global
 
+var startTime int64
+
 const (
-	socketConnectionRetries = 10
+	socketConnectionRetries    = 10
 	socketConnectionRetryDelay = time.Second * 2
-	socketHandshakeTimeout = time.Second * 2
+	socketHandshakeTimeout     = time.Second * 2
 )
 
 func main() {
@@ -89,17 +94,17 @@ func main() {
 			panic("API server address must be provided with --api-server-address when using --tap")
 		}
 
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
 		tapTargets := getTapTargets()
 		if tapTargets != nil {
-			tap.SetFilterAuthorities(tapTargets)
-			logger.Log.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
+			tapOpts.FilterAuthorities = tapTargets
+			logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
 		}
 
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 
 		filteringOptions := getTrafficFilteringOptions()
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
 		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
 		socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
 		if err != nil {
@@ -109,7 +114,8 @@ func main() {
 
 		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
 	} else if *apiServerMode {
-		database.InitDataBase(config.Config.AgentDatabasePath)
+		startBasenineServer(shared.BasenineHost, shared.BaseninePort)
+		startTime = time.Now().UnixNano() / int64(time.Millisecond)
 		api.StartResolving(*namespace)
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
@@ -142,6 +148,53 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
+func startBasenineServer(host string, port string) {
+	cmd := exec.Command("basenine", "-addr", host, "-port", port, "-persistent")
+	cmd.Dir = config.Config.AgentDatabasePath
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Start()
+	if err != nil {
+		logger.Log.Panicf("Failed starting Basenine: %v", err)
+	}
+
+	if !wait.New(
+		wait.WithProto("tcp"),
+		wait.WithWait(200*time.Millisecond),
+		wait.WithBreak(50*time.Millisecond),
+		wait.WithDeadline(5*time.Second),
+		wait.WithDebug(true),
+	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
+		logger.Log.Panicf("Basenine is not available: %v", err)
+	}
+
+	// Make a channel to gracefully exit Basenine.
+	channel := make(chan os.Signal)
+	signal.Notify(channel, os.Interrupt, syscall.SIGTERM)
+
+	// Handle the channel.
+	go func() {
+		<-channel
+		cmd.Process.Signal(syscall.SIGTERM)
+	}()
+
+	// Limit the database size to default 200MB
+	err = basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
+	if err != nil {
+		logger.Log.Panicf("Error while limiting database size: %v", err)
+	}
+
+	for _, extension := range extensions {
+		macros := extension.Dissector.Macros()
+		for macro, expanded := range macros {
+			err = basenine.Macro(host, port, macro, expanded)
+			if err != nil {
+				logger.Log.Panicf("Error while adding a macro: %v", err)
+			}
+		}
+	}
+}
+
 func loadExtensions() {
 	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
 	extensionsDir := path.Join(dir, "./extensions/")
@@ -154,7 +207,7 @@ func loadExtensions() {
 	extensionsMap = make(map[string]*tapApi.Extension)
 	for i, file := range files {
 		filename := file.Name()
-		logger.Log.Infof("Loading extension: %s\n", filename)
+		logger.Log.Infof("Loading extension: %s", filename)
 		extension := &tapApi.Extension{
 			Path: path.Join(extensionsDir, filename),
 		}
@@ -166,7 +219,7 @@ func loadExtensions() {
 		var ok bool
 		dissector, ok = symDissector.(tapApi.Dissector)
 		if err != nil || !ok {
-			panic(fmt.Sprintf("Failed to load the extension: %s\n", extension.Path))
+			panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
 		}
 		dissector.Register(extension)
 		extension.Dissector = dissector
@@ -179,7 +232,7 @@ func loadExtensions() {
 	})
 
 	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v\n", extension)
+		logger.Log.Infof("Extension Properties: %+v", extension)
 	}
 
 	controllers.InitExtensionsMap(extensionsMap)
@@ -200,7 +253,8 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	app.Use(static.ServeRoot("/", "./site"))
 	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
-	api.WebSocketRoutes(app, &eventHandlers)
+	api.WebSocketRoutes(app, &eventHandlers, startTime)
+	routes.QueryRoutes(app)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
@@ -344,10 +398,11 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig {
 }
 
 func determineLogLevel() (logLevel logging.Level) {
-	logLevel = logging.INFO
-	if os.Getenv(shared.DebugModeEnvVar) == "1" {
-		logLevel = logging.DEBUG
+	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
+	if err != nil {
+		logLevel = logging.INFO
 	}
+
 	return
 }
 
@@ -361,7 +416,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 		socketConnection, _, err := dialer.Dial(socketAddress, nil)
 		if err != nil {
 			if i < retryAmount {
-				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay / time.Second)
+				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
 				time.Sleep(retryDelay)
 			}
 		} else {
@@ -371,8 +426,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 	return nil, lastErr
 }
 
-
-func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error){
+func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error) {
 	provider, err := kubernetes.NewProviderInCluster()
 	if err != nil {
 		return nil, err
@@ -385,10 +439,11 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e
 		AgentImage:               config.Config.AgentImage,
 		TapperResources:          config.Config.TapperResources,
 		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
-		DumpLogs:                 config.Config.DumpLogs,
+		LogLevel:                 config.Config.LogLevel,
 		IgnoredUserAgents:        config.Config.IgnoredUserAgents,
 		MizuApiFilteringOptions:  config.Config.MizuApiFilteringOptions,
 		MizuServiceAccountExists: true, //assume service account exists since daemon mode will not function without it anyway
+		Istio:                    config.Config.Istio,
 	})
 
 	if err != nil {
@@ -405,7 +460,7 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e
 					return
 				}
 				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
-			case _, ok := <-tapperSyncer.TapPodChangesOut:
+			case tapPodChangeEvent, ok := <-tapperSyncer.TapPodChangesOut:
 				if !ok {
 					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
 					return
@@ -418,6 +473,7 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e
 				}
 				api.BroadcastToBrowserClients(serializedTapStatus)
 				providers.TapStatus.Pods = tapStatus.Pods
+				providers.ExpectedTapperAmount = tapPodChangeEvent.ExpectedTapperAmount
 			case <-ctx.Done():
 				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
 				return

agent/pkg/api/main.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"os"
@@ -14,15 +13,16 @@ import (
 	"strings"
 	"time"
 
-	"go.mongodb.org/mongo-driver/bson/primitive"
-
 	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
+
+	basenine "github.com/up9inc/basenine/client/go"
 )
 
 var k8sResolver *resolver.Resolver
@@ -76,7 +76,7 @@ func startReadingFiles(workingDir string) {
 		sort.Sort(utils.ByModTime(harFiles))
 
 		if len(harFiles) == 0 {
-			logger.Log.Infof("Waiting for new files\n")
+			logger.Log.Infof("Waiting for new files")
 			time.Sleep(3 * time.Second)
 			continue
 		}
@@ -99,11 +99,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
+	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		panic(err)
+	}
+	connection.InsertMode()
+
 	disableOASValidation := false
 	ctx := context.Background()
 	doc, contractContent, router, err := loadOAS(ctx)
 	if err != nil {
-		logger.Log.Infof("Disabled OAS validation: %s\n", err.Error())
+		logger.Log.Infof("Disabled OAS validation: %s", err.Error())
 		disableOASValidation = true
 	}
 
@@ -112,13 +118,13 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
 		baseEntry := extension.Dissector.Summarize(mizuEntry)
-		mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
+		mizuEntry.Base = baseEntry
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
-				json.Unmarshal([]byte(mizuEntry.Entry), &httpPair)
+				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
 				baseEntry.ContractStatus = contract.Status
@@ -128,18 +134,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			var pair tapApi.RequestResponsePair
-			json.Unmarshal([]byte(mizuEntry.Entry), &pair)
-			harEntry, err := utils.NewEntry(&pair)
+			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
 				baseEntry.Rules = rules
 			}
 		}
-		database.CreateEntry(mizuEntry)
 
-		baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)
-		BroadcastToBrowserClients(baseEntryBytes)
+		data, err := json.Marshal(mizuEntry)
+		if err != nil {
+			panic(err)
+		}
+		connection.SendText(string(data))
 	}
 }
 
@@ -148,7 +154,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedSource := connectionInfo.ClientIP
 		resolvedSource = k8sResolver.Resolve(unresolvedSource)
 		if resolvedSource == "" {
-			logger.Log.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
+			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
@@ -156,7 +162,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
 		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
 		if resolvedDestination == "" {
-			logger.Log.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
+			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
@@ -171,21 +177,3 @@ func CheckIsServiceIP(address string) bool {
 	}
 	return k8sResolver.CheckIsServiceIP(address)
 }
-
-// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
-func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int {
-	sizeBytes := len(mizuEntry.Entry)
-	sizeBytes += len(mizuEntry.EntryId)
-	sizeBytes += len(mizuEntry.Service)
-	sizeBytes += len(mizuEntry.Url)
-	sizeBytes += len(mizuEntry.Method)
-	sizeBytes += len(mizuEntry.RequestSenderIp)
-	sizeBytes += len(mizuEntry.ResolvedDestination)
-	sizeBytes += len(mizuEntry.ResolvedSource)
-	sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
-	sizeBytes += 8 // Timestamp bytes
-	sizeBytes += 8 // SizeBytes bytes
-	sizeBytes += 1 // IsOutgoing bytes
-
-	return sizeBytes
-}

agent/pkg/api/socket_routes.go

@@ -1,13 +1,18 @@
 package api
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
+	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
 )
@@ -39,17 +44,17 @@ func init() {
 	connectedWebsockets = make(map[int]*SocketConnection, 0)
 }
 
-func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
 	app.GET("/ws", func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, false)
+		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
 	})
 	app.GET("/wsTapper", func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, true)
+		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
 	})
 }
 
-func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
-	conn, err := websocketUpgrader.Upgrade(w, r, nil)
+func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
+	ws, err := websocketUpgrader.Upgrade(w, r, nil)
 	if err != nil {
 		logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
 		return
@@ -59,30 +64,110 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 
 	connectedWebsocketIdCounter++
 	socketId := connectedWebsocketIdCounter
-	connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
+	connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
 
 	websocketIdsLock.Unlock()
 
+	var connection *basenine.Connection
+	var isQuerySet bool
+
+	// `!isTapper` means it's a connection from the web UI
+	if !isTapper {
+		connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	data := make(chan []byte)
+	meta := make(chan []byte)
+
 	defer func() {
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
 		socketCleanup(socketId, connectedWebsockets[socketId])
 	}()
 
 	eventHandlers.WebSocketConnect(socketId, isTapper)
 
+	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
+	SendToSocket(socketId, startTimeBytes)
+
 	for {
-		_, msg, err := conn.ReadMessage()
+		_, msg, err := ws.ReadMessage()
 		if err != nil {
 			logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
 			break
 		}
-		eventHandlers.WebSocketMessage(socketId, msg)
+
+		if !isTapper && !isQuerySet {
+			query := string(msg)
+			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+			if err != nil {
+				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
+					Type:      "error",
+					AutoClose: 5000,
+					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
+				})
+				SendToSocket(socketId, toastBytes)
+				break
+			}
+
+			isQuerySet = true
+
+			handleDataChannel := func(c *basenine.Connection, data chan []byte) {
+				for {
+					bytes := <-data
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var dataMap map[string]interface{}
+					err = json.Unmarshal(bytes, &dataMap)
+
+					base := dataMap["base"].(map[string]interface{})
+					base["id"] = uint(dataMap["id"].(float64))
+
+					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
+					SendToSocket(socketId, baseEntryBytes)
+				}
+			}
+
+			handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
+				for {
+					bytes := <-meta
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var metadata *basenine.Metadata
+					err = json.Unmarshal(bytes, &metadata)
+					if err != nil {
+						logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+					}
+
+					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
+					SendToSocket(socketId, metadataBytes)
+				}
+			}
+
+			go handleDataChannel(connection, data)
+			go handleMetaChannel(connection, meta)
+
+			connection.Query(query, data, meta)
+		} else {
+			eventHandlers.WebSocketMessage(socketId, msg)
+		}
 	}
 }
 
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		logger.Log.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
+		logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
 	}
 
 	websocketIdsLock.Lock()

agent/pkg/api/socket_server_handlers.go

@@ -65,14 +65,14 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	var socketMessageBase shared.WebSocketMessageMetadata
 	err := json.Unmarshal(message, &socketMessageBase)
 	if err != nil {
-		logger.Log.Infof("Could not unmarshal websocket message %v\n", err)
+		logger.Log.Infof("Could not unmarshal websocket message %v", err)
 	} else {
 		switch socketMessageBase.MessageType {
 		case shared.WebSocketMessageTypeTappedEntry:
 			var tappedEntryMessage models.WebSocketTappedEntryMessage
 			err := json.Unmarshal(message, &tappedEntryMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
 				h.SocketOutChannel <- tappedEntryMessage.Data
@@ -81,7 +81,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			var statusMessage shared.WebSocketStatusMessage
 			err := json.Unmarshal(message, &statusMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
 				BroadcastToBrowserClients(message)
@@ -90,7 +90,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			var outboundLinkMessage models.WebsocketOutboundLinkMessage
 			err := json.Unmarshal(message, &outboundLinkMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				handleTLSLink(outboundLinkMessage)
 			}

agent/pkg/controllers/entries_controller.go

@@ -2,14 +2,17 @@ package controllers
 
 import (
 	"encoding/json"
-	"fmt"
-	"github.com/gin-gonic/gin"
-	tapApi "github.com/up9inc/mizu/tap/api"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/utils"
-	"mizuserver/pkg/validation"
 	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 var extensionsMap map[string]*tapApi.Extension // global
@@ -18,68 +21,47 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
 	extensionsMap = ref
 }
 
-func GetEntries(c *gin.Context) {
-	entriesFilter := &models.EntriesFilter{}
-
-	if err := c.BindQuery(entriesFilter); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-	err := validation.Validate(entriesFilter)
+func Error(c *gin.Context, err error) bool {
 	if err != nil {
-		c.JSON(http.StatusBadRequest, err)
+		logger.Log.Errorf("Error getting entry: %v", err)
+		c.Error(err)
+		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err.Error(),
+		})
+		return true // signal that there was an error and the caller should return
 	}
-
-	order := database.OperatorToOrderMapping[entriesFilter.Operator]
-	operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
-	var entries []tapApi.MizuEntry
-	database.GetEntriesTable().
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
-		Limit(entriesFilter.Limit).
-		Find(&entries)
-
-	if len(entries) > 0 && order == database.OrderDesc {
-		// the entries always order from oldest to newest - we should reverse
-		utils.ReverseSlice(entries)
-	}
-
-	baseEntries := make([]tapApi.BaseEntryDetails, 0)
-	for _, entry := range entries {
-		baseEntryDetails := tapApi.BaseEntryDetails{}
-		if err := models.GetEntry(&entry, &baseEntryDetails); err != nil {
-			continue
-		}
-
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entry.Entry), &pair)
-		harEntry, err := utils.NewEntry(&pair)
-		if err == nil {
-			rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service)
-			baseEntryDetails.Rules = rules
-		}
-
-		baseEntries = append(baseEntries, baseEntryDetails)
-	}
-
-	c.JSON(http.StatusOK, baseEntries)
+	return false // no error, can continue
 }
 
 func GetEntry(c *gin.Context) {
-	var entryData tapApi.MizuEntry
-	database.GetEntriesTable().
-		Where(map[string]string{"entryId": c.Param("entryId")}).
-		First(&entryData)
+	id, _ := strconv.Atoi(c.Param("id"))
+	var entry tapApi.MizuEntry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id)
+	if Error(c, err) {
+		return // exit
+	}
+	err = json.Unmarshal(bytes, &entry)
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       string(bytes),
+		})
+		return // exit
+	}
 
-	extension := extensionsMap[entryData.ProtocolName]
-	protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
+	extension := extensionsMap[entry.Protocol.Name]
+	protocol, representation, bodySize, _ := extension.Dissector.Represent(entry.Protocol, entry.Request, entry.Response)
 
 	var rules []map[string]interface{}
 	var isRulesEnabled bool
-	if entryData.ProtocolName == "http" {
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entryData.Entry), &pair)
-		harEntry, _ := utils.NewEntry(&pair)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service)
+	if entry.Protocol.Name == "http" {
+		harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Service)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)
 		json.Unmarshal(inrec, &rules)
@@ -89,7 +71,7 @@ func GetEntry(c *gin.Context) {
 		Protocol:       protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,
-		Data:           entryData,
+		Data:           entry,
 		Rules:          rules,
 		IsRulesEnabled: isRulesEnabled,
 	})

agent/pkg/controllers/query_controller.go

@@ -0,0 +1,31 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+)
+
+type ValidateResponse struct {
+	Valid   bool   `json:"valid"`
+	Message string `json:"message"`
+}
+
+func PostValidate(c *gin.Context) {
+	query := c.PostForm("query")
+	valid := true
+	message := ""
+
+	err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+	if err != nil {
+		valid = false
+		message = err.Error()
+	}
+
+	c.JSON(http.StatusOK, ValidateResponse{
+		Valid:   valid,
+		Message: message,
+	})
+}

agent/pkg/controllers/status_controller.go

@@ -2,7 +2,9 @@ package controllers
 
 import (
 	"encoding/json"
+	"fmt"
 	"mizuserver/pkg/api"
+	"mizuserver/pkg/config"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"mizuserver/pkg/up9"
@@ -15,6 +17,13 @@ import (
 )
 
 func HealthCheck(c *gin.Context) {
+	if config.Config.DaemonMode {
+		if providers.ExpectedTapperAmount != providers.TappersCount {
+			c.JSON(http.StatusInternalServerError, fmt.Sprintf("expecting more tappers than are actually connected (%d expected, %d connected)", providers.ExpectedTapperAmount, providers.TappersCount))
+			return
+		}
+	}
+
 	response := shared.HealthResponse{
 		TapStatus:    providers.TapStatus,
 		TappersCount: providers.TappersCount,
@@ -37,7 +46,7 @@ func PostTappedPods(c *gin.Context) {
 	providers.TapStatus.Pods = tapStatus.Pods
 	message := shared.CreateWebSocketStatusMessage(*tapStatus)
 	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v\n", err)
+		logger.Log.Errorf("Could not Marshal message %v", err)
 	} else {
 		api.BroadcastToBrowserClients(jsonBytes)
 	}

agent/pkg/database/main.go

@@ -1,78 +0,0 @@
-package database
-
-import (
-	"fmt"
-	"mizuserver/pkg/utils"
-	"time"
-
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-const (
-	OrderDesc  = "desc"
-	OrderAsc   = "asc"
-	LT         = "lt"
-	GT         = "gt"
-	TimeFormat = "2006-01-02 15:04:05.000000000"
-)
-
-var (
-	DB                      *gorm.DB
-	IsDBLocked              = false
-	OperatorToSymbolMapping = map[string]string{
-		LT: "<",
-		GT: ">",
-	}
-	OperatorToOrderMapping = map[string]string{
-		LT: OrderDesc,
-		GT: OrderAsc,
-	}
-)
-
-var DBPath string
-
-func GetEntriesTable() *gorm.DB {
-	return DB.Table("mizu_entries")
-}
-
-func CreateEntry(entry *tapApi.MizuEntry) {
-	if IsDBLocked {
-		return
-	}
-	GetEntriesTable().Create(entry)
-}
-
-func InitDataBase(databasePath string) *gorm.DB {
-	DBPath = databasePath
-	DB, _ = gorm.Open(sqlite.Open(databasePath), &gorm.Config{
-		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
-	})
-	_ = DB.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created
-	go StartEnforcingDatabaseSize()
-	return DB
-}
-
-func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry {
-	order := OrderDesc
-	protocolNameCondition := "1 = 1"
-	if protocolName != nil {
-		protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
-	}
-
-	var entries []tapApi.MizuEntry
-	GetEntriesTable().
-		Where(protocolNameCondition).
-		Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))).
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Find(&entries)
-
-	if len(entries) > 0 {
-		// the entries always order from oldest to newest so we should revers
-		utils.ReverseSlice(entries)
-	}
-	return entries
-}

agent/pkg/database/size_enforcer.go

@@ -1,102 +0,0 @@
-package database
-
-import (
-	"mizuserver/pkg/config"
-	"os"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/shared/units"
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-const percentageOfMaxSizeBytesToPrune = 15
-
-func StartEnforcingDatabaseSize() {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
-		return
-	}
-
-	checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
-		checkFileSize(config.Config.MaxDBSizeBytes)
-	})
-
-	go func() {
-		for {
-			select {
-			case event, ok := <-watcher.Events:
-				if !ok {
-					return // closed channel
-				}
-				if event.Op == fsnotify.Write {
-					checkFileSizeDebouncer.SetOn()
-				}
-			case err, ok := <-watcher.Errors:
-				if !ok {
-					return // closed channel
-				}
-				logger.Log.Errorf("filesystem watcher encountered error:%v", err)
-			}
-		}
-	}()
-
-	err = watcher.Add(DBPath)
-	if err != nil {
-		logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
-	}
-}
-
-func checkFileSize(maxSizeBytes int64) {
-	fileStat, err := os.Stat(DBPath)
-	if err != nil {
-		logger.Log.Errorf("Error checking %s file size: %v", DBPath, err)
-	} else {
-		if fileStat.Size() > maxSizeBytes {
-			pruneOldEntries(fileStat.Size())
-		}
-	}
-}
-
-func pruneOldEntries(currentFileSize int64) {
-	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
-	IsDBLocked = true
-	defer func() { IsDBLocked = false }()
-
-	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
-
-	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
-	if err != nil {
-		logger.Log.Errorf("Error getting 10000 first db rows: %v", err)
-		return
-	}
-
-	entryIdsToRemove := make([]uint, 0)
-	bytesToBeRemoved := int64(0)
-	for rows.Next() {
-		if bytesToBeRemoved >= amountOfBytesToTrim {
-			break
-		}
-		var entry tapApi.MizuEntry
-		err = DB.ScanRows(rows, &entry)
-		if err != nil {
-			logger.Log.Errorf("Error scanning db row: %v", err)
-			continue
-		}
-
-		entryIdsToRemove = append(entryIdsToRemove, entry.ID)
-		bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
-	}
-
-	if len(entryIdsToRemove) > 0 {
-		GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{})
-		// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
-		DB.Exec("VACUUM")
-		logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
-	} else {
-		logger.Log.Error("Found no rows to remove when pruning")
-	}
-}

agent/pkg/models/models.go

@@ -2,12 +2,12 @@ package models
 
 import (
 	"encoding/json"
+	"mizuserver/pkg/rules"
 
 	tapApi "github.com/up9inc/mizu/tap/api"
 
-	"mizuserver/pkg/rules"
-
 	"github.com/google/martian/har"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
 )
@@ -16,15 +16,9 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
-type EntriesFilter struct {
-	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
-	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
-	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
-}
-
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data *tapApi.BaseEntryDetails `json:"data,omitempty"`
+	Data map[string]interface{} `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
@@ -42,7 +36,28 @@ type AuthStatus struct {
 	Model string `json:"model"`
 }
 
-func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) {
+type ToastMessage struct {
+	Type      string `json:"type"`
+	AutoClose uint   `json:"autoClose"`
+	Text      string `json:"text"`
+}
+
+type WebSocketToastMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *ToastMessage `json:"data,omitempty"`
+}
+
+type WebSocketQueryMetadataMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *basenine.Metadata `json:"data,omitempty"`
+}
+
+type WebSocketStartTimeMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data int64 `json:"data"`
+}
+
+func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,
@@ -72,6 +87,36 @@ func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error)
 	return json.Marshal(message)
 }
 
+func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
+	message := &WebSocketToastMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeToast,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) {
+	message := &WebSocketQueryMetadataMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeQueryMetadata,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
+	message := &WebSocketStartTimeMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeStartTime,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
 // ExtendedHAR is the top level object of a HAR log.
 type ExtendedHAR struct {
 	Log *ExtendedLog `json:"log"`

agent/pkg/providers/stats_provider_test.go

@@ -2,16 +2,10 @@ package providers_test
 
 import (
 	"fmt"
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/providers"
 	"testing"
 )
 
-func init() {
-	database.InitDataBase(config.DefaultDatabasePath)
-}
-
 func TestNoEntryAddedCount(t *testing.T) {
 	entriesStats := providers.GetGeneralStats()
 

agent/pkg/providers/status_provider.go

@@ -19,7 +19,7 @@ var (
 	TapStatus      shared.TapStatus
 	authStatus     *models.AuthStatus
 	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
-
+	ExpectedTapperAmount = -1 //only relevant in daemon mode as cli manages tappers otherwise
 	tappersCountLock = sync.Mutex{}
 )
 

agent/pkg/resolver/resolver.go

@@ -164,10 +164,10 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
 		resolver.nameMap.Remove(key)
-		logger.Log.Infof("setting %s=nil\n", key)
+		logger.Log.Infof("setting %s=nil", key)
 	} else {
 		resolver.nameMap.Set(key, resolved)
-		logger.Log.Infof("setting %s=%s\n", key, resolved)
+		logger.Log.Infof("setting %s=%s", key, resolved)
 	}
 }
 
@@ -188,7 +188,7 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
 			var statusError *k8serrors.StatusError
 			if errors.As(err, &statusError) {
 				if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
-					logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
+					logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v", err)
 					return
 				}
 			}

agent/pkg/routes/entries_routes.go

@@ -1,14 +1,14 @@
 package routes
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/controllers"
+
+	"github.com/gin-gonic/gin"
 )
 
 // EntriesRoutes defines the group of har entries routes.
 func EntriesRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/entries")
 
-	routeGroup.GET("/", controllers.GetEntries)        // get entries (base/thin entries)
-	routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry
 }

agent/pkg/routes/query_routes.go

@@ -0,0 +1,13 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+
+	"github.com/gin-gonic/gin"
+)
+
+func QueryRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/query")
+
+	routeGroup.POST("/validate", controllers.PostValidate)
+}

agent/pkg/sensitiveDataFiltering/consts.go

@@ -1,10 +0,0 @@
-package sensitiveDataFiltering
-
-const maskedFieldPlaceholderValue = "[REDACTED]"
-
-//these values MUST be all lower case and contain no `-` or `_` characters
-var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
-	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
-	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
-	"zip", "zipcode", "address", "country", "firstname", "lastname",
-	"middlename", "fname", "lname", "birthdate"}

agent/pkg/up9/main.go

@@ -7,15 +7,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/google/martian/har"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
@@ -23,6 +24,7 @@ import (
 
 const (
 	AnalyzeCheckSleepTime = 5 * time.Second
+	SentCountLogInterval  = 100
 )
 
 type GuestToken struct {
@@ -110,14 +112,14 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus {
 }
 
 func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
-	logger.Log.Infof("Sync entries - started\n")
+	logger.Log.Infof("Sync entries - started")
 
 	var (
 		token, model string
 		guestMode    bool
 	)
 	if syncEntriesConfig.Token == "" {
-		logger.Log.Infof("Sync entries - creating anonymous token. env %s\n", syncEntriesConfig.Env)
+		logger.Log.Infof("Sync entries - creating anonymous token. env %s", syncEntriesConfig.Env)
 		guestToken, err := createAnonymousToken(syncEntriesConfig.Env)
 		if err != nil {
 			return fmt.Errorf("failed creating anonymous token, err: %v", err)
@@ -131,7 +133,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
 		model = syncEntriesConfig.Workspace
 		guestMode = false
 
-		logger.Log.Infof("Sync entries - upserting model. env %s, model %s\n", syncEntriesConfig.Env, model)
+		logger.Log.Infof("Sync entries - upserting model. env %s, model %s", syncEntriesConfig.Env, model)
 		if err := upsertModel(token, model, syncEntriesConfig.Env); err != nil {
 			return fmt.Errorf("failed upserting model, err: %v", err)
 		}
@@ -142,7 +144,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
 		return fmt.Errorf("invalid model name, model name: %s", model)
 	}
 
-	logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode)
+	logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v", token, model, guestMode)
 	go syncEntriesImpl(token, model, syncEntriesConfig.Env, syncEntriesConfig.UploadIntervalSec, guestMode)
 
 	return nil
@@ -204,51 +206,80 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 	analyzeInformation.AnalyzeDestination = envPrefix
 	analyzeInformation.SentCount = 0
 
-	sleepTime := time.Second * time.Duration(uploadIntervalSec)
+	// "http or grpc" filter indicates that we're only interested in HTTP and gRPC entries
+	query := "http or grpc"
 
-	var timeFrom time.Time
-	protocolFilter := "http"
+	logger.Log.Infof("Getting entries from the database")
 
-	for {
-		timeTo := time.Now()
-		logger.Log.Infof("Getting entries from %v, to %v\n", timeFrom.Format(time.RFC3339Nano), timeTo.Format(time.RFC3339Nano))
-		entriesArray := database.GetEntriesFromDb(timeFrom, timeTo, &protocolFilter)
+	var connection *basenine.Connection
+	var err error
+	connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		panic(err)
+	}
 
-		if len(entriesArray) > 0 {
-			result := make([]har.Entry, 0)
-			for _, data := range entriesArray {
-				var pair tapApi.RequestResponsePair
-				if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
-					continue
-				}
-				harEntry, err := utils.NewEntry(&pair)
-				if err != nil {
-					continue
-				}
-				if data.ResolvedSource != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource})
-				}
-				if data.ResolvedDestination != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination})
-					harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination)
-				}
+	data := make(chan []byte)
+	meta := make(chan []byte)
 
-				// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
-				if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
-					continue
-				}
+	defer func() {
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
+	}()
 
-				result = append(result, *harEntry)
+	lastTimeSynced := time.Time{}
+
+	batch := make([]har.Entry, 0)
+
+	handleDataChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, data chan []byte) {
+		defer wg.Done()
+		for {
+			dataBytes := <-data
+
+			if string(dataBytes) == basenine.CloseChannel {
+				return
 			}
 
-			logger.Log.Infof("About to upload %v entries\n", len(result))
+			var dataMap map[string]interface{}
+			err = json.Unmarshal(dataBytes, &dataMap)
 
-			body, jMarshalErr := json.Marshal(result)
+			var entry tapApi.MizuEntry
+			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
+				continue
+			}
+			harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+			if err != nil {
+				continue
+			}
+			if entry.ResolvedSource != "" {
+				harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
+			}
+			if entry.ResolvedDestination != "" {
+				harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
+				harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.ResolvedDestination)
+			}
+
+			// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
+			if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
+				continue
+			}
+
+			batch = append(batch, *harEntry)
+
+			now := time.Now()
+			if lastTimeSynced.Add(time.Duration(uploadIntervalSec) * time.Second).After(now) {
+				continue
+			}
+			lastTimeSynced = now
+
+			body, jMarshalErr := json.Marshal(batch)
+			batchSize := len(batch)
 			if jMarshalErr != nil {
 				analyzeInformation.Reset()
 				logger.Log.Infof("Stopping sync entries")
 				logger.Log.Fatal(jMarshalErr)
 			}
+			batch = make([]har.Entry, 0)
 
 			var in bytes.Buffer
 			w := zlib.NewWriter(&in)
@@ -273,18 +304,33 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 				logger.Log.Info("Stopping sync entries")
 				logger.Log.Fatal(postErr)
 			}
-			analyzeInformation.SentCount += len(entriesArray)
-			logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
+			analyzeInformation.SentCount += batchSize
 
-			logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
-		} else {
-			logger.Log.Infof("Nothing to upload")
+			if analyzeInformation.SentCount%SentCountLogInterval == 0 {
+				logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
+			}
 		}
-
-		logger.Log.Infof("Sleeping for %v...\n", sleepTime)
-		time.Sleep(sleepTime)
-		timeFrom = timeTo
 	}
+
+	handleMetaChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, meta chan []byte) {
+		defer wg.Done()
+		for {
+			metaBytes := <-meta
+
+			if string(metaBytes) == basenine.CloseChannel {
+				return
+			}
+		}
+	}
+
+	var wg sync.WaitGroup
+	go handleDataChannel(&wg, connection, data)
+	go handleMetaChannel(&wg, connection, meta)
+	wg.Add(2)
+
+	connection.Query(query, data, meta)
+
+	wg.Wait()
 }
 
 func UpdateAnalyzeStatus(callback func(data []byte)) {

agent/pkg/utils/har.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
 )
 
 // Keep it because we might want cookies in the future
@@ -120,13 +119,11 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 	return params
 }
 
-func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) {
-	reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{})
+func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
+	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
-	headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{}))
-
-	postData, _ := reqDetails["postData"].(map[string]interface{})
+	postData, _ := request["postData"].(map[string]interface{})
 	mimeType, _ := postData["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
@@ -138,7 +135,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	}
 
 	queryString := make([]har.QueryString, 0)
-	for _, _qs := range reqDetails["queryString"].([]interface{}) {
+	for _, _qs := range request["_queryString"].([]interface{}) {
 		qs := _qs.(map[string]interface{})
 		queryString = append(queryString, har.QueryString{
 			Name:  qs["name"].(string),
@@ -146,7 +143,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 		})
 	}
 
-	url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string))
+	url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
 		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
 	}
@@ -157,9 +154,9 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	}
 
 	harRequest = &har.Request{
-		Method:      reqDetails["method"].(string),
+		Method:      request["method"].(string),
 		URL:         url,
-		HTTPVersion: reqDetails["httpVersion"].(string),
+		HTTPVersion: request["httpVersion"].(string),
 		HeadersSize: -1,
 		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
 		QueryString: queryString,
@@ -175,13 +172,11 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	return
 }
 
-func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) {
-	resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{})
+func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
+	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
-	headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{}))
-
-	content, _ := resDetails["content"].(map[string]interface{})
+	content, _ := response["content"].(map[string]interface{})
 	mimeType, _ := content["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
@@ -200,9 +195,11 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 		Size:     int64(len(bodyText)),
 	}
 
-	status := int(resDetails["status"].(float64))
+	status := int(response["status"].(float64))
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
-		status, err = strconv.Atoi(_status)
+		if _status != "" {
+			status, err = strconv.Atoi(_status)
+		}
 		if err != nil {
 			logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
 			return nil, errors.New("failed converting response status to int for HAR")
@@ -210,9 +207,9 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	}
 
 	harResponse = &har.Response{
-		HTTPVersion: resDetails["httpVersion"].(string),
+		HTTPVersion: response["httpVersion"].(string),
 		Status:      status,
-		StatusText:  resDetails["statusText"].(string),
+		StatusText:  response["statusText"].(string),
 		HeadersSize: -1,
 		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
 		Headers:     headers,
@@ -222,34 +219,33 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	return
 }
 
-func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
-	harRequest, err := NewRequest(&pair.Request)
+func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
+	harRequest, err := NewRequest(request)
 	if err != nil {
 		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting request to HAR")
 	}
 
-	harResponse, err := NewResponse(&pair.Response)
+	harResponse, err := NewResponse(response)
 	if err != nil {
 		logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting response to HAR")
 	}
 
-	totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	if totalTime < 1 {
-		totalTime = 1
+	if elapsedTime < 1 {
+		elapsedTime = 1
 	}
 
 	harEntry := har.Entry{
-		StartedDateTime: pair.Request.CaptureTime,
-		Time:            totalTime,
+		StartedDateTime: startTime,
+		Time:            elapsedTime,
 		Request:         harRequest,
 		Response:        harResponse,
 		Cache:           &har.Cache{},
 		Timings: &har.Timings{
 			Send:    -1,
 			Wait:    -1,
-			Receive: totalTime,
+			Receive: elapsedTime,
 		},
 	}
 

agent/pkg/utils/truncating_logger.go

@@ -1,60 +0,0 @@
-package utils
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	loggerShared "github.com/up9inc/mizu/shared/logger"
-	"gorm.io/gorm/logger"
-	"gorm.io/gorm/utils"
-)
-
-// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
-type TruncatingLogger struct {
-	LogLevel      logger.LogLevel
-	SlowThreshold time.Duration
-}
-
-func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface {
-	truncatingLogger.LogLevel = logLevel
-	return truncatingLogger
-}
-
-func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Info {
-		return
-	}
-	loggerShared.Log.Errorf("gorm info: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Warn {
-		return
-	}
-	loggerShared.Log.Errorf("gorm warning: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Error {
-		return
-	}
-	loggerShared.Log.Errorf("gorm error: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
-	if truncatingLogger.LogLevel == logger.Silent {
-		return
-	}
-	elapsed := time.Since(begin)
-	if err != nil {
-		sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary
-		truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql))
-	} else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold {
-		sql, rows := fc()
-		truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
-	} else if truncatingLogger.LogLevel >= logger.Info {
-		sql, rows := fc()
-		truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
-	}
-}

cli/cmd/common.go

@@ -4,9 +4,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"os"
 	"os/signal"
+	"path"
 	"syscall"
+	"time"
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
@@ -64,3 +70,42 @@ func handleKubernetesProviderError(err error) {
 		logger.Log.Error(err)
 	}
 }
+
+func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) {
+	telemetry.ReportAPICalls(apiProvider)
+	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
+	defer cancel()
+	dumpLogsIfNeeded(removalCtx, kubernetesProvider)
+	cleanUpMizuResources(removalCtx, cancel, kubernetesProvider)
+}
+
+func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
+	if !config.Config.DumpLogs {
+		return
+	}
+	mizuDir := mizu.GetMizuFolderPath()
+	filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
+	if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
+		logger.Log.Errorf("Failed dump logs %v", err)
+	}
+}
+
+func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
+	logger.Log.Infof("\nRemoving mizu resources")
+
+	var leftoverResources []string
+
+	if config.Config.IsNsRestrictedMode() {
+		leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider)
+	} else {
+		leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider)
+	}
+
+	if len(leftoverResources) > 0 {
+		errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath())
+		for _, resource := range leftoverResources {
+			errMsg += "\n- " + resource
+		}
+		logger.Log.Errorf(uiUtils.Error, errMsg)
+	}
+}

cli/cmd/root.go

@@ -23,6 +23,7 @@ Further info is available at https://github.com/up9inc/mizu`,
 		if err := config.InitConfig(cmd); err != nil {
 			logger.Log.Fatal(err)
 		}
+
 		return nil
 	},
 }

cli/cmd/tap.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/cli/up9"
 	"os"
 
 	"github.com/creasty/defaults"
@@ -62,6 +63,12 @@ Supported protocols are HTTP and gRPC.`,
 						logger.Log.Errorf("failed to log in, err: %v", err)
 						return nil
 					}
+				} else if isValidToken := up9.IsTokenValid(config.Config.Auth.Token, config.Config.Auth.EnvName); !isValidToken {
+					logger.Log.Errorf("Token is not valid, please log in again to continue")
+					if err := auth.Login(); err != nil {
+						logger.Log.Errorf("failed to log in, err: %v", err)
+						return nil
+					}
 				}
 			}
 		}
@@ -113,4 +120,5 @@ func init() {
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
 	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
 	tapCmd.Flags().Bool(configStructs.DaemonModeTapName, defaultTapConfig.DaemonMode, "Run mizu in daemon mode, detached from the cli")
+	tapCmd.Flags().Bool(configStructs.IstioName, defaultTapConfig.Istio, "Record decrypted traffic if the cluster configured with istio and mtls")
 }

cli/cmd/tapRunner.go

@@ -4,27 +4,25 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/cli/cmd/goUtils"
 	"io/ioutil"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"path"
 	"regexp"
 	"strings"
 	"time"
 
 	"github.com/getkin/kin-openapi/openapi3"
+	"gopkg.in/yaml.v3"
+	core "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
 	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/cmd/goUtils"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"gopkg.in/yaml.v3"
-	core "k8s.io/api/core/v1"
-
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -118,6 +116,9 @@ func RunMizuTap() {
 	logger.Log.Infof("Tapping pods in %s", namespacesStr)
 
 	if config.Config.Tap.DryRun {
+		if err := printTappedPodsPreview(ctx, kubernetesProvider, targetNamespaces); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error listing pods: %v", errormessage.FormatError(err)))
+		}
 		return
 	}
 
@@ -133,7 +134,7 @@ func RunMizuTap() {
 		return
 	}
 	if config.Config.Tap.DaemonMode {
-		if err := handleDaemonModePostCreation(cancel, kubernetesProvider); err != nil {
+		if err := handleDaemonModePostCreation(ctx, cancel, kubernetesProvider, targetNamespaces); err != nil {
 			defer finishMizuExecution(kubernetesProvider, apiProvider)
 			cancel()
 		} else {
@@ -155,28 +156,38 @@ func RunMizuTap() {
 	}
 }
 
-func handleDaemonModePostCreation(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) error {
+func handleDaemonModePostCreation(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, namespaces []string) error {
+	if err := printTappedPodsPreview(ctx, kubernetesProvider, namespaces); err != nil {
+		return err
+	}
+
 	apiProvider := apiserver.NewProvider(GetApiServerUrl(), 90, 1*time.Second)
 
 	if err := waitForDaemonModeToBeReady(cancel, kubernetesProvider, apiProvider); err != nil {
 		return err
 	}
-	if err := printDaemonModeTappedPods(apiProvider); err != nil {
-		return err
-	}
 
 	return nil
 }
 
-func printDaemonModeTappedPods(apiProvider *apiserver.Provider) error {
-	if healthStatus, err := apiProvider.GetHealthStatus(); err != nil {
+/*
+this function is a bit problematic as it might be detached from the actual pods the mizu api server will tap.
+The alternative would be to wait for api server to be ready and then query it for the pods it listens to, this has
+the arguably worse drawback of taking a relatively very long time before the user sees which pods are targeted, if any.
+*/
+func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.Provider, namespaces []string) error {
+	if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), namespaces); err != nil {
 		return err
 	} else {
-		for _, tappedPod := range healthStatus.TapStatus.Pods {
+		if len(matchingPods) == 0 {
+			printNoPodsFoundSuggestion(namespaces)
+		}
+		logger.Log.Info("Pods that match the provided criteria at this instant:")
+		for _, tappedPod := range matchingPods {
 			logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", tappedPod.Name))
 		}
+		return nil
 	}
-	return nil
 }
 
 func waitForDaemonModeToBeReady(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) error {
@@ -199,10 +210,11 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 		AgentImage:               config.Config.AgentImage,
 		TapperResources:          config.Config.Tap.TapperResources,
 		ImagePullPolicy:          config.Config.ImagePullPolicy(),
-		DumpLogs:                 config.Config.DumpLogs,
+		LogLevel:                 config.Config.LogLevel(),
 		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
 		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
+		Istio:                    config.Config.Tap.Istio,
 	})
 
 	if err != nil {
@@ -214,11 +226,7 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 	}
 
 	if len(tapperSyncer.CurrentlyTappedPods) == 0 {
-		var suggestionStr string
-		if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
-			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
-		}
-		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
+		printNoPodsFoundSuggestion(targetNamespaces)
 	}
 
 	go func() {
@@ -251,6 +259,14 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 	return nil
 }
 
+func printNoPodsFoundSuggestion(targetNamespaces []string) {
+	var suggestionStr string
+	if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
+		suggestionStr = ". You can also try selecting a different namespace with -n or tap all namespaces with -A"
+	}
+	logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any currently running pods that match the regex argument, mizu will automatically tap matching pods if any are created later%s", suggestionStr))
+}
+
 func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError) string {
 	switch err.TapManagerReason {
 	case kubernetes.TapManagerPodListError:
@@ -281,7 +297,7 @@ func createMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne
 	}
 
 	if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
-		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err)))
 	}
 
 	var err error
@@ -309,6 +325,7 @@ func createMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne
 		MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(),
 		Resources:             config.Config.Tap.ApiServerResources,
 		ImagePullPolicy:       config.Config.ImagePullPolicy(),
+		LogLevel:              config.Config.LogLevel(),
 	}
 
 	if config.Config.Tap.DaemonMode {
@@ -357,20 +374,9 @@ func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.
 }
 
 func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error {
-	isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx)
 	volumeClaimCreated := false
-	if err != nil {
-		return err
-	}
-	if isDefaultStorageClassAvailable {
-		if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName, config.Config.Tap.MaxEntriesDBSizeBytes()+mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil {
-			logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this will mean that mizu's data will be lost on pod restart")
-			logger.Log.Debugf("error creating persistent volume claim: %v", err)
-		} else {
-			volumeClaimCreated = true
-		}
-	} else {
-		logger.Log.Warningf(uiUtils.Yellow, "Could not find default volume provider in this cluster, this will mean that mizu's data will be lost on pod restart")
+	if !config.Config.Tap.NoPersistentVolumeClaim {
+		volumeClaimCreated = TryToCreatePersistentVolumeClaim(ctx, kubernetesProvider)
 	}
 
 	pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName)
@@ -385,6 +391,26 @@ func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kube
 	return nil
 }
 
+func TryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
+	isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx)
+	if err != nil {
+		logger.Log.Warningf(uiUtils.Yellow, "An error occured when checking if a default storage provider exists in this cluster, this means mizu data will be lost on mizu-api-server pod restart")
+		logger.Log.Debugf("error checking if default storage class exists: %v", err)
+		return false
+	} else if !isDefaultStorageClassAvailable {
+		logger.Log.Warningf(uiUtils.Yellow, "Could not find default storage provider in this cluster, this means mizu data will be lost on mizu-api-server pod restart")
+		return false
+	}
+
+	if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName, config.Config.Tap.MaxEntriesDBSizeBytes()+mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil {
+		logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this means mizu data will be lost on mizu-api-server pod restart")
+		logger.Log.Debugf("error creating persistent volume claim: %v", err)
+		return false
+	}
+
+	return true
+}
+
 func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
 	var compiledRegexSlice []*api.SerializableRegexp
 
@@ -419,45 +445,6 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig {
 	}
 }
 
-func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) {
-	telemetry.ReportAPICalls(apiProvider)
-	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
-	defer cancel()
-	dumpLogsIfNeeded(removalCtx, kubernetesProvider)
-	cleanUpMizuResources(removalCtx, cancel, kubernetesProvider)
-}
-
-func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
-	if !config.Config.DumpLogs {
-		return
-	}
-	mizuDir := mizu.GetMizuFolderPath()
-	filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
-	if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
-		logger.Log.Errorf("Failed dump logs %v", err)
-	}
-}
-
-func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
-	logger.Log.Infof("\nRemoving mizu resources\n")
-
-	var leftoverResources []string
-
-	if config.Config.IsNsRestrictedMode() {
-		leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider)
-	} else {
-		leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider)
-	}
-
-	if len(leftoverResources) > 0 {
-		errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath())
-		for _, resource := range leftoverResources {
-			errMsg += "\n- " + resource
-		}
-		logger.Log.Errorf(uiUtils.Error, errMsg)
-	}
-}
-
 func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) []string {
 	leftoverResources := make([]string, 0)
 
@@ -568,7 +555,8 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k
 
 func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
+	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 	isPodReady := false
 	timeAfter := time.After(25 * time.Second)
 	for {
@@ -589,12 +577,19 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 			logger.Log.Infof("%s removed", kubernetes.ApiServerPodName)
 			cancel()
 			return
-		case modifiedPod, ok := <-modified:
+		case wEvent, ok := <-modified:
 			if !ok {
 				modified = nil
 				continue
 			}
 
+			modifiedPod, err := wEvent.ToPod()
+			if err != nil {
+				logger.Log.Errorf(uiUtils.Error, err)
+				cancel()
+				continue
+			}
+
 			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
 
 			if modifiedPod.Status.Phase == core.PodPending {
@@ -624,8 +619,10 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 					break
 				}
 
-				logger.Log.Infof("Mizu is available at %s\n", url)
-				uiUtils.OpenBrowser(url)
+				logger.Log.Infof("Mizu is available at %s", url)
+				if !config.Config.HeadlessMode {
+					uiUtils.OpenBrowser(url)
+				}
 				if err := apiProvider.ReportTappedPods(state.tapperSyncer.CurrentlyTappedPods); err != nil {
 					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
 				}
@@ -653,34 +650,57 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 
 func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.TapperDaemonSetName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
+	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 	var prevPodPhase core.PodPhase
 	for {
 		select {
-		case addedPod, ok := <-added:
+		case wEvent, ok := <-added:
 			if !ok {
 				added = nil
 				continue
 			}
 
+			addedPod, err := wEvent.ToPod()
+			if err != nil {
+				logger.Log.Errorf(uiUtils.Error, err)
+				cancel()
+				continue
+			}
+
 			logger.Log.Debugf("Tapper is created [%s]", addedPod.Name)
-		case removedPod, ok := <-removed:
+		case wEvent, ok := <-removed:
 			if !ok {
 				removed = nil
 				continue
 			}
 
+			removedPod, err := wEvent.ToPod()
+			if err != nil {
+				logger.Log.Errorf(uiUtils.Error, err)
+				cancel()
+				continue
+			}
+
+
 			logger.Log.Debugf("Tapper is removed [%s]", removedPod.Name)
-		case modifiedPod, ok := <-modified:
+		case wEvent, ok := <-modified:
 			if !ok {
 				modified = nil
 				continue
 			}
 
+			modifiedPod, err := wEvent.ToPod()
+			if err != nil {
+				logger.Log.Errorf(uiUtils.Error, err)
+				cancel()
+				continue
+			}
+
 			if modifiedPod.Status.Phase == core.PodPending && modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
 				logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Wasn't able to deploy the tapper %s. Reason: \"%s\"", modifiedPod.Name, modifiedPod.Status.Conditions[0].Message))
 				cancel()
-				break
+				continue
 			}
 
 			podStatus := modifiedPod.Status

cli/cmd/viewRunner.go

@@ -56,9 +56,11 @@ func runMizuView() {
 		return
 	}
 
-	logger.Log.Infof("Mizu is available at %s\n", url)
+	logger.Log.Infof("Mizu is available at %s", url)
 
-	uiUtils.OpenBrowser(url)
+	if !config.Config.HeadlessMode {
+		uiUtils.OpenBrowser(url)
+	}
 
 	if isCompatible, err := version.CheckVersionCompatibility(apiServerProvider); err != nil {
 		logger.Log.Errorf("Failed to check versions compatibility %v", err)

cli/config/config.go

@@ -3,14 +3,15 @@ package config
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
-	"k8s.io/apimachinery/pkg/util/json"
 	"os"
 	"reflect"
 	"strconv"
 	"strings"
 
+	"github.com/up9inc/mizu/tap/api"
+	"k8s.io/apimachinery/pkg/util/json"
+
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 
@@ -51,6 +52,10 @@ func InitConfig(cmd *cobra.Command) error {
 
 	cmd.Flags().Visit(initFlag)
 
+	if err := Config.validate(); err != nil {
+		return fmt.Errorf("config validation failed, err: %v", err)
+	}
+
 	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
 	logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
 
@@ -391,12 +396,13 @@ func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.
 		TargetNamespaces:        targetNamespaces,
 		AgentImage:              Config.AgentImage,
 		PullPolicy:              Config.ImagePullPolicyStr,
-		DumpLogs:                Config.DumpLogs,
+		LogLevel:                Config.LogLevel(),
 		IgnoredUserAgents:       Config.Tap.IgnoredUserAgents,
 		TapperResources:         Config.Tap.TapperResources,
 		MizuResourcesNamespace:  Config.MizuResourcesNamespace,
 		MizuApiFilteringOptions: *mizuApiFilteringOptions,
-		AgentDatabasePath:       fmt.Sprintf("%s%s", shared.DataDirPath, "entries.db"),
+		AgentDatabasePath:       shared.DataDirPath,
+		Istio:                   Config.Tap.Istio,
 	}
 	return &config, nil
 }

cli/config/configStruct.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/mizu"
 	v1 "k8s.io/api/core/v1"
@@ -31,6 +32,16 @@ type ConfigStruct struct {
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
 	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
+	HeadlessMode           bool                        `yaml:"headless" default:"false"`
+	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
+}
+
+func(config *ConfigStruct) validate() error {
+	if _, err := logging.LogLevel(config.LogLevelStr); err != nil {
+		return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err)
+	}
+
+	return nil
 }
 
 func (config *ConfigStruct) SetDefaults() {
@@ -59,3 +70,8 @@ func (config *ConfigStruct) KubeConfigPath() string {
 	home := homedir.HomeDir()
 	return filepath.Join(home, ".kube", "config")
 }
+
+func (config *ConfigStruct) LogLevel() logging.Level {
+	logLevel, _ := logging.LogLevel(config.LogLevelStr)
+	return logLevel
+}

cli/config/configStructs/tapConfig.go

@@ -3,9 +3,12 @@ package configStructs
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared/logger"
 	"regexp"
 
+	"github.com/up9inc/mizu/shared"
+
 	"github.com/up9inc/mizu/shared/units"
 )
 
@@ -22,28 +25,31 @@ const (
 	EnforcePolicyFile             = "traffic-validation-file"
 	ContractFile                  = "contract"
 	DaemonModeTapName             = "daemon"
+	IstioName                     = "istio"
 )
 
 type TapConfig struct {
-	UploadIntervalSec      int              `yaml:"upload-interval" default:"10"`
-	PodRegexStr            string           `yaml:"regex" default:".*"`
-	GuiPort                uint16           `yaml:"gui-port" default:"8899"`
-	ProxyHost              string           `yaml:"proxy-host" default:"127.0.0.1"`
-	Namespaces             []string         `yaml:"namespaces"`
-	Analysis               bool             `yaml:"analysis" default:"false"`
-	AllNamespaces          bool             `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string         `yaml:"regex-masking"`
-	IgnoredUserAgents      []string         `yaml:"ignored-user-agents"`
-	DisableRedaction       bool             `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
-	DryRun                 bool             `yaml:"dry-run" default:"false"`
-	Workspace              string           `yaml:"workspace"`
-	EnforcePolicyFile      string           `yaml:"traffic-validation-file"`
-	ContractFile           string           `yaml:"contract"`
-	AskUploadConfirmation  bool             `yaml:"ask-upload-confirmation" default:"true"`
-	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
-	TapperResources        shared.Resources `yaml:"tapper-resources"`
-	DaemonMode             bool             `yaml:"daemon" default:"false"`
+	UploadIntervalSec       int              `yaml:"upload-interval" default:"10"`
+	PodRegexStr             string           `yaml:"regex" default:".*"`
+	GuiPort                 uint16           `yaml:"gui-port" default:"8899"`
+	ProxyHost               string           `yaml:"proxy-host" default:"127.0.0.1"`
+	Namespaces              []string         `yaml:"namespaces"`
+	Analysis                bool             `yaml:"analysis" default:"false"`
+	AllNamespaces           bool             `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes  []string         `yaml:"regex-masking"`
+	IgnoredUserAgents       []string         `yaml:"ignored-user-agents"`
+	DisableRedaction        bool             `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize   string           `yaml:"max-entries-db-size" default:"200MB"`
+	DryRun                  bool             `yaml:"dry-run" default:"false"`
+	Workspace               string           `yaml:"workspace"`
+	EnforcePolicyFile       string           `yaml:"traffic-validation-file"`
+	ContractFile            string           `yaml:"contract"`
+	AskUploadConfirmation   bool             `yaml:"ask-upload-confirmation" default:"true"`
+	ApiServerResources      shared.Resources `yaml:"api-server-resources"`
+	TapperResources         shared.Resources `yaml:"tapper-resources"`
+	DaemonMode              bool             `yaml:"daemon" default:"false"`
+	NoPersistentVolumeClaim bool             `yaml:"no-persistent-volume-claim" default:"false"`
+	Istio                   bool             `yaml:"istio" default:"false"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -78,5 +84,9 @@ func (config *TapConfig) Validate() error {
 		return errors.New(fmt.Sprintf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName))
 	}
 
+	if config.NoPersistentVolumeClaim && !config.DaemonMode {
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("the --set tap.no-persistent-volume-claim=true flag has no effect without the --%s flag, the claim will not be created anyway.", DaemonModeTapName))
+	}
+
 	return nil
 }

cli/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/getkin/kin-openapi v0.79.0
 	github.com/google/go-github/v37 v37.0.0
 	github.com/google/uuid v1.1.2
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -78,6 +78,6 @@ func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath strin
 		logger.Log.Debugf("Successfully added file %s", GetLogFilePath())
 	}
 
-	logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath)
+	logger.Log.Infof("You can find the zip file with all logs in %s", filePath)
 	return nil
 }

cli/up9/provider.go

@@ -0,0 +1,31 @@
+package up9
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+)
+
+func IsTokenValid(tokenString string, envName string) bool {
+	whoAmIUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/admin/whoami", envName))
+
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    whoAmIUrl,
+		Header: map[string][]string{
+			"Authorization": {fmt.Sprintf("bearer %s", tokenString)},
+		},
+	}
+
+	response, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return false
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return false
+	}
+
+	return true
+}

debug.Dockerfile

@@ -12,7 +12,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash
+RUN apk add libpcap-dev gcc g++ make bash perl-utils
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -23,7 +23,7 @@ COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
+RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 ARG COMMIT_HASH
 ARG GIT_BRANCH
@@ -36,17 +36,25 @@ COPY tap ../tap
 COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
+# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.10/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.10/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+RUN shasum -a 256 -c basenine_linux_amd64.sha256
+RUN chmod +x ./basenine_linux_amd64
+
 COPY devops/build_extensions_debug.sh ..
 RUN cd .. && /bin/bash build_extensions_debug.sh
 
 
 FROM golang:1.16-alpine
 
-RUN apk add bash libpcap-dev tcpdump
+RUN apk add bash libpcap-dev
+
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 

docs/DAEMON_MODE.md

@@ -0,0 +1,80 @@
+# Mizu daemon mode
+
+Mizu can be run detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run
+indefinitely in the cluster.
+
+Please note that daemon mode requires you to have RBAC creation permissions, see the [permissions](PERMISSIONS.md)
+doc for more details.
+
+```bash
+$ mizu tap "^ca.*" --daemon
+  Mizu will store up to 200MB of traffic, old traffic will be cleared once the limit is reached.
+  Tapping pods in namespaces "sock-shop"
+  Waiting for mizu to be ready... (may take a few minutes)
+  +carts-66c77f5fbb-fq65r
+  +catalogue-5f4cb7cf5-7zrmn
+  ..
+```
+
+## Stop mizu daemon
+
+To stop the detached mizu instance and clean all cluster side resources, run `mizu clean`
+
+```bash
+$ mizu clean # mizu will continue running in cluster until clean is executed
+  Removing mizu resources
+```
+
+## Expose mizu web app
+
+Mizu could be exposed at a later stage in any of the following ways:
+
+### Using mizu view command
+
+In a machine that can access both the cluster and a browser, you can run `mizu view` command which creates a proxy.
+Besides that, all the regular ways to expose k8s pods are valid.
+
+```bash
+$ mizu view
+  Establishing connection to k8s cluster...
+  Mizu is available at http://localhost:8899
+  ^C
+  ..
+```
+
+### Port Forward
+
+```bash
+$ kubectl port-forward -n mizu deployment/mizu-api-server 8899:8899
+```
+
+### NodePort
+
+```bash
+$ kubectl expose -n mizu deployment mizu-api-server --name mizu-node-port --type NodePort --port 80 --target-port 8899
+```
+
+Mizu's IP is the IP of any node (get the IP with `kubectl get nodes -o wide`) and the port is the target port of the new
+service (`kubectl get services -n mizu mizu-node-port`). Note that this method will expose Mizu to public access if your
+nodes are public.
+
+### LoadBalancer
+
+```bash
+$ kubectl expose deployment -n mizu --port 80 --target-port 8899 mizu-api-server --type=LoadBalancer --name=mizu-lb
+  service/mizu-lb exposed
+  ..
+  
+$ kubectl get services -n mizu
+  NAME              TYPE           CLUSTER-IP       EXTERNAL-IP     PORT(S)        AGE
+  mizu-api-server   ClusterIP      10.107.200.100   <none>          80/TCP         5m5s
+  mizu-lb           LoadBalancer   10.107.200.101   34.77.120.116   80:30141/TCP   76s
+```
+
+Note that `LoadBalancer` services only work on supported clusters (usually cloud providers) and might incur extra costs
+
+If you changed the `mizu-resources-namespace` value, make sure the `-n mizu` flag of the `kubectl expose` command is
+changed to the value of `mizu-resources-namespace`
+
+mizu will now be available both by running `mizu view` or by accessing the `EXTERNAL-IP` of the `mizu-lb` service
+through your browser.

docs/ISTIO.md

@@ -0,0 +1,46 @@
+![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
+# Istio mutual tls (mtls) with Mizu
+This document describe how Mizu tapper handles workloads configured with mtls, making the internal traffic between services in a cluster to be encrypted.
+
+Besides Istio there are other service meshes that implement mtls. However, as of now Istio is the most used one, and this is why we are focusing on it.
+
+In order to create an Istio setup for development, follow those steps:
+1. Deploy a sample application to a Kubernetes cluster, the sample application needs to make internal service to service calls
+2. SSH to one of the nodes, and run `tcpdump`
+3. Make sure you see the internal service to service calls in a plain text
+4. Deploy Istio to the cluster - make sure it is attached to all pods of the sample application, and that it is configured with mtls (default)
+5. Run `tcpdump` again, make sure you don't see the internal service to service calls in a plain text 
+
+## The connection between Istio and Envoy
+In order to implement its service mesh capabilities, [Istio](https://istio.io) use an [Envoy](https://www.envoyproxy.io) sidecar in front of every pod in the cluster. The Envoy is responsible for the mtls communication, and that's why we are focusing on Envoy proxy.
+
+In the future we might see more players in that field, then we'll have to either add support for each of them or go with a unified eBPF solution.
+
+## Network namespaces
+A [linux network namespace](https://man7.org/linux/man-pages/man7/network_namespaces.7.html) is an isolation that limit the process view of the network. In the container world it used to isolate one container from another. In the Kubernetes world it used to isolate a pod from another. That means that two containers running on the same pod share the same network namespace. A container can reach a container in the same pod by accessing `localhost`.
+
+An Envoy proxy configured with mtls receives the inbound traffic directed to the pod, decrypts it and sends it via `localhost` to the target container.
+
+## Tapping mtls traffic
+In order for Mizu to be able to see the decrypted traffic it needs to listen on the same network namespace of the target pod. Multiple threads of the same process can have different network namespaces. 
+
+[gopacket](https://github.com/google/gopacket) uses [libpacp](https://github.com/the-tcpdump-group/libpcap) by default for capturing the traffic. Libpacap doesn't support network namespaces and we can't ask it to listen to traffic on a different namespace. However, we can change the network namespace of the calling thread and then start libpcap to see the traffic on a different namespace.
+
+## Finding the network namespace of a running process
+The network namespace of a running process can be found in `/proc/PID/ns/net` link. Once we have this link, we can ask Linux to change the network namespace of a thread to this one.
+
+This mean that Mizu needs to have access to the `/proc` (procfs) of the running node.
+
+## Finding the network namespace of a running pod
+In order for Mizu to be able to listen to mtls traffic, it needs to get the PIDs of the the running pods, filter them according to the user filters and then start listen to their internal network namespace traffic.
+
+There is no official way in Kubernetes to get from pod to PID. The CRI implementation purposefully doesn't force a pod to be a processes on the host. It can be a Virtual Machine as well like [Kata containers](https://katacontainers.io)
+
+While we can provide a solution for various CRIs (like Docker, Containerd and CRI-O) it's better to have a unified solution. In order to achieve that, Mizu scans all the processes in the host, and finds the Envoy processes using their `/proc/PID/exe` link.
+
+Once Mizu detects an Envoy process, it need to check whether this specific Envoy process is relevant according the user filters. The user filters are a list of `CLUSTER_IPS`. The tapper gets them via the `TapOpts.FilterAuthorities` list.
+
+Istio sends an `INSTANCE_IP` environment variable to every Envoy proxy process. By examining the Envoy process's environment variables we can see whether it's relevant or not. Examining a process environment variables is done by reading the `/proc/PID/envion` file.
+
+## Edge cases
+The method we use to find Envoy processes and correlate them to the cluster IPs may be inaccurate in certain situations. If, for example, a user runs an Envoy process manually, and set its `INSTANCE_IP` environment variable to one of the `CLUSTER_IPS` the tapper gets, then Mizu will capture traffic for it.

examples/roles/permissions-all-namespaces-daemon.yaml

@@ -7,15 +7,15 @@ rules:
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["get", "list", "watch", "delete"]
-  - apiGroups: [ "" ]
+  - apiGroups: [ "apps" ]
     resources: [ "deployments" ]
-    verbs: [ "create", "delete" ]
+    verbs: [ "get", "create", "delete" ]
   - apiGroups: [""]
     resources: ["services"]
     verbs: ["get", "list", "watch", "create", "delete"]
   - apiGroups: ["apps"]
     resources: ["daemonsets"]
-    verbs: ["create", "patch", "delete"]
+    verbs: ["get", "create", "patch", "delete", "list"]
   - apiGroups: [""]
     resources: ["namespaces"]
     verbs: ["get", "list", "watch", "create", "delete"]

examples/roles/permissions-ns-daemon.yaml

@@ -8,7 +8,7 @@ rules:
 - apiGroups: [""]
   resources: ["pods"]
   verbs: ["get", "list", "watch", "delete"]
-- apiGroups: [ "" ]
+- apiGroups: [ "apps" ]
   resources: [ "deployments" ]
   verbs: [ "get", "create", "delete" ]
 - apiGroups: [""]
@@ -16,7 +16,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "delete"]
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
-  verbs: ["get", "create", "patch", "delete"]
+  verbs: ["get", "create", "patch", "delete", "list"]
 - apiGroups: [""]
   resources: ["services/proxy"]
   verbs: ["get"]
@@ -32,7 +32,7 @@ rules:
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["rolebindings"]
   verbs: ["get", "create", "delete"]
-- apiGroups: ["apps", "extensions"]
+- apiGroups: ["apps", "extensions", ""]
   resources: ["pods"]
   verbs: ["get", "list", "watch"]
 - apiGroups: ["apps", "extensions"]

shared/consts.go

@@ -13,5 +13,7 @@ const (
 	ConfigFileName                   = "mizu-config.json"
 	GoGCEnvVar                       = "GOGC"
 	DefaultApiServerPort             = 8899
-	DebugModeEnvVar                  = "MIZU_DEBUG"
+	LogLevelEnvVar                   = "LOG_LEVEL"
+	BasenineHost                     = "localhost"
+	BaseninePort                     = "9099"
 )

shared/kubernetes/mizuTapperSyncer.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"context"
 	"fmt"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
@@ -15,18 +16,20 @@ import (
 const updateTappersDelay = 5 * time.Second
 
 type TappedPodChangeEvent struct {
-	Added   []core.Pod
-	Removed []core.Pod
+	Added                []core.Pod
+	Removed              []core.Pod
+	ExpectedTapperAmount int
 }
 
 // MizuTapperSyncer uses a k8s pod watch to update tapper daemonsets when targeted pods are removed or created
 type MizuTapperSyncer struct {
-	context             context.Context
-	CurrentlyTappedPods []core.Pod
-	config              TapperSyncerConfig
-	kubernetesProvider  *Provider
-	TapPodChangesOut    chan TappedPodChangeEvent
-	ErrorOut            chan K8sTapManagerError
+	context              context.Context
+	CurrentlyTappedPods  []core.Pod
+	config               TapperSyncerConfig
+	kubernetesProvider   *Provider
+	TapPodChangesOut     chan TappedPodChangeEvent
+	ErrorOut             chan K8sTapManagerError
+	nodeToTappedPodIPMap map[string][]string
 }
 
 type TapperSyncerConfig struct {
@@ -36,10 +39,11 @@ type TapperSyncerConfig struct {
 	AgentImage               string
 	TapperResources          shared.Resources
 	ImagePullPolicy          core.PullPolicy
-	DumpLogs                 bool
+	LogLevel                 logging.Level
 	IgnoredUserAgents        []string
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
+	Istio                    bool
 }
 
 func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig) (*MizuTapperSyncer, error) {
@@ -65,7 +69,8 @@ func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Pro
 }
 
 func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
-	added, modified, removed, errorChan := FilteredWatch(tapperSyncer.context, tapperSyncer.kubernetesProvider, tapperSyncer.config.TargetNamespaces, &tapperSyncer.config.PodFilterRegex)
+	podWatchHelper := NewPodWatchHelper(tapperSyncer.kubernetesProvider, &tapperSyncer.config.PodFilterRegex)
+	added, modified, removed, errorChan := FilteredWatch(tapperSyncer.context, podWatchHelper, tapperSyncer.config.TargetNamespaces, podWatchHelper)
 
 	restartTappers := func() {
 		err, changeFound := tapperSyncer.updateCurrentlyTappedPods()
@@ -91,28 +96,48 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
 
 	for {
 		select {
-		case pod, ok := <-added:
+		case wEvent, ok := <-added:
 			if !ok {
 				added = nil
 				continue
 			}
 
+			pod, err := wEvent.ToPod()
+			if err != nil {
+				tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer)
+				continue
+			}
+
+
 			logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-		case pod, ok := <-removed:
+		case wEvent, ok := <-removed:
 			if !ok {
 				removed = nil
 				continue
 			}
 
+			pod, err := wEvent.ToPod()
+			if err != nil {
+				tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer)
+				continue
+			}
+
 			logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-		case pod, ok := <-modified:
+		case wEvent, ok := <-modified:
 			if !ok {
 				modified = nil
 				continue
 			}
 
+			pod, err := wEvent.ToPod()
+			if err != nil {
+				tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer)
+				continue
+			}
+
+
 			logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
@@ -129,12 +154,8 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
 				continue
 			}
 
-			logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
-			restartTappersDebouncer.Cancel()
-			tapperSyncer.ErrorOut <- K8sTapManagerError{
-				OriginalError:    err,
-				TapManagerReason: TapManagerPodWatchError,
-			}
+			tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer)
+			continue
 
 		case <-tapperSyncer.context.Done():
 			logger.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`")
@@ -145,6 +166,15 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
 	}
 }
 
+func (tapperSyncer *MizuTapperSyncer) handleErrorInWatchLoop(err error, restartTappersDebouncer *debounce.Debouncer) {
+	logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
+	restartTappersDebouncer.Cancel()
+	tapperSyncer.ErrorOut <- K8sTapManagerError{
+		OriginalError:    err,
+		TapManagerReason: TapManagerPodWatchError,
+	}
+}
+
 func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, changesFound bool) {
 	if matchingPods, err := tapperSyncer.kubernetesProvider.ListAllRunningPodsMatchingRegex(tapperSyncer.context, &tapperSyncer.config.PodFilterRegex, tapperSyncer.config.TargetNamespaces); err != nil {
 		return err, false
@@ -159,9 +189,11 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
 		}
 		if len(addedPods) > 0 || len(removedPods) > 0 {
 			tapperSyncer.CurrentlyTappedPods = podsToTap
+			tapperSyncer.nodeToTappedPodIPMap = GetNodeHostToTappedPodIpsMap(tapperSyncer.CurrentlyTappedPods)
 			tapperSyncer.TapPodChangesOut <- TappedPodChangeEvent{
-				Added:   addedPods,
-				Removed: removedPods,
+				Added:                addedPods,
+				Removed:              removedPods,
+				ExpectedTapperAmount: len(tapperSyncer.nodeToTappedPodIPMap),
 			}
 			return nil, true
 		}
@@ -170,9 +202,7 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
 }
 
 func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
-	nodeToTappedPodIPMap := GetNodeHostToTappedPodIpsMap(tapperSyncer.CurrentlyTappedPods)
-
-	if len(nodeToTappedPodIPMap) > 0 {
+	if len(tapperSyncer.nodeToTappedPodIPMap) > 0 {
 		var serviceAccountName string
 		if tapperSyncer.config.MizuServiceAccountExists {
 			serviceAccountName = ServiceAccountName
@@ -187,16 +217,17 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
 			tapperSyncer.config.AgentImage,
 			TapperPodName,
 			fmt.Sprintf("%s.%s.svc.cluster.local", ApiServerPodName, tapperSyncer.config.MizuResourcesNamespace),
-			nodeToTappedPodIPMap,
+			tapperSyncer.nodeToTappedPodIPMap,
 			serviceAccountName,
 			tapperSyncer.config.TapperResources,
 			tapperSyncer.config.ImagePullPolicy,
 			tapperSyncer.config.MizuApiFilteringOptions,
-			tapperSyncer.config.DumpLogs,
+			tapperSyncer.config.LogLevel,
+			tapperSyncer.config.Istio,
 		); err != nil {
 			return err
 		}
-		logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
+		logger.Log.Debugf("Successfully created %v tappers", len(tapperSyncer.nodeToTappedPodIPMap))
 	} else {
 		if err := tapperSyncer.kubernetesProvider.RemoveDaemonSet(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, TapperDaemonSetName); err != nil {
 			return err

shared/kubernetes/podWatchHelper.go

@@ -0,0 +1,45 @@
+package kubernetes
+
+import (
+	"context"
+	"regexp"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+type PodWatchHelper struct {
+	kubernetesProvider *Provider
+	NameRegexFilter    *regexp.Regexp
+}
+
+func NewPodWatchHelper(kubernetesProvider *Provider, NameRegexFilter *regexp.Regexp) *PodWatchHelper {
+	return &PodWatchHelper{
+		kubernetesProvider: kubernetesProvider,
+		NameRegexFilter: NameRegexFilter,
+	}
+}
+
+// Implements the EventFilterer Interface
+func (pwh *PodWatchHelper) Filter(wEvent *WatchEvent) (bool, error) {
+	pod, err := wEvent.ToPod()
+	if err != nil {
+		return false, nil
+	}
+
+	if !pwh.NameRegexFilter.MatchString(pod.Name) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// Implements the WatchCreator Interface
+func (pwh *PodWatchHelper) NewWatcher(ctx context.Context, namespace string) (watch.Interface, error) {
+	watcher, err := pwh.kubernetesProvider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
+	if err != nil {
+		return nil, err
+	}
+
+	return watcher, nil
+}

shared/kubernetes/provider.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
@@ -152,14 +153,6 @@ func (provider *Provider) WaitUtilNamespaceDeleted(ctx context.Context, name str
 	return err
 }
 
-func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface {
-	watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
-	if err != nil {
-		panic(err.Error())
-	}
-	return watcher
-}
-
 func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*core.Namespace, error) {
 	namespaceSpec := &core.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
@@ -179,7 +172,7 @@ type ApiServerOptions struct {
 	MaxEntriesDBSizeBytes int64
 	Resources             shared.Resources
 	ImagePullPolicy       core.PullPolicy
-	DumpLogs              bool
+	LogLevel              logging.Level
 }
 
 func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, mountVolumeClaim bool, volumeClaimName string) (*core.Pod, error) {
@@ -241,22 +234,17 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 			},
 		})
 		volumeMounts = append(volumeMounts, core.VolumeMount{
-			Name:             volumeClaimName,
-			MountPath:        shared.DataDirPath,
+			Name:      volumeClaimName,
+			MountPath: shared.DataDirPath,
 		})
 	}
 
 	port := intstr.FromInt(shared.DefaultApiServerPort)
 
-	debugMode := ""
-	if opts.DumpLogs {
-		debugMode = "1"
-	}
-
 	pod := &core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      opts.PodName,
-			Labels:    map[string]string{"app": opts.PodName},
+			Name:   opts.PodName,
+			Labels: map[string]string{"app": opts.PodName},
 		},
 		Spec: core.PodSpec{
 			Containers: []core.Container{
@@ -264,16 +252,16 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 					Name:            opts.PodName,
 					Image:           opts.PodImage,
 					ImagePullPolicy: opts.ImagePullPolicy,
-					VolumeMounts: volumeMounts,
-					Command: command,
+					VolumeMounts:    volumeMounts,
+					Command:         command,
 					Env: []core.EnvVar{
 						{
 							Name:  shared.SyncEntriesConfigEnvVar,
 							Value: string(marshaledSyncEntriesConfig),
 						},
 						{
-							Name:  shared.DebugModeEnvVar,
-							Value: debugMode,
+							Name:  shared.LogLevelEnvVar,
+							Value: opts.LogLevel.String(),
 						},
 					},
 					Resources: core.ResourceRequirements{
@@ -307,7 +295,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 					},
 				},
 			},
-			Volumes: volumes,
+			Volumes:                       volumes,
 			DNSPolicy:                     core.DNSClusterFirstWithHostNet,
 			TerminationGracePeriodSeconds: new(int64),
 		},
@@ -320,7 +308,6 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 	return pod, nil
 }
 
-
 func (provider *Provider) CreatePod(ctx context.Context, namespace string, podSpec *core.Pod) (*core.Pod, error) {
 	return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, podSpec, metav1.CreateOptions{})
 }
@@ -335,14 +322,14 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string
 	}
 	deployment := &v1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      deploymentName,
+			Name: deploymentName,
 		},
 		Spec: v1.DeploymentSpec{
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{"app": podSpec.ObjectMeta.Labels["app"]},
 			},
-			Template:                *podTemplate,
-			Strategy:                v1.DeploymentStrategy{},
+			Template: *podTemplate,
+			Strategy: v1.DeploymentStrategy{},
 		},
 	}
 	return provider.clientSet.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{})
@@ -351,7 +338,7 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string
 func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) {
 	service := core.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      serviceName,
+			Name: serviceName,
 		},
 		Spec: core.ServiceSpec{
 			Ports:    []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}},
@@ -383,8 +370,8 @@ func (provider *Provider) doesResourceExist(resource interface{}, err error) (bo
 func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error {
 	serviceAccount := &core.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      serviceAccountName,
-			Labels:    map[string]string{"mizu-cli-version": version},
+			Name:   serviceAccountName,
+			Labels: map[string]string{"mizu-cli-version": version},
 		},
 	}
 	clusterRole := &rbac.ClusterRole{
@@ -436,8 +423,8 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
 func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error {
 	serviceAccount := &core.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      serviceAccountName,
-			Labels:    map[string]string{"mizu-cli-version": version},
+			Name:   serviceAccountName,
+			Labels: map[string]string{"mizu-cli-version": version},
 		},
 	}
 	role := &rbac.Role{
@@ -584,6 +571,11 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string,
 	return provider.handleRemovalError(err)
 }
 
+func (provider *Provider) RemovePersistentVolumeClaim(ctx context.Context, namespace string, volumeClaimName string) error {
+	err := provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, volumeClaimName, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
+}
+
 func (provider *Provider) handleRemovalError(err error) error {
 	// Ignore NotFound - There is nothing to delete.
 	// Ignore Forbidden - Assume that a user could not have created the resource in the first place.
@@ -610,7 +602,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      configMapName,
+			Name: configMapName,
 		},
 		Data: configMapData,
 	}
@@ -620,7 +612,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, dumpLogs bool) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, istio bool) error {
 	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodIPMap) == 0 {
@@ -643,12 +635,10 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"--tap",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
 		"--nodefrag",
-		"--procfs", procfsMountPath,
 	}
-
-	debugMode := ""
-	if dumpLogs {
-		debugMode = "1"
+	
+	if istio {
+		mizuCmd = append(mizuCmd, "--procfs", procfsMountPath, "--istio")
 	}
 
 	agentContainer := applyconfcore.Container()
@@ -658,7 +648,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
-		applyconfcore.EnvVar().WithName(shared.DebugModeEnvVar).WithValue(debugMode),
+		applyconfcore.EnvVar().WithName(shared.LogLevelEnvVar).WithValue(logLevel.String()),
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
 		applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)),
 		applyconfcore.EnvVar().WithName(shared.GoGCEnvVar).WithValue("12800"),
@@ -853,9 +843,9 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names
 		ObjectMeta: metav1.ObjectMeta{
 			Name: volumeClaimName,
 		},
-		Spec:       core.PersistentVolumeClaimSpec{
-			AccessModes:      []core.PersistentVolumeAccessMode{core.ReadWriteOnce},
-			Resources:        core.ResourceRequirements{
+		Spec: core.PersistentVolumeClaimSpec{
+			AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce},
+			Resources: core.ResourceRequirements{
 				Limits: core.ResourceList{
 					core.ResourceStorage: *sizeLimitQuantity,
 				},
@@ -869,10 +859,6 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names
 	return provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, volumeClaim, metav1.CreateOptions{})
 }
 
-func (provider *Provider) RemovePersistentVolumeClaim(ctx context.Context, namespace string, volumeClaimName string) error {
-	return provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, volumeClaimName, metav1.DeleteOptions{})
-}
-
 func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) {
 	clientSet, err := kubernetes.NewForConfig(config)
 	if err != nil {

shared/kubernetes/proxy.go

@@ -28,9 +28,9 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
 		return err
 	}
 	mux := http.NewServeMux()
-	mux.Handle(k8sProxyApiPrefix, proxyHandler)
+	mux.Handle(k8sProxyApiPrefix, getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
 	mux.Handle("/static/", getRerouteHttpHandlerMizuStatic(proxyHandler, mizuNamespace, mizuServiceName))
-	mux.Handle("/mizu/", getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
+
 
 	l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", proxyHost, int(mizuPort)))
 	if err != nil {
@@ -45,16 +45,21 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
 }
 
 func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
-	return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort)
+	return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy", mizuNamespace, mizuServiceName, mizuServicePort)
 }
 
 func GetMizuApiServerProxiedHostAndPath(mizuPort uint16) string {
-	return fmt.Sprintf("localhost:%d/mizu", mizuPort)
+	return fmt.Sprintf("localhost:%d", mizuPort)
 }
 
 func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
+		proxiedPath := getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName)
+
+		//avoid redirecting several times
+		if !strings.Contains(r.URL.Path, proxiedPath) {
+			r.URL.Path = fmt.Sprintf("%s%s", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), r.URL.Path)
+		}
 		proxyHandler.ServeHTTP(w, r)
 	})
 }

shared/kubernetes/watch.go

@@ -6,19 +6,25 @@ import (
 	"fmt"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
-	"regexp"
 	"sync"
 	"time"
 
-	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/watch"
 )
 
-func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetNamespaces []string, podFilter *regexp.Regexp) (chan *corev1.Pod, chan *corev1.Pod, chan *corev1.Pod, chan error) {
-	addedChan := make(chan *corev1.Pod)
-	modifiedChan := make(chan *corev1.Pod)
-	removedChan := make(chan *corev1.Pod)
+type EventFilterer interface {
+	Filter(*WatchEvent) (bool, error)
+}
+
+type WatchCreator interface {
+	NewWatcher(ctx context.Context, namespace string) (watch.Interface, error)
+}
+
+func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNamespaces []string, filterer EventFilterer) (chan *WatchEvent, chan *WatchEvent, chan *WatchEvent, chan error) {
+	addedChan := make(chan *WatchEvent)
+	modifiedChan := make(chan *WatchEvent)
+	removedChan := make(chan *WatchEvent)
 	errorChan := make(chan error)
 
 	var wg sync.WaitGroup
@@ -31,8 +37,13 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
 			watchRestartDebouncer := debounce.NewDebouncer(1 * time.Minute, func() {})
 
 			for {
-				watcher := kubernetesProvider.GetPodWatcher(ctx, targetNamespace)
-				err := startWatchLoop(ctx, watcher, podFilter, addedChan, modifiedChan, removedChan) // blocking
+				watcher, err := watcherCreator.NewWatcher(ctx, targetNamespace)
+				if err != nil {
+					errorChan <- fmt.Errorf("error in k8 watch: %v", err)
+					break
+				}
+
+				err = startWatchLoop(ctx, watcher, filterer, addedChan, modifiedChan, removedChan) // blocking
 				watcher.Stop()
 
 				select {
@@ -72,7 +83,7 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
 	return addedChan, modifiedChan, removedChan, errorChan
 }
 
-func startWatchLoop(ctx context.Context, watcher watch.Interface, podFilter *regexp.Regexp, addedChan chan *corev1.Pod, modifiedChan chan *corev1.Pod, removedChan chan *corev1.Pod) error {
+func startWatchLoop(ctx context.Context, watcher watch.Interface, filterer EventFilterer, addedChan chan *WatchEvent, modifiedChan chan *WatchEvent, removedChan chan *WatchEvent) error {
 	resultChan := watcher.ResultChan()
 	for {
 		select {
@@ -81,26 +92,25 @@ func startWatchLoop(ctx context.Context, watcher watch.Interface, podFilter *reg
 				return nil
 			}
 
-			if e.Type == watch.Error {
-				return apierrors.FromObject(e.Object)
+			wEvent := WatchEvent(e)
+
+			if wEvent.Type == watch.Error {
+				return apierrors.FromObject(wEvent.Object)
 			}
 
-			pod, ok := e.Object.(*corev1.Pod)
-			if !ok {
+			if pass, err := filterer.Filter(&wEvent); err != nil {
+				return err
+			} else if !pass {
 				continue
 			}
 
-			if !podFilter.MatchString(pod.Name) {
-				continue
-			}
-
-			switch e.Type {
+			switch wEvent.Type {
 			case watch.Added:
-				addedChan <- pod
+				addedChan <- &wEvent
 			case watch.Modified:
-				modifiedChan <- pod
+				modifiedChan <- &wEvent
 			case watch.Deleted:
-				removedChan <- pod
+				removedChan <- &wEvent
 			}
 		case <-ctx.Done():
 			return nil

shared/kubernetes/watchEvent.go

@@ -0,0 +1,18 @@
+package kubernetes
+
+import (
+	"fmt"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+type WatchEvent watch.Event
+
+func (we *WatchEvent) ToPod() (*corev1.Pod, error) {
+	pod, ok := we.Object.(*corev1.Pod)
+	if !ok {
+		return nil, fmt.Errorf("Invalid object type on pod event stream")
+	}
+
+	return pod, nil
+}

shared/logger/logger.go

@@ -9,7 +9,7 @@ import (
 var Log = logging.MustGetLogger("mizu")
 
 var format = logging.MustStringFormatter(
-	`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
+	`%{time:2006-01-02T15:04:05.999Z-07:00} %{level:-5s} ▶ %{message} ▶ %{pid} %{shortfile} %{shortfunc}`,
 )
 
 func InitLogger(logPath string) {

shared/models.go

@@ -1,9 +1,10 @@
 package shared
 
 import (
+	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
-	"log"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -17,6 +18,9 @@ const (
 	WebSocketMessageTypeUpdateStatus  WebSocketMessageType = "status"
 	WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
 	WebsocketMessageTypeOutboundLink  WebSocketMessageType = "outboundLink"
+	WebSocketMessageTypeToast         WebSocketMessageType = "toast"
+	WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata"
+	WebSocketMessageTypeStartTime     WebSocketMessageType = "startTime"
 )
 
 type Resources struct {
@@ -33,12 +37,13 @@ type MizuAgentConfig struct {
 	TargetNamespaces        []string                    `json:"targetNamespaces"`
 	AgentImage              string                      `json:"agentImage"`
 	PullPolicy              string                      `json:"pullPolicy"`
-	DumpLogs                bool                        `json:"dumpLogs"`
+	LogLevel                logging.Level               `json:"logLevel"`
 	IgnoredUserAgents       []string                    `json:"ignoredUserAgents"`
 	TapperResources         Resources                   `json:"tapperResources"`
 	MizuResourcesNamespace  string                      `json:"mizuResourceNamespace"`
 	MizuApiFilteringOptions api.TrafficFilteringOptions `json:"mizuApiFilteringOptions"`
 	AgentDatabasePath       string                      `json:"agentDatabasePath"`
+	Istio                   bool                        `json:"istio"`
 }
 
 type WebSocketMessageMetadata struct {
@@ -137,14 +142,12 @@ func (r *RulePolicy) validateType() bool {
 	permitedTypes := []string{"json", "header", "slo"}
 	_, found := Find(permitedTypes, r.Type)
 	if !found {
-		log.Printf("Error: %s. ", r.Name)
-		log.Printf("Only json, header and slo types are supported on rule definition. This rule will be ignored\n")
+		logger.Log.Errorf("Only json, header and slo types are supported on rule definition. This rule will be ignored. rule name: %s", r.Name)
 		found = false
 	}
 	if strings.ToLower(r.Type) == "slo" {
 		if r.ResponseTime <= 0 {
-			log.Printf("Error: %s. ", r.Name)
-			log.Printf("When type=slo, the field response-time should be specified and have a value >= 1\n\n")
+			logger.Log.Errorf("When rule type is slo, the field response-time should be specified and have a value >= 1. rule name: %s", r.Name)
 			found = false
 		}
 	}

tap/api/api.go

@@ -18,7 +18,8 @@ import (
 type Protocol struct {
 	Name            string   `json:"name"`
 	LongName        string   `json:"longName"`
-	Abbreviation    string   `json:"abbreviation"`
+	Abbreviation    string   `json:"abbr"`
+	Macro           string   `json:"macro"`
 	Version         string   `json:"version"`
 	BackgroundColor string   `json:"backgroundColor"`
 	ForegroundColor string   `json:"foregroundColor"`
@@ -28,6 +29,12 @@ type Protocol struct {
 	Priority        uint8    `json:"priority"`
 }
 
+type TCP struct {
+	IP   string `json:"ip"`
+	Port string `json:"port"`
+	Name string `json:"name"`
+}
+
 type Extension struct {
 	Protocol   *Protocol
 	Path       string
@@ -74,6 +81,7 @@ type OutputChannelItem struct {
 	Timestamp      int64
 	ConnectionInfo *ConnectionInfo
 	Pair           *RequestResponsePair
+	Summary        *BaseEntryDetails
 }
 
 type SuperTimer struct {
@@ -89,9 +97,10 @@ type Dissector interface {
 	Register(*Extension)
 	Ping()
 	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error
-	Analyze(item *OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *MizuEntry
+	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *MizuEntry
 	Summarize(entry *MizuEntry) *BaseEntryDetails
-	Represent(entry *MizuEntry) (protocol Protocol, object []byte, bodySize int64, err error)
+	Represent(pIn Protocol, request map[string]interface{}, response map[string]interface{}) (pOut Protocol, object []byte, bodySize int64, err error)
+	Macros() map[string]string
 }
 
 type Emitting struct {
@@ -109,39 +118,36 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 }
 
 type MizuEntry struct {
-	ID                      uint `gorm:"primarykey"`
-	CreatedAt               time.Time
-	UpdatedAt               time.Time
-	ProtocolName            string         `json:"protocolName" gorm:"column:protocolName"`
-	ProtocolLongName        string         `json:"protocolLongName" gorm:"column:protocolLongName"`
-	ProtocolAbbreviation    string         `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"`
-	ProtocolVersion         string         `json:"protocolVersion" gorm:"column:protocolVersion"`
-	ProtocolBackgroundColor string         `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"`
-	ProtocolForegroundColor string         `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"`
-	ProtocolFontSize        int8           `json:"protocolFontSize" gorm:"column:protocolFontSize"`
-	ProtocolReferenceLink   string         `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"`
-	Entry                   string         `json:"entry,omitempty" gorm:"column:entry"`
-	EntryId                 string         `json:"entryId" gorm:"column:entryId"`
-	Url                     string         `json:"url" gorm:"column:url"`
-	Method                  string         `json:"method" gorm:"column:method"`
-	Status                  int            `json:"status" gorm:"column:status"`
-	RequestSenderIp         string         `json:"requestSenderIp" gorm:"column:requestSenderIp"`
-	Service                 string         `json:"service" gorm:"column:service"`
-	Timestamp               int64          `json:"timestamp" gorm:"column:timestamp"`
-	ElapsedTime             int64          `json:"elapsedTime" gorm:"column:elapsedTime"`
-	Path                    string         `json:"path" gorm:"column:path"`
-	ResolvedSource          string         `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
-	ResolvedDestination     string         `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
-	SourceIp                string         `json:"sourceIp,omitempty" gorm:"column:sourceIp"`
-	DestinationIp           string         `json:"destinationIp,omitempty" gorm:"column:destinationIp"`
-	SourcePort              string         `json:"sourcePort,omitempty" gorm:"column:sourcePort"`
-	DestinationPort         string         `json:"destinationPort,omitempty" gorm:"column:destinationPort"`
-	IsOutgoing              bool           `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
-	ContractStatus          ContractStatus `json:"contractStatus,omitempty" gorm:"column:contractStatus"`
-	ContractRequestReason   string         `json:"contractRequestReason,omitempty" gorm:"column:contractRequestReason"`
-	ContractResponseReason  string         `json:"contractResponseReason,omitempty" gorm:"column:contractResponseReason"`
-	ContractContent         string         `json:"contractContent,omitempty" gorm:"column:contractContent"`
-	EstimatedSizeBytes      int            `json:"-" gorm:"column:estimatedSizeBytes"`
+	Id                     uint                   `json:"id"`
+	Protocol               Protocol               `json:"proto"`
+	Source                 *TCP                   `json:"src"`
+	Destination            *TCP                   `json:"dst"`
+	Outgoing               bool                   `json:"outgoing"`
+	Timestamp              int64                  `json:"timestamp"`
+	StartTime              time.Time              `json:"startTime"`
+	Request                map[string]interface{} `json:"request"`
+	Response               map[string]interface{} `json:"response"`
+	Base                   *BaseEntryDetails      `json:"base"`
+	Summary                string                 `json:"summary"`
+	Url                    string                 `json:"url"`
+	Method                 string                 `json:"method"`
+	Status                 int                    `json:"status"`
+	RequestSenderIp        string                 `json:"requestSenderIp"`
+	Service                string                 `json:"service"`
+	ElapsedTime            int64                  `json:"elapsedTime"`
+	Path                   string                 `json:"path"`
+	ResolvedSource         string                 `json:"resolvedSource,omitempty"`
+	ResolvedDestination    string                 `json:"resolvedDestination,omitempty"`
+	SourceIp               string                 `json:"sourceIp,omitempty"`
+	DestinationIp          string                 `json:"destinationIp,omitempty"`
+	SourcePort             string                 `json:"sourcePort,omitempty"`
+	DestinationPort        string                 `json:"destinationPort,omitempty"`
+	IsOutgoing             bool                   `json:"isOutgoing,omitempty"`
+	ContractStatus         ContractStatus         `json:"contractStatus,omitempty"`
+	ContractRequestReason  string                 `json:"contractRequestReason,omitempty"`
+	ContractResponseReason string                 `json:"contractResponseReason,omitempty"`
+	ContractContent        string                 `json:"contractContent,omitempty"`
+	HTTPPair               string                 `json:"httpPair,omitempty"`
 }
 
 type MizuEntryWrapper struct {
@@ -154,7 +160,7 @@ type MizuEntryWrapper struct {
 }
 
 type BaseEntryDetails struct {
-	Id              string          `json:"id,omitempty"`
+	Id              uint            `json:"id"`
 	Protocol        Protocol        `json:"protocol,omitempty"`
 	Url             string          `json:"url,omitempty"`
 	RequestSenderIp string          `json:"requestSenderIp,omitempty"`
@@ -194,17 +200,8 @@ type DataUnmarshaler interface {
 }
 
 func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
-	bed.Protocol = Protocol{
-		Name:            entry.ProtocolName,
-		LongName:        entry.ProtocolLongName,
-		Abbreviation:    entry.ProtocolAbbreviation,
-		Version:         entry.ProtocolVersion,
-		BackgroundColor: entry.ProtocolBackgroundColor,
-		ForegroundColor: entry.ProtocolForegroundColor,
-		FontSize:        entry.ProtocolFontSize,
-		ReferenceLink:   entry.ProtocolReferenceLink,
-	}
-	bed.Id = entry.EntryId
+	bed.Protocol = entry.Protocol
+	bed.Id = entry.Id
 	bed.Url = entry.Url
 	bed.RequestSenderIp = entry.RequestSenderIp
 	bed.Service = entry.Service
@@ -228,6 +225,21 @@ const (
 	BODY  string = "body"
 )
 
+type SectionData struct {
+	Type     string `json:"type"`
+	Title    string `json:"title"`
+	Data     string `json:"data"`
+	Encoding string `json:"encoding,omitempty"`
+	MimeType string `json:"mimeType,omitempty"`
+	Selector string `json:"selector,omitempty"`
+}
+
+type TableData struct {
+	Name     string      `json:"name"`
+	Value    interface{} `json:"value"`
+	Selector string      `json:"selector"`
+}
+
 const (
 	TypeHttpRequest = iota
 	TypeHttpResponse
@@ -275,7 +287,7 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) {
 			RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)},
 		})
 	default:
-		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type))
+		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s", h.Type))
 	}
 }
 

tap/diagnose/diagnose.go

@@ -39,7 +39,7 @@ func StartMemoryProfiler(envDumpPath string, envTimeInterval string) {
 
 			filename := fmt.Sprintf("%s/%s__mem.prof", dumpPath, t.Format("15_04_05"))
 
-			logger.Log.Infof("Writing memory profile to %s\n", filename)
+			logger.Log.Infof("Writing memory profile to %s", filename)
 
 			f, err := os.Create(filename)
 			if err != nil {

tap/extensions/amqp/helpers.go

@@ -131,97 +131,109 @@ func representProperties(properties map[string]interface{}, rep []interface{}) (
 	userId := ""
 	appId := ""
 
-	if properties["ContentType"] != nil {
-		contentType = properties["ContentType"].(string)
+	if properties["contentType"] != nil {
+		contentType = properties["contentType"].(string)
 	}
-	if properties["ContentEncoding"] != nil {
-		contentEncoding = properties["ContentEncoding"].(string)
+	if properties["contentEncoding"] != nil {
+		contentEncoding = properties["contentEncoding"].(string)
 	}
-	if properties["Delivery Mode"] != nil {
-		deliveryMode = fmt.Sprintf("%g", properties["DeliveryMode"].(float64))
+	if properties["deliveryMode"] != nil {
+		deliveryMode = fmt.Sprintf("%g", properties["deliveryMode"].(float64))
 	}
-	if properties["Priority"] != nil {
-		priority = fmt.Sprintf("%g", properties["Priority"].(float64))
+	if properties["priority"] != nil {
+		priority = fmt.Sprintf("%g", properties["priority"].(float64))
 	}
-	if properties["CorrelationId"] != nil {
-		correlationId = properties["CorrelationId"].(string)
+	if properties["correlationId"] != nil {
+		correlationId = properties["correlationId"].(string)
 	}
-	if properties["ReplyTo"] != nil {
-		replyTo = properties["ReplyTo"].(string)
+	if properties["replyTo"] != nil {
+		replyTo = properties["replyTo"].(string)
 	}
-	if properties["Expiration"] != nil {
-		expiration = properties["Expiration"].(string)
+	if properties["expiration"] != nil {
+		expiration = properties["expiration"].(string)
 	}
-	if properties["MessageId"] != nil {
-		messageId = properties["MessageId"].(string)
+	if properties["messageId"] != nil {
+		messageId = properties["messageId"].(string)
 	}
-	if properties["Timestamp"] != nil {
-		timestamp = properties["Timestamp"].(string)
+	if properties["timestamp"] != nil {
+		timestamp = properties["timestamp"].(string)
 	}
-	if properties["Type"] != nil {
-		_type = properties["Type"].(string)
+	if properties["type"] != nil {
+		_type = properties["type"].(string)
 	}
-	if properties["UserId"] != nil {
-		userId = properties["UserId"].(string)
+	if properties["userId"] != nil {
+		userId = properties["userId"].(string)
 	}
-	if properties["AppId"] != nil {
-		appId = properties["AppId"].(string)
+	if properties["appId"] != nil {
+		appId = properties["appId"].(string)
 	}
 
-	props, _ := json.Marshal([]map[string]string{
+	props, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Content Type",
-			"value": contentType,
+			Name:     "Content Type",
+			Value:    contentType,
+			Selector: `request.properties.contentType`,
 		},
 		{
-			"name":  "Content Encoding",
-			"value": contentEncoding,
+			Name:     "Content Encoding",
+			Value:    contentEncoding,
+			Selector: `request.properties.contentEncoding`,
 		},
 		{
-			"name":  "Delivery Mode",
-			"value": deliveryMode,
+			Name:     "Delivery Mode",
+			Value:    deliveryMode,
+			Selector: `request.properties.deliveryMode`,
 		},
 		{
-			"name":  "Priority",
-			"value": priority,
+			Name:     "Priority",
+			Value:    priority,
+			Selector: `request.properties.priority`,
 		},
 		{
-			"name":  "Correlation ID",
-			"value": correlationId,
+			Name:     "Correlation ID",
+			Value:    correlationId,
+			Selector: `request.properties.correlationId`,
 		},
 		{
-			"name":  "Reply To",
-			"value": replyTo,
+			Name:     "Reply To",
+			Value:    replyTo,
+			Selector: `request.properties.replyTo`,
 		},
 		{
-			"name":  "Expiration",
-			"value": expiration,
+			Name:     "Expiration",
+			Value:    expiration,
+			Selector: `request.properties.expiration`,
 		},
 		{
-			"name":  "Message ID",
-			"value": messageId,
+			Name:     "Message ID",
+			Value:    messageId,
+			Selector: `request.properties.messageId`,
 		},
 		{
-			"name":  "Timestamp",
-			"value": timestamp,
+			Name:     "Timestamp",
+			Value:    timestamp,
+			Selector: `request.properties.timestamp`,
 		},
 		{
-			"name":  "Type",
-			"value": _type,
+			Name:     "Type",
+			Value:    _type,
+			Selector: `request.properties.type`,
 		},
 		{
-			"name":  "User ID",
-			"value": userId,
+			Name:     "User ID",
+			Value:    userId,
+			Selector: `request.properties.userId`,
 		},
 		{
-			"name":  "App ID",
-			"value": appId,
+			Name:     "App ID",
+			Value:    appId,
+			Selector: `request.properties.appId`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Properties",
-		"data":  string(props),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Properties",
+		Data:  string(props),
 	})
 
 	return rep, contentType, contentEncoding
@@ -230,56 +242,62 @@ func representProperties(properties map[string]interface{}, rep []interface{}) (
 func representBasicPublish(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "Routing Key",
-			"value": event["RoutingKey"].(string),
+			Name:     "Routing Key",
+			Value:    event["routingKey"].(string),
+			Selector: `request.routingKey`,
 		},
 		{
-			"name":  "Mandatory",
-			"value": strconv.FormatBool(event["Mandatory"].(bool)),
+			Name:     "Mandatory",
+			Value:    strconv.FormatBool(event["mandatory"].(bool)),
+			Selector: `request.mandatory`,
 		},
 		{
-			"name":  "Immediate",
-			"value": strconv.FormatBool(event["Immediate"].(bool)),
+			Name:     "Immediate",
+			Value:    strconv.FormatBool(event["immediate"].(bool)),
+			Selector: `request.immediate`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	properties := event["Properties"].(map[string]interface{})
+	properties := event["properties"].(map[string]interface{})
 	rep, contentType, _ := representProperties(properties, rep)
 
-	if properties["Headers"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range properties["Headers"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if properties["headers"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range properties["headers"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Headers",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Headers",
+			Data:  string(headersMarshaled),
 		})
 	}
 
-	if event["Body"] != nil {
-		rep = append(rep, map[string]string{
-			"type":      api.BODY,
-			"title":     "Body",
-			"encoding":  "base64",
-			"mime_type": contentType,
-			"data":      event["Body"].(string),
+	if event["body"] != nil {
+		rep = append(rep, api.SectionData{
+			Type:     api.BODY,
+			Title:    "Body",
+			Encoding: "base64",
+			MimeType: contentType,
+			Data:     event["body"].(string),
+			Selector: `request.body`,
 		})
 	}
 
@@ -293,70 +311,77 @@ func representBasicDeliver(event map[string]interface{}) []interface{} {
 	deliveryTag := ""
 	redelivered := ""
 
-	if event["ConsumerTag"] != nil {
-		consumerTag = event["ConsumerTag"].(string)
+	if event["consumerTag"] != nil {
+		consumerTag = event["consumerTag"].(string)
 	}
-	if event["DeliveryTag"] != nil {
-		deliveryTag = fmt.Sprintf("%g", event["DeliveryTag"].(float64))
+	if event["deliveryTag"] != nil {
+		deliveryTag = fmt.Sprintf("%g", event["deliveryTag"].(float64))
 	}
-	if event["Redelivered"] != nil {
-		redelivered = strconv.FormatBool(event["Redelivered"].(bool))
+	if event["redelivered"] != nil {
+		redelivered = strconv.FormatBool(event["redelivered"].(bool))
 	}
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Consumer Tag",
-			"value": consumerTag,
+			Name:     "Consumer Tag",
+			Value:    consumerTag,
+			Selector: `request.consumerTag`,
 		},
 		{
-			"name":  "Delivery Tag",
-			"value": deliveryTag,
+			Name:     "Delivery Tag",
+			Value:    deliveryTag,
+			Selector: `request.deliveryTag`,
 		},
 		{
-			"name":  "Redelivered",
-			"value": redelivered,
+			Name:     "Redelivered",
+			Value:    redelivered,
+			Selector: `request.redelivered`,
 		},
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "Routing Key",
-			"value": event["RoutingKey"].(string),
+			Name:     "Routing Key",
+			Value:    event["routingKey"].(string),
+			Selector: `request.routingKey`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	properties := event["Properties"].(map[string]interface{})
+	properties := event["properties"].(map[string]interface{})
 	rep, contentType, _ := representProperties(properties, rep)
 
-	if properties["Headers"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range properties["Headers"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if properties["headers"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range properties["headers"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Headers",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Headers",
+			Data:  string(headersMarshaled),
 		})
 	}
 
-	if event["Body"] != nil {
-		rep = append(rep, map[string]string{
-			"type":      api.BODY,
-			"title":     "Body",
-			"encoding":  "base64",
-			"mime_type": contentType,
-			"data":      event["Body"].(string),
+	if event["body"] != nil {
+		rep = append(rep, api.SectionData{
+			Type:     api.BODY,
+			Title:    "Body",
+			Encoding: "base64",
+			MimeType: contentType,
+			Data:     event["body"].(string),
+			Selector: `request.body`,
 		})
 	}
 
@@ -366,51 +391,58 @@ func representBasicDeliver(event map[string]interface{}) []interface{} {
 func representQueueDeclare(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Queue",
-			"value": event["Queue"].(string),
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Passive",
-			"value": strconv.FormatBool(event["Passive"].(bool)),
+			Name:     "Passive",
+			Value:    strconv.FormatBool(event["passive"].(bool)),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Durable",
-			"value": strconv.FormatBool(event["Durable"].(bool)),
+			Name:     "Durable",
+			Value:    strconv.FormatBool(event["durable"].(bool)),
+			Selector: `request.durable`,
 		},
 		{
-			"name":  "Exclusive",
-			"value": strconv.FormatBool(event["Exclusive"].(bool)),
+			Name:     "Exclusive",
+			Value:    strconv.FormatBool(event["exclusive"].(bool)),
+			Selector: `request.exclusive`,
 		},
 		{
-			"name":  "Auto Delete",
-			"value": strconv.FormatBool(event["AutoDelete"].(bool)),
+			Name:     "Auto Delete",
+			Value:    strconv.FormatBool(event["autoDelete"].(bool)),
+			Selector: `request.autoDelete`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -420,55 +452,63 @@ func representQueueDeclare(event map[string]interface{}) []interface{} {
 func representExchangeDeclare(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "Type",
-			"value": event["Type"].(string),
+			Name:     "Type",
+			Value:    event["type"].(string),
+			Selector: `request.type`,
 		},
 		{
-			"name":  "Passive",
-			"value": strconv.FormatBool(event["Passive"].(bool)),
+			Name:     "Passive",
+			Value:    strconv.FormatBool(event["passive"].(bool)),
+			Selector: `request.passive`,
 		},
 		{
-			"name":  "Durable",
-			"value": strconv.FormatBool(event["Durable"].(bool)),
+			Name:     "Durable",
+			Value:    strconv.FormatBool(event["durable"].(bool)),
+			Selector: `request.durable`,
 		},
 		{
-			"name":  "Auto Delete",
-			"value": strconv.FormatBool(event["AutoDelete"].(bool)),
+			Name:     "Auto Delete",
+			Value:    strconv.FormatBool(event["autoDelete"].(bool)),
+			Selector: `request.autoDelete`,
 		},
 		{
-			"name":  "Internal",
-			"value": strconv.FormatBool(event["Internal"].(bool)),
+			Name:     "Internal",
+			Value:    strconv.FormatBool(event["internal"].(bool)),
+			Selector: `request.internal`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -478,33 +518,37 @@ func representExchangeDeclare(event map[string]interface{}) []interface{} {
 func representConnectionStart(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Version Major",
-			"value": fmt.Sprintf("%g", event["VersionMajor"].(float64)),
+			Name:     "Version Major",
+			Value:    fmt.Sprintf("%g", event["versionMajor"].(float64)),
+			Selector: `request.versionMajor`,
 		},
 		{
-			"name":  "Version Minor",
-			"value": fmt.Sprintf("%g", event["VersionMinor"].(float64)),
+			Name:     "Version Minor",
+			Value:    fmt.Sprintf("%g", event["versionMinor"].(float64)),
+			Selector: `request.versionMinor`,
 		},
 		{
-			"name":  "Mechanisms",
-			"value": event["Mechanisms"].(string),
+			Name:     "Mechanisms",
+			Value:    event["mechanisms"].(string),
+			Selector: `request.mechanisms`,
 		},
 		{
-			"name":  "Locales",
-			"value": event["Locales"].(string),
+			Name:     "Locales",
+			Value:    event["locales"].(string),
+			Selector: `request.locales`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["ServerProperties"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["ServerProperties"].(map[string]interface{}) {
+	if event["serverProperties"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["serverProperties"].(map[string]interface{}) {
 			var outcome string
 			switch value.(type) {
 			case string:
@@ -517,16 +561,17 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 			default:
 				panic("Unknown data type for the server property!")
 			}
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": outcome,
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    outcome,
+				Selector: fmt.Sprintf(`request.serverProperties["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Server Properties",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Server Properties",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -536,28 +581,32 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 func representConnectionClose(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Reply Code",
-			"value": fmt.Sprintf("%g", event["ReplyCode"].(float64)),
+			Name:     "Reply Code",
+			Value:    fmt.Sprintf("%g", event["replyCode"].(float64)),
+			Selector: `request.replyCode`,
 		},
 		{
-			"name":  "Reply Text",
-			"value": event["ReplyText"].(string),
+			Name:     "Reply Text",
+			Value:    event["replyText"].(string),
+			Selector: `request.replyText`,
 		},
 		{
-			"name":  "Class ID",
-			"value": fmt.Sprintf("%g", event["ClassId"].(float64)),
+			Name:     "Class ID",
+			Value:    fmt.Sprintf("%g", event["classId"].(float64)),
+			Selector: `request.classId`,
 		},
 		{
-			"name":  "Method ID",
-			"value": fmt.Sprintf("%g", event["MethodId"].(float64)),
+			Name:     "Method ID",
+			Value:    fmt.Sprintf("%g", event["methodId"].(float64)),
+			Selector: `request.methodId`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
 	return rep
@@ -566,43 +615,48 @@ func representConnectionClose(event map[string]interface{}) []interface{} {
 func representQueueBind(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Queue",
-			"value": event["Queue"].(string),
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Exchange",
-			"value": event["Exchange"].(string),
+			Name:     "Exchange",
+			Value:    event["exchange"].(string),
+			Selector: `request.exchange`,
 		},
 		{
-			"name":  "RoutingKey",
-			"value": event["RoutingKey"].(string),
+			Name:     "RoutingKey",
+			Value:    event["routingKey"].(string),
+			Selector: `request.routingKey`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 
@@ -612,51 +666,58 @@ func representQueueBind(event map[string]interface{}) []interface{} {
 func representBasicConsume(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Queue",
-			"value": event["Queue"].(string),
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `request.queue`,
 		},
 		{
-			"name":  "Consumer Tag",
-			"value": event["ConsumerTag"].(string),
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `request.consumerTag`,
 		},
 		{
-			"name":  "No Local",
-			"value": strconv.FormatBool(event["NoLocal"].(bool)),
+			Name:     "No Local",
+			Value:    strconv.FormatBool(event["noLocal"].(bool)),
+			Selector: `request.noLocal`,
 		},
 		{
-			"name":  "No Ack",
-			"value": strconv.FormatBool(event["NoAck"].(bool)),
+			Name:     "No Ack",
+			Value:    strconv.FormatBool(event["noAck"].(bool)),
+			Selector: `request.noAck`,
 		},
 		{
-			"name":  "Exclusive",
-			"value": strconv.FormatBool(event["Exclusive"].(bool)),
+			Name:     "Exclusive",
+			Value:    strconv.FormatBool(event["exclusive"].(bool)),
+			Selector: `request.exclusive`,
 		},
 		{
-			"name":  "NoWait",
-			"value": strconv.FormatBool(event["NoWait"].(bool)),
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	if event["Arguments"] != nil {
-		headers := make([]map[string]string, 0)
-		for name, value := range event["Arguments"].(map[string]interface{}) {
-			headers = append(headers, map[string]string{
-				"name":  name,
-				"value": value.(string),
+	if event["arguments"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["arguments"].(map[string]interface{}) {
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    value.(string),
+				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
 		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, map[string]string{
-			"type":  api.TABLE,
-			"title": "Arguments",
-			"data":  string(headersMarshaled),
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Arguments",
+			Data:  string(headersMarshaled),
 		})
 	}
 

tap/extensions/amqp/main.go

@@ -16,6 +16,7 @@ var protocol api.Protocol = api.Protocol{
 	Name:            "amqp",
 	LongName:        "Advanced Message Queuing Protocol 0-9-1",
 	Abbreviation:    "AMQP",
+	Macro:           "amqp",
 	Version:         "0-9-1",
 	BackgroundColor: "#ff6600",
 	ForegroundColor: "#ffffff",
@@ -36,7 +37,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", protocol.Name)
+	log.Printf("pong %s", protocol.Name)
 }
 
 const amqpRequest string = "amqp_request"
@@ -217,12 +218,12 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			}
 
 		default:
-			// log.Printf("unexpected frame: %+v\n", f)
+			// log.Printf("unexpected frame: %+v", f)
 		}
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 	service := "amqp"
@@ -235,75 +236,79 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	summary := ""
 	switch request["method"] {
 	case basicMethodMap[40]:
-		summary = reqDetails["Exchange"].(string)
+		summary = reqDetails["exchange"].(string)
 		break
 	case basicMethodMap[60]:
-		summary = reqDetails["Exchange"].(string)
+		summary = reqDetails["exchange"].(string)
 		break
 	case exchangeMethodMap[10]:
-		summary = reqDetails["Exchange"].(string)
+		summary = reqDetails["exchange"].(string)
 		break
 	case queueMethodMap[10]:
-		summary = reqDetails["Queue"].(string)
+		summary = reqDetails["queue"].(string)
 		break
 	case connectionMethodMap[10]:
 		summary = fmt.Sprintf(
 			"%s.%s",
-			strconv.Itoa(int(reqDetails["VersionMajor"].(float64))),
-			strconv.Itoa(int(reqDetails["VersionMinor"].(float64))),
+			strconv.Itoa(int(reqDetails["versionMajor"].(float64))),
+			strconv.Itoa(int(reqDetails["versionMinor"].(float64))),
 		)
 		break
 	case connectionMethodMap[50]:
-		summary = reqDetails["ReplyText"].(string)
+		summary = reqDetails["replyText"].(string)
 		break
 	case queueMethodMap[20]:
-		summary = reqDetails["Queue"].(string)
+		summary = reqDetails["queue"].(string)
 		break
 	case basicMethodMap[20]:
-		summary = reqDetails["Queue"].(string)
+		summary = reqDetails["queue"].(string)
 		break
 	}
 
 	request["url"] = summary
-	entryBytes, _ := json.Marshal(item.Pair)
+	reqDetails["method"] = request["method"]
 	return &api.MizuEntry{
-		ProtocolName:            protocol.Name,
-		ProtocolLongName:        protocol.LongName,
-		ProtocolAbbreviation:    protocol.Abbreviation,
-		ProtocolVersion:         protocol.Version,
-		ProtocolBackgroundColor: protocol.BackgroundColor,
-		ProtocolForegroundColor: protocol.ForegroundColor,
-		ProtocolFontSize:        protocol.FontSize,
-		ProtocolReferenceLink:   protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, summary),
-		Method:                  request["method"].(string),
-		Status:                  0,
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             0,
-		Path:                    summary,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Url:                 fmt.Sprintf("%s%s", service, summary),
+		Method:              request["method"].(string),
+		Status:              0,
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         0,
+		Summary:             summary,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
 	}
 
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        protocol,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -320,39 +325,35 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	}
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	p = protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	protoOut = protocol
 	bodySize = 0
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
 	var repRequest []interface{}
-	details := request["details"].(map[string]interface{})
 	switch request["method"].(string) {
 	case basicMethodMap[40]:
-		repRequest = representBasicPublish(details)
+		repRequest = representBasicPublish(request)
 		break
 	case basicMethodMap[60]:
-		repRequest = representBasicDeliver(details)
+		repRequest = representBasicDeliver(request)
 		break
 	case queueMethodMap[10]:
-		repRequest = representQueueDeclare(details)
+		repRequest = representQueueDeclare(request)
 		break
 	case exchangeMethodMap[10]:
-		repRequest = representExchangeDeclare(details)
+		repRequest = representExchangeDeclare(request)
 		break
 	case connectionMethodMap[10]:
-		repRequest = representConnectionStart(details)
+		repRequest = representConnectionStart(request)
 		break
 	case connectionMethodMap[50]:
-		repRequest = representConnectionClose(details)
+		repRequest = representConnectionClose(request)
 		break
 	case queueMethodMap[20]:
-		repRequest = representQueueBind(details)
+		repRequest = representQueueBind(request)
 		break
 	case basicMethodMap[20]:
-		repRequest = representBasicConsume(details)
+		repRequest = representBasicConsume(request)
 		break
 	}
 	representation["request"] = repRequest
@@ -360,4 +361,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`amqp`: fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation),
+	}
+}
+
 var Dissector dissecting

tap/extensions/amqp/spec091.go

@@ -71,11 +71,11 @@ func isSoftExceptionCode(code int) bool {
 }
 
 type ConnectionStart struct {
-	VersionMajor     byte
-	VersionMinor     byte
-	ServerProperties Table
-	Mechanisms       string
-	Locales          string
+	VersionMajor     byte   `json:"versionMajor"`
+	VersionMinor     byte   `json:"versionMinor"`
+	ServerProperties Table  `json:"serverProperties"`
+	Mechanisms       string `json:"mechanisms"`
+	Locales          string `json:"locales"`
 }
 
 func (msg *ConnectionStart) id() (uint16, uint16) {
@@ -429,10 +429,10 @@ func (msg *connectionOpenOk) read(r io.Reader) (err error) {
 }
 
 type ConnectionClose struct {
-	ReplyCode uint16
-	ReplyText string
-	ClassId   uint16
-	MethodId  uint16
+	ReplyCode uint16 `json:"relyCode"`
+	ReplyText string `json:"replyText"`
+	ClassId   uint16 `json:"classId"`
+	MethodId  uint16 `json:"methodId"`
 }
 
 func (msg *ConnectionClose) id() (uint16, uint16) {
@@ -767,14 +767,14 @@ func (msg *channelCloseOk) read(r io.Reader) (err error) {
 
 type ExchangeDeclare struct {
 	reserved1  uint16
-	Exchange   string
-	Type       string
-	Passive    bool
-	Durable    bool
-	AutoDelete bool
-	Internal   bool
-	NoWait     bool
-	Arguments  Table
+	Exchange   string `json:"exchange"`
+	Type       string `json:"type"`
+	Passive    bool   `json:"passive"`
+	Durable    bool   `json:"durable"`
+	AutoDelete bool   `json:"autoDelete"`
+	Internal   bool   `json:"internal"`
+	NoWait     bool   `json:"noWait"`
+	Arguments  Table  `json:"arguments"`
 }
 
 func (msg *ExchangeDeclare) id() (uint16, uint16) {
@@ -1163,13 +1163,13 @@ func (msg *exchangeUnbindOk) read(r io.Reader) (err error) {
 
 type QueueDeclare struct {
 	reserved1  uint16
-	Queue      string
-	Passive    bool
-	Durable    bool
-	Exclusive  bool
-	AutoDelete bool
-	NoWait     bool
-	Arguments  Table
+	Queue      string `json:"queue"`
+	Passive    bool   `json:"passive"`
+	Durable    bool   `json:"durable"`
+	Exclusive  bool   `json:"exclusive"`
+	AutoDelete bool   `json:"autoDelete"`
+	NoWait     bool   `json:"noWait"`
+	Arguments  Table  `json:"arguments"`
 }
 
 func (msg *QueueDeclare) id() (uint16, uint16) {
@@ -1297,11 +1297,11 @@ func (msg *QueueDeclareOk) read(r io.Reader) (err error) {
 
 type QueueBind struct {
 	reserved1  uint16
-	Queue      string
-	Exchange   string
-	RoutingKey string
-	NoWait     bool
-	Arguments  Table
+	Queue      string `json:"queue"`
+	Exchange   string `json:"exchange"`
+	RoutingKey string `json:"routingKey"`
+	NoWait     bool   `json:"noWait"`
+	Arguments  Table  `json:"arguments"`
 }
 
 func (msg *QueueBind) id() (uint16, uint16) {
@@ -1737,13 +1737,13 @@ func (msg *basicQosOk) read(r io.Reader) (err error) {
 
 type BasicConsume struct {
 	reserved1   uint16
-	Queue       string
-	ConsumerTag string
-	NoLocal     bool
-	NoAck       bool
-	Exclusive   bool
-	NoWait      bool
-	Arguments   Table
+	Queue       string `json:"queue"`
+	ConsumerTag string `json:"consumerTag"`
+	NoLocal     bool   `json:"noLocal"`
+	NoAck       bool   `json:"noAck"`
+	Exclusive   bool   `json:"exclusive"`
+	NoWait      bool   `json:"noWait"`
+	Arguments   Table  `json:"arguments"`
 }
 
 func (msg *BasicConsume) id() (uint16, uint16) {
@@ -1932,12 +1932,12 @@ func (msg *basicCancelOk) read(r io.Reader) (err error) {
 
 type BasicPublish struct {
 	reserved1  uint16
-	Exchange   string
-	RoutingKey string
-	Mandatory  bool
-	Immediate  bool
-	Properties Properties
-	Body       []byte
+	Exchange   string     `json:"exchange"`
+	RoutingKey string     `json:"routingKey"`
+	Mandatory  bool       `json:"mandatory"`
+	Immediate  bool       `json:"immediate"`
+	Properties Properties `json:"properties"`
+	Body       []byte     `json:"body"`
 }
 
 func (msg *BasicPublish) id() (uint16, uint16) {
@@ -2072,13 +2072,13 @@ func (msg *basicReturn) read(r io.Reader) (err error) {
 }
 
 type BasicDeliver struct {
-	ConsumerTag string
-	DeliveryTag uint64
-	Redelivered bool
-	Exchange    string
-	RoutingKey  string
-	Properties  Properties
-	Body        []byte
+	ConsumerTag string     `json:"consumerTag"`
+	DeliveryTag uint64     `json:"deliveryTag"`
+	Redelivered bool       `json:"redelivered"`
+	Exchange    string     `json:"exchange"`
+	RoutingKey  string     `json:"routingKey"`
+	Properties  Properties `json:"properties"`
+	Body        []byte     `json:"body"`
 }
 
 func (msg *BasicDeliver) id() (uint16, uint16) {

tap/extensions/amqp/types.go

@@ -93,19 +93,19 @@ func (e Error) Error() string {
 
 // Used by header frames to capture routing and header information
 type Properties struct {
-	ContentType     string    // MIME content type
-	ContentEncoding string    // MIME content encoding
-	Headers         Table     // Application or header exchange table
-	DeliveryMode    uint8     // queue implementation use - Transient (1) or Persistent (2)
-	Priority        uint8     // queue implementation use - 0 to 9
-	CorrelationId   string    // application use - correlation identifier
-	ReplyTo         string    // application use - address to to reply to (ex: RPC)
-	Expiration      string    // implementation use - message expiration spec
-	MessageId       string    // application use - message identifier
-	Timestamp       time.Time // application use - message timestamp
-	Type            string    // application use - message type name
-	UserId          string    // application use - creating user id
-	AppId           string    // application use - creating application
+	ContentType     string    `json:"contentType"`     // MIME content type
+	ContentEncoding string    `json:"contentEncoding"` // MIME content encoding
+	Headers         Table     `json:"headers"`         // Application or header exchange table
+	DeliveryMode    uint8     `json:"deliveryMode"`    // queue implementation use - Transient (1) or Persistent (2)
+	Priority        uint8     `json:"priority"`        // queue implementation use - 0 to 9
+	CorrelationId   string    `json:"correlationId"`   // application use - correlation identifier
+	ReplyTo         string    `json:"replyTo"`         // application use - address to to reply to (ex: RPC)
+	Expiration      string    `json:"expiration"`      // implementation use - message expiration spec
+	MessageId       string    `json:"messageId"`       // application use - message identifier
+	Timestamp       time.Time `json:"timestamp"`       // application use - message timestamp
+	Type            string    `json:"type"`            // application use - message type name
+	UserId          string    `json:"userId"`          // application use - creating user id
+	AppId           string    `json:"appId"`           // application use - creating application
 	reserved1       string    // was cluster-id - process for buffer consumption
 }
 

tap/extensions/http/helpers.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) {
+	newMap = make(map[string]interface{})
+	for _, header := range mapSlice {
+		h := header.(map[string]interface{})
+		newMap[h["name"].(string)] = h["value"]
+	}
+
+	return
+}
+
+func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
+	var table []api.TableData
+	for _, header := range mapSlice {
+		h := header.(map[string]interface{})
+		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string))
+		table = append(table, api.TableData{
+			Name:     h["name"].(string),
+			Value:    h["value"],
+			Selector: selector,
+		})
+	}
+
+	obj, _ := json.Marshal(table)
+	representation = string(obj)
+	return
+}

tap/extensions/http/main.go

@@ -17,6 +17,7 @@ var protocol api.Protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
 	Abbreviation:    "HTTP",
+	Macro:           "http",
 	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
@@ -30,6 +31,7 @@ var http2Protocol api.Protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) (gRPC)",
 	Abbreviation:    "HTTP/2",
+	Macro:           "grpc",
 	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
@@ -56,7 +58,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", protocol.Name)
+	log.Printf("pong %s", protocol.Name)
 }
 
 func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
@@ -117,7 +119,7 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	var host, scheme, authority, path, service string
 
 	request := item.Pair.Request.Payload.(map[string]interface{})
@@ -141,14 +143,40 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 		}
 	}
 
+	if resDetails["bodySize"].(float64) < 0 {
+		resDetails["bodySize"] = 0
+	}
+
 	if item.Protocol.Version == "2.0" {
 		service = fmt.Sprintf("%s://%s", scheme, authority)
 	} else {
 		service = fmt.Sprintf("http://%s", host)
-		path = reqDetails["url"].(string)
+		u, err := url.Parse(reqDetails["url"].(string))
+		if err != nil {
+			path = reqDetails["url"].(string)
+		} else {
+			path = u.Path
+		}
 	}
 
-	request["url"] = path
+	request["url"] = reqDetails["url"].(string)
+	reqDetails["path"] = path
+	reqDetails["summary"] = path
+
+	// Rearrange the maps for the querying
+	reqDetails["_headers"] = reqDetails["headers"]
+	reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{}))
+	resDetails["_headers"] = resDetails["headers"]
+	resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{}))
+
+	reqDetails["_cookies"] = reqDetails["cookies"]
+	reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{}))
+	resDetails["_cookies"] = resDetails["cookies"]
+	resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{}))
+
+	reqDetails["_queryString"] = reqDetails["queryString"]
+	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryString"].([]interface{}))
+
 	if resolvedDestination != "" {
 		service = SetHostname(service, resolvedDestination)
 	} else if resolvedSource != "" {
@@ -156,51 +184,62 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	}
 
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	entryBytes, _ := json.Marshal(item.Pair)
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
+	httpPair, _ := json.Marshal(item.Pair)
+	_protocol := protocol
+	_protocol.Version = item.Protocol.Version
 	return &api.MizuEntry{
-		ProtocolName:            protocol.Name,
-		ProtocolLongName:        protocol.LongName,
-		ProtocolAbbreviation:    protocol.Abbreviation,
-		ProtocolVersion:         item.Protocol.Version,
-		ProtocolBackgroundColor: protocol.BackgroundColor,
-		ProtocolForegroundColor: protocol.ForegroundColor,
-		ProtocolFontSize:        protocol.FontSize,
-		ProtocolReferenceLink:   protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, path),
-		Method:                  reqDetails["method"].(string),
-		Status:                  int(resDetails["status"].(float64)),
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             elapsedTime,
-		Path:                    path,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: _protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Response:            resDetails,
+		Url:                 fmt.Sprintf("%s%s", service, path),
+		Method:              reqDetails["method"].(string),
+		Status:              int(resDetails["status"].(float64)),
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         elapsedTime,
+		Summary:             path,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
+		HTTPPair:            string(httpPair),
 	}
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	var p api.Protocol
-	if entry.ProtocolVersion == "2.0" {
+	if entry.Protocol.Version == "2.0" {
 		p = http2Protocol
 	} else {
 		p = protocol
 	}
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        p,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
 		Path:            entry.Path,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -218,45 +257,50 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 }
 
 func representRequest(request map[string]interface{}) (repRequest []interface{}) {
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Method",
-			"value": request["method"].(string),
+			Name:     "Method",
+			Value:    request["method"].(string),
+			Selector: `request.method`,
 		},
 		{
-			"name":  "URL",
-			"value": request["url"].(string),
+			Name:     "URL",
+			Value:    request["url"].(string),
+			Selector: `request.url`,
 		},
 		{
-			"name":  "Body Size",
-			"value": fmt.Sprintf("%g bytes", request["bodySize"].(float64)),
+			Name:     "Path",
+			Value:    request["path"].(string),
+			Selector: `request.path`,
+		},
+		{
+			Name:     "Body Size (bytes)",
+			Value:    int64(request["bodySize"].(float64)),
+			Selector: `request.bodySize`,
 		},
 	})
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	headers, _ := json.Marshal(request["headers"].([]interface{}))
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Headers",
-		"data":  string(headers),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Headers",
+		Data:  representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`),
 	})
 
-	cookies, _ := json.Marshal(request["cookies"].([]interface{}))
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Cookies",
-		"data":  string(cookies),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Cookies",
+		Data:  representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`),
 	})
 
-	queryString, _ := json.Marshal(request["queryString"].([]interface{}))
-	repRequest = append(repRequest, map[string]string{
-		"type":  api.TABLE,
-		"title": "Query String",
-		"data":  string(queryString),
+	repRequest = append(repRequest, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Query String",
+		Data:  representMapSliceAsTable(request["_queryString"].([]interface{}), `request.queryString`),
 	})
 
 	postData, _ := request["postData"].(map[string]interface{})
@@ -266,12 +310,12 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 	}
 	text, _ := postData["text"]
 	if text != nil {
-		repRequest = append(repRequest, map[string]string{
-			"type":      api.BODY,
-			"title":     "POST Data (text/plain)",
-			"encoding":  "",
-			"mime_type": mimeType.(string),
-			"data":      text.(string),
+		repRequest = append(repRequest, api.SectionData{
+			Type:     api.BODY,
+			Title:    "POST Data (text/plain)",
+			MimeType: mimeType.(string),
+			Data:     text.(string),
+			Selector: `request.postData.text`,
 		})
 	}
 
@@ -285,16 +329,16 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 						"value": string(params),
 					},
 				})
-				repRequest = append(repRequest, map[string]string{
-					"type":  api.TABLE,
-					"title": "POST Data (multipart/form-data)",
-					"data":  string(multipart),
+				repRequest = append(repRequest, api.SectionData{
+					Type:  api.TABLE,
+					Title: "POST Data (multipart/form-data)",
+					Data:  string(multipart),
 				})
 			} else {
-				repRequest = append(repRequest, map[string]string{
-					"type":  api.TABLE,
-					"title": "POST Data (application/x-www-form-urlencoded)",
-					"data":  string(params),
+				repRequest = append(repRequest, api.SectionData{
+					Type:  api.TABLE,
+					Title: "POST Data (application/x-www-form-urlencoded)",
+					Data:  representMapSliceAsTable(postData["params"].([]interface{}), `request.postData.params`),
 				})
 			}
 		}
@@ -308,38 +352,39 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 
 	bodySize = int64(response["bodySize"].(float64))
 
-	details, _ := json.Marshal([]map[string]string{
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Status",
-			"value": fmt.Sprintf("%g", response["status"].(float64)),
+			Name:     "Status",
+			Value:    int64(response["status"].(float64)),
+			Selector: `response.status`,
 		},
 		{
-			"name":  "Status Text",
-			"value": response["statusText"].(string),
+			Name:     "Status Text",
+			Value:    response["statusText"].(string),
+			Selector: `response.statusText`,
 		},
 		{
-			"name":  "Body Size",
-			"value": fmt.Sprintf("%d bytes", bodySize),
+			Name:     "Body Size (bytes)",
+			Value:    bodySize,
+			Selector: `response.bodySize`,
 		},
 	})
-	repResponse = append(repResponse, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	repResponse = append(repResponse, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
-	headers, _ := json.Marshal(response["headers"].([]interface{}))
-	repResponse = append(repResponse, map[string]string{
-		"type":  api.TABLE,
-		"title": "Headers",
-		"data":  string(headers),
+	repResponse = append(repResponse, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Headers",
+		Data:  representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`),
 	})
 
-	cookies, _ := json.Marshal(response["cookies"].([]interface{}))
-	repResponse = append(repResponse, map[string]string{
-		"type":  api.TABLE,
-		"title": "Cookies",
-		"data":  string(cookies),
+	repResponse = append(repResponse, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Cookies",
+		Data:  representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`),
 	})
 
 	content, _ := response["content"].(map[string]interface{})
@@ -350,37 +395,40 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 	encoding, _ := content["encoding"]
 	text, _ := content["text"]
 	if text != nil {
-		repResponse = append(repResponse, map[string]string{
-			"type":      api.BODY,
-			"title":     "Body",
-			"encoding":  encoding.(string),
-			"mime_type": mimeType.(string),
-			"data":      text.(string),
+		repResponse = append(repResponse, api.SectionData{
+			Type:     api.BODY,
+			Title:    "Body",
+			Encoding: encoding.(string),
+			MimeType: mimeType.(string),
+			Data:     text.(string),
+			Selector: `response.content.text`,
 		})
 	}
 
 	return
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	if entry.ProtocolVersion == "2.0" {
-		p = http2Protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	if protoIn.Version == "2.0" {
+		protoOut = http2Protocol
 	} else {
-		p = protocol
+		protoOut = protocol
 	}
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
-	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
-	reqDetails := request["details"].(map[string]interface{})
-	resDetails := response["details"].(map[string]interface{})
-	repRequest := representRequest(reqDetails)
-	repResponse, bodySize := representResponse(resDetails)
+	repRequest := representRequest(request)
+	repResponse, bodySize := representResponse(response)
 	representation["request"] = repRequest
 	representation["response"] = repResponse
 	object, err = json.Marshal(representation)
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`http`:  fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, protocol.Version),
+		`grpc`:  fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version),
+		`http2`: fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version),
+	}
+}
+
 var Dissector dissecting

tap/extensions/http/sensitive_data_cleaner.go

@@ -16,6 +16,7 @@ import (
 )
 
 const maskedFieldPlaceholderValue = "[REDACTED]"
+const userAgent = "user-agent"
 
 //these values MUST be all lower case and contain no `-` or `_` characters
 var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
@@ -32,7 +33,7 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 	request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
 
 	for headerKey, headerValues := range request.Header {
-		if strings.ToLower(headerKey) == "user-agent" {
+		if strings.ToLower(headerKey) == userAgent {
 			for _, userAgent := range options.IgnoredUserAgents {
 				for _, headerValue := range headerValues {
 					if strings.Contains(strings.ToLower(headerValue), strings.ToLower(userAgent)) {
@@ -89,6 +90,10 @@ func filterResponseBody(response *http.Response, options *api.TrafficFilteringOp
 
 func filterHeaders(headers *http.Header) {
 	for key, _ := range *headers {
+		if strings.ToLower(key) == userAgent {
+			continue
+		}
+
 		if strings.ToLower(key) == "cookie" {
 			headers.Del(key)
 		} else if isFieldNameSensitive(key) {

tap/extensions/kafka/helpers.go

@@ -27,48 +27,54 @@ type KafkaWrapper struct {
 }
 
 func representRequestHeader(data map[string]interface{}, rep []interface{}) []interface{} {
-	requestHeader, _ := json.Marshal([]map[string]string{
+	requestHeader, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "ApiKey",
-			"value": apiNames[int(data["ApiKey"].(float64))],
+			Name:     "ApiKey",
+			Value:    apiNames[int(data["apiKey"].(float64))],
+			Selector: `request.apiKey`,
 		},
 		{
-			"name":  "ApiVersion",
-			"value": fmt.Sprintf("%d", int(data["ApiVersion"].(float64))),
+			Name:     "ApiVersion",
+			Value:    fmt.Sprintf("%d", int(data["apiVersion"].(float64))),
+			Selector: `request.apiVersion`,
 		},
 		{
-			"name":  "Client ID",
-			"value": data["ClientID"].(string),
+			Name:     "Client ID",
+			Value:    data["clientID"].(string),
+			Selector: `request.clientID`,
 		},
 		{
-			"name":  "Correlation ID",
-			"value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))),
+			Name:     "Correlation ID",
+			Value:    fmt.Sprintf("%d", int(data["correlationID"].(float64))),
+			Selector: `request.correlationID`,
 		},
 		{
-			"name":  "Size",
-			"value": fmt.Sprintf("%d", int(data["Size"].(float64))),
+			Name:     "Size",
+			Value:    fmt.Sprintf("%d", int(data["size"].(float64))),
+			Selector: `request.size`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Request Header",
-		"data":  string(requestHeader),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Request Header",
+		Data:  string(requestHeader),
 	})
 
 	return rep
 }
 
 func representResponseHeader(data map[string]interface{}, rep []interface{}) []interface{} {
-	requestHeader, _ := json.Marshal([]map[string]string{
+	requestHeader, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Correlation ID",
-			"value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))),
+			Name:     "Correlation ID",
+			Value:    fmt.Sprintf("%d", int(data["correlationID"].(float64))),
+			Selector: `response.correlationID`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Response Header",
-		"data":  string(requestHeader),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Response Header",
+		Data:  string(requestHeader),
 	})
 
 	return rep
@@ -79,46 +85,50 @@ func representMetadataRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
 	allowAutoTopicCreation := ""
 	includeClusterAuthorizedOperations := ""
 	includeTopicAuthorizedOperations := ""
-	if payload["Topics"] != nil {
-		x, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		x, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(x)
 	}
-	if payload["AllowAutoTopicCreation"] != nil {
-		allowAutoTopicCreation = strconv.FormatBool(payload["AllowAutoTopicCreation"].(bool))
+	if payload["allowAutoTopicCreation"] != nil {
+		allowAutoTopicCreation = strconv.FormatBool(payload["allowAutoTopicCreation"].(bool))
 	}
-	if payload["IncludeClusterAuthorizedOperations"] != nil {
-		includeClusterAuthorizedOperations = strconv.FormatBool(payload["IncludeClusterAuthorizedOperations"].(bool))
+	if payload["includeClusterAuthorizedOperations"] != nil {
+		includeClusterAuthorizedOperations = strconv.FormatBool(payload["includeClusterAuthorizedOperations"].(bool))
 	}
-	if payload["IncludeTopicAuthorizedOperations"] != nil {
-		includeTopicAuthorizedOperations = strconv.FormatBool(payload["IncludeTopicAuthorizedOperations"].(bool))
+	if payload["includeTopicAuthorizedOperations"] != nil {
+		includeTopicAuthorizedOperations = strconv.FormatBool(payload["includeTopicAuthorizedOperations"].(bool))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Allow Auto Topic Creation",
-			"value": allowAutoTopicCreation,
+			Name:     "Allow Auto Topic Creation",
+			Value:    allowAutoTopicCreation,
+			Selector: `request.payload.allowAutoTopicCreation`,
 		},
 		{
-			"name":  "Include Cluster Authorized Operations",
-			"value": includeClusterAuthorizedOperations,
+			Name:     "Include Cluster Authorized Operations",
+			Value:    includeClusterAuthorizedOperations,
+			Selector: `request.payload.includeClusterAuthorizedOperations`,
 		},
 		{
-			"name":  "Include Topic Authorized Operations",
-			"value": includeTopicAuthorizedOperations,
+			Name:     "Include Topic Authorized Operations",
+			Value:    includeTopicAuthorizedOperations,
+			Selector: `request.payload.includeTopicAuthorizedOperations`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -129,63 +139,69 @@ func representMetadataResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		_topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		_topics, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(_topics)
 	}
 	brokers := ""
-	if payload["Brokers"] != nil {
-		_brokers, _ := json.Marshal(payload["Brokers"].([]interface{}))
+	if payload["brokers"] != nil {
+		_brokers, _ := json.Marshal(payload["brokers"].([]interface{}))
 		brokers = string(_brokers)
 	}
 	controllerID := ""
 	clusterID := ""
 	throttleTimeMs := ""
 	clusterAuthorizedOperations := ""
-	if payload["ControllerID"] != nil {
-		controllerID = fmt.Sprintf("%d", int(payload["ControllerID"].(float64)))
+	if payload["controllerID"] != nil {
+		controllerID = fmt.Sprintf("%d", int(payload["controllerID"].(float64)))
 	}
-	if payload["ClusterID"] != nil {
-		clusterID = payload["ClusterID"].(string)
+	if payload["clusterID"] != nil {
+		clusterID = payload["clusterID"].(string)
 	}
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	if payload["ClusterAuthorizedOperations"] != nil {
-		clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["ClusterAuthorizedOperations"].(float64)))
+	if payload["clusterAuthorizedOperations"] != nil {
+		clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["clusterAuthorizedOperations"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Brokers",
-			"value": brokers,
+			Name:     "Brokers",
+			Value:    brokers,
+			Selector: `response.payload.brokers`,
 		},
 		{
-			"name":  "Cluster ID",
-			"value": clusterID,
+			Name:     "Cluster ID",
+			Value:    clusterID,
+			Selector: `response.payload.clusterID`,
 		},
 		{
-			"name":  "Controller ID",
-			"value": controllerID,
+			Name:     "Controller ID",
+			Value:    controllerID,
+			Selector: `response.payload.controllerID`,
 		},
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `response.payload.topics`,
 		},
 		{
-			"name":  "Cluster Authorized Operations",
-			"value": clusterAuthorizedOperations,
+			Name:     "Cluster Authorized Operations",
+			Value:    clusterAuthorizedOperations,
+			Selector: `response.payload.clusterAuthorizedOperations`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -196,29 +212,31 @@ func representApiVersionsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	clientSoftwareName := ""
 	clientSoftwareVersion := ""
-	if payload["ClientSoftwareName"] != nil {
-		clientSoftwareName = payload["ClientSoftwareName"].(string)
+	if payload["clientSoftwareName"] != nil {
+		clientSoftwareName = payload["clientSoftwareName"].(string)
 	}
-	if payload["ClientSoftwareVersion"] != nil {
-		clientSoftwareVersion = payload["ClientSoftwareVersion"].(string)
+	if payload["clientSoftwareVersion"] != nil {
+		clientSoftwareVersion = payload["clientSoftwareVersion"].(string)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Client Software Name",
-			"value": clientSoftwareName,
+			Name:     "Client Software Name",
+			Value:    clientSoftwareName,
+			Selector: `request.payload.clientSoftwareName`,
 		},
 		{
-			"name":  "Client Software Version",
-			"value": clientSoftwareVersion,
+			Name:     "Client Software Version",
+			Value:    clientSoftwareVersion,
+			Selector: `request.payload.clientSoftwareVersion`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -229,34 +247,37 @@ func representApiVersionsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	apiKeys := ""
-	if payload["TopicNames"] != nil {
-		x, _ := json.Marshal(payload["ApiKeys"].([]interface{}))
+	if payload["apiKeys"] != nil {
+		x, _ := json.Marshal(payload["apiKeys"].([]interface{}))
 		apiKeys = string(x)
 	}
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Error Code",
-			"value": fmt.Sprintf("%d", int(payload["ErrorCode"].(float64))),
+			Name:     "Error Code",
+			Value:    fmt.Sprintf("%d", int(payload["errorCode"].(float64))),
+			Selector: `response.payload.errorCode`,
 		},
 		{
-			"name":  "ApiKeys",
-			"value": apiKeys,
+			Name:     "ApiKeys",
+			Value:    apiKeys,
+			Selector: `response.payload.apiKeys`,
 		},
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -267,39 +288,43 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topicData := ""
-	_topicData := payload["TopicData"]
+	_topicData := payload["topicData"]
 	if _topicData != nil {
 		x, _ := json.Marshal(_topicData.([]interface{}))
 		topicData = string(x)
 	}
 	transactionalID := ""
-	if payload["TransactionalID"] != nil {
-		transactionalID = payload["TransactionalID"].(string)
+	if payload["transactionalID"] != nil {
+		transactionalID = payload["transactionalID"].(string)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Transactional ID",
-			"value": transactionalID,
+			Name:     "Transactional ID",
+			Value:    transactionalID,
+			Selector: `request.payload.transactionalID`,
 		},
 		{
-			"name":  "Required Acknowledgements",
-			"value": fmt.Sprintf("%d", int(payload["RequiredAcks"].(float64))),
+			Name:     "Required Acknowledgements",
+			Value:    fmt.Sprintf("%d", int(payload["requiredAcks"].(float64))),
+			Selector: `request.payload.requiredAcks`,
 		},
 		{
-			"name":  "Timeout",
-			"value": fmt.Sprintf("%d", int(payload["Timeout"].(float64))),
+			Name:     "Timeout",
+			Value:    fmt.Sprintf("%d", int(payload["timeout"].(float64))),
+			Selector: `request.payload.timeout`,
 		},
 		{
-			"name":  "Topic Data",
-			"value": topicData,
+			Name:     "Topic Data",
+			Value:    topicData,
+			Selector: `request.payload.topicData`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -310,30 +335,32 @@ func representProduceResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	responses := ""
-	if payload["Responses"] != nil {
-		_responses, _ := json.Marshal(payload["Responses"].([]interface{}))
+	if payload["responses"] != nil {
+		_responses, _ := json.Marshal(payload["responses"].([]interface{}))
 		responses = string(_responses)
 	}
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Responses",
-			"value": string(responses),
+			Name:     "Responses",
+			Value:    string(responses),
+			Selector: `response.payload.responses`,
 		},
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -344,87 +371,97 @@ func representFetchRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		_topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		_topics, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(_topics)
 	}
 	replicaId := ""
-	if payload["ReplicaId"] != nil {
-		replicaId = fmt.Sprintf("%d", int(payload["ReplicaId"].(float64)))
+	if payload["replicaId"] != nil {
+		replicaId = fmt.Sprintf("%d", int(payload["replicaId"].(float64)))
 	}
 	maxBytes := ""
-	if payload["MaxBytes"] != nil {
-		maxBytes = fmt.Sprintf("%d", int(payload["MaxBytes"].(float64)))
+	if payload["maxBytes"] != nil {
+		maxBytes = fmt.Sprintf("%d", int(payload["maxBytes"].(float64)))
 	}
 	isolationLevel := ""
-	if payload["IsolationLevel"] != nil {
-		isolationLevel = fmt.Sprintf("%d", int(payload["IsolationLevel"].(float64)))
+	if payload["isolationLevel"] != nil {
+		isolationLevel = fmt.Sprintf("%d", int(payload["isolationLevel"].(float64)))
 	}
 	sessionId := ""
-	if payload["SessionId"] != nil {
-		sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64)))
+	if payload["sessionId"] != nil {
+		sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64)))
 	}
 	sessionEpoch := ""
-	if payload["SessionEpoch"] != nil {
-		sessionEpoch = fmt.Sprintf("%d", int(payload["SessionEpoch"].(float64)))
+	if payload["sessionEpoch"] != nil {
+		sessionEpoch = fmt.Sprintf("%d", int(payload["sessionEpoch"].(float64)))
 	}
 	forgottenTopicsData := ""
-	if payload["ForgottenTopicsData"] != nil {
-		x, _ := json.Marshal(payload["ForgottenTopicsData"].(map[string]interface{}))
+	if payload["forgottenTopicsData"] != nil {
+		x, _ := json.Marshal(payload["forgottenTopicsData"].(map[string]interface{}))
 		forgottenTopicsData = string(x)
 	}
 	rackId := ""
-	if payload["RackId"] != nil {
-		rackId = payload["RackId"].(string)
+	if payload["rackId"] != nil {
+		rackId = payload["rackId"].(string)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Replica ID",
-			"value": replicaId,
+			Name:     "Replica ID",
+			Value:    replicaId,
+			Selector: `request.payload.replicaId`,
 		},
 		{
-			"name":  "Maximum Wait (ms)",
-			"value": fmt.Sprintf("%d", int(payload["MaxWaitMs"].(float64))),
+			Name:     "Maximum Wait (ms)",
+			Value:    fmt.Sprintf("%d", int(payload["maxWaitMs"].(float64))),
+			Selector: `request.payload.maxWaitMs`,
 		},
 		{
-			"name":  "Minimum Bytes",
-			"value": fmt.Sprintf("%d", int(payload["MinBytes"].(float64))),
+			Name:     "Minimum Bytes",
+			Value:    fmt.Sprintf("%d", int(payload["minBytes"].(float64))),
+			Selector: `request.payload.minBytes`,
 		},
 		{
-			"name":  "Maximum Bytes",
-			"value": maxBytes,
+			Name:     "Maximum Bytes",
+			Value:    maxBytes,
+			Selector: `request.payload.maxBytes`,
 		},
 		{
-			"name":  "Isolation Level",
-			"value": isolationLevel,
+			Name:     "Isolation Level",
+			Value:    isolationLevel,
+			Selector: `request.payload.isolationLevel`,
 		},
 		{
-			"name":  "Session ID",
-			"value": sessionId,
+			Name:     "Session ID",
+			Value:    sessionId,
+			Selector: `request.payload.sessionId`,
 		},
 		{
-			"name":  "Session Epoch",
-			"value": sessionEpoch,
+			Name:     "Session Epoch",
+			Value:    sessionEpoch,
+			Selector: `request.payload.sessionEpoch`,
 		},
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Forgotten Topics Data",
-			"value": forgottenTopicsData,
+			Name:     "Forgotten Topics Data",
+			Value:    forgottenTopicsData,
+			Selector: `request.payload.forgottenTopicsData`,
 		},
 		{
-			"name":  "Rack ID",
-			"value": rackId,
+			Name:     "Rack ID",
+			Value:    rackId,
+			Selector: `request.payload.rackId`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -435,46 +472,50 @@ func representFetchResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	responses := ""
-	if payload["Responses"] != nil {
-		_responses, _ := json.Marshal(payload["Responses"].([]interface{}))
+	if payload["responses"] != nil {
+		_responses, _ := json.Marshal(payload["responses"].([]interface{}))
 		responses = string(_responses)
 	}
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
 	errorCode := ""
-	if payload["ErrorCode"] != nil {
-		errorCode = fmt.Sprintf("%d", int(payload["ErrorCode"].(float64)))
+	if payload["errorCode"] != nil {
+		errorCode = fmt.Sprintf("%d", int(payload["errorCode"].(float64)))
 	}
 	sessionId := ""
-	if payload["SessionId"] != nil {
-		sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64)))
+	if payload["sessionId"] != nil {
+		sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Error Code",
-			"value": errorCode,
+			Name:     "Error Code",
+			Value:    errorCode,
+			Selector: `response.payload.errorCode`,
 		},
 		{
-			"name":  "Session ID",
-			"value": sessionId,
+			Name:     "Session ID",
+			Value:    sessionId,
+			Selector: `response.payload.sessionId`,
 		},
 		{
-			"name":  "Responses",
-			"value": responses,
+			Name:     "Responses",
+			Value:    responses,
+			Selector: `response.payload.responses`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -485,26 +526,28 @@ func representListOffsetsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		_topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		_topics, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(_topics)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Replica ID",
-			"value": fmt.Sprintf("%d", int(payload["ReplicaId"].(float64))),
+			Name:     "Replica ID",
+			Value:    fmt.Sprintf("%d", int(payload["replicaId"].(float64))),
+			Selector: `request.payload.replicaId`,
 		},
 		{
-			"name":  "Topics",
-			"value": topics,
+			Name:     "Topics",
+			Value:    topics,
+			Selector: `request.payload.topics`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -515,26 +558,28 @@ func representListOffsetsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `response.payload.topics`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -545,30 +590,33 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	validateOnly := ""
-	if payload["ValidateOnly"] != nil {
-		validateOnly = strconv.FormatBool(payload["ValidateOnly"].(bool))
+	if payload["validateOnly"] != nil {
+		validateOnly = strconv.FormatBool(payload["validateOnly"].(bool))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Timeout (ms)",
-			"value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))),
+			Name:     "Timeout (ms)",
+			Value:    fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))),
+			Selector: `request.payload.timeoutMs`,
 		},
 		{
-			"name":  "Validate Only",
-			"value": validateOnly,
+			Name:     "Validate Only",
+			Value:    validateOnly,
+			Selector: `request.payload.validateOnly`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -579,26 +627,28 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["Topics"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `response.payload.topics`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -609,35 +659,38 @@ func representDeleteTopicsRequest(data map[string]interface{}) []interface{} {
 
 	rep = representRequestHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
+	payload := data["payload"].(map[string]interface{})
 	topics := ""
-	if payload["Topics"] != nil {
-		x, _ := json.Marshal(payload["Topics"].([]interface{}))
+	if payload["topics"] != nil {
+		x, _ := json.Marshal(payload["topics"].([]interface{}))
 		topics = string(x)
 	}
 	topicNames := ""
-	if payload["TopicNames"] != nil {
-		x, _ := json.Marshal(payload["TopicNames"].([]interface{}))
+	if payload["topicNames"] != nil {
+		x, _ := json.Marshal(payload["topicNames"].([]interface{}))
 		topicNames = string(x)
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "TopicNames",
-			"value": string(topicNames),
+			Name:     "TopicNames",
+			Value:    string(topicNames),
+			Selector: `request.payload.topicNames`,
 		},
 		{
-			"name":  "Topics",
-			"value": string(topics),
+			Name:     "Topics",
+			Value:    string(topics),
+			Selector: `request.payload.topics`,
 		},
 		{
-			"name":  "Timeout (ms)",
-			"value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))),
+			Name:     "Timeout (ms)",
+			Value:    fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))),
+			Selector: `request.payload.timeoutMs`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep
@@ -648,26 +701,28 @@ func representDeleteTopicsResponse(data map[string]interface{}) []interface{} {
 
 	rep = representResponseHeader(data, rep)
 
-	payload := data["Payload"].(map[string]interface{})
-	responses, _ := json.Marshal(payload["Responses"].([]interface{}))
+	payload := data["payload"].(map[string]interface{})
+	responses, _ := json.Marshal(payload["responses"].([]interface{}))
 	throttleTimeMs := ""
-	if payload["ThrottleTimeMs"] != nil {
-		throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64)))
+	if payload["throttleTimeMs"] != nil {
+		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
-	repPayload, _ := json.Marshal([]map[string]string{
+	repPayload, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Throttle Time (ms)",
-			"value": throttleTimeMs,
+			Name:     "Throttle Time (ms)",
+			Value:    throttleTimeMs,
+			Selector: `response.payload.throttleTimeMs`,
 		},
 		{
-			"name":  "Responses",
-			"value": string(responses),
+			Name:     "Responses",
+			Value:    string(responses),
+			Selector: `response.payload.responses`,
 		},
 	})
-	rep = append(rep, map[string]string{
-		"type":  api.TABLE,
-		"title": "Payload",
-		"data":  string(repPayload),
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Payload",
+		Data:  string(repPayload),
 	})
 
 	return rep

tap/extensions/kafka/main.go

@@ -15,6 +15,7 @@ var _protocol api.Protocol = api.Protocol{
 	Name:            "kafka",
 	LongName:        "Apache Kafka Protocol",
 	Abbreviation:    "KAFKA",
+	Macro:           "kafka",
 	Version:         "12",
 	BackgroundColor: "#000000",
 	ForegroundColor: "#ffffff",
@@ -36,7 +37,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", _protocol.Name)
+	log.Printf("pong %s", _protocol.Name)
 }
 
 func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
@@ -61,7 +62,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 	service := "kafka"
@@ -70,76 +71,76 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	} else if resolvedSource != "" {
 		service = resolvedSource
 	}
-	apiKey := ApiKey(reqDetails["ApiKey"].(float64))
+	apiKey := ApiKey(reqDetails["apiKey"].(float64))
 
 	summary := ""
 	switch apiKey {
 	case Metadata:
-		_topics := reqDetails["Payload"].(map[string]interface{})["Topics"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case ApiVersions:
-		summary = reqDetails["ClientID"].(string)
+		summary = reqDetails["clientID"].(string)
 		break
 	case Produce:
-		_topics := reqDetails["Payload"].(map[string]interface{})["TopicData"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topicData"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case Fetch:
-		_topics := reqDetails["Payload"].(map[string]interface{})["Topics"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case ListOffsets:
-		_topics := reqDetails["Payload"].(map[string]interface{})["Topics"]
+		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
 			break
 		}
 		topics := _topics.([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case CreateTopics:
-		topics := reqDetails["Payload"].(map[string]interface{})["Topics"].([]interface{})
+		topics := reqDetails["payload"].(map[string]interface{})["topics"].([]interface{})
 		for _, topic := range topics {
-			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string))
+			summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string))
 		}
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
 		break
 	case DeleteTopics:
-		topicNames := reqDetails["TopicNames"].([]string)
+		topicNames := reqDetails["topicNames"].([]string)
 		for _, name := range topicNames {
 			summary += fmt.Sprintf("%s, ", name)
 		}
@@ -148,44 +149,51 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 
 	request["url"] = summary
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	entryBytes, _ := json.Marshal(item.Pair)
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
 	return &api.MizuEntry{
-		ProtocolName:            _protocol.Name,
-		ProtocolLongName:        _protocol.LongName,
-		ProtocolAbbreviation:    _protocol.Abbreviation,
-		ProtocolVersion:         _protocol.Version,
-		ProtocolBackgroundColor: _protocol.BackgroundColor,
-		ProtocolForegroundColor: _protocol.ForegroundColor,
-		ProtocolFontSize:        _protocol.FontSize,
-		ProtocolReferenceLink:   _protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, summary),
-		Method:                  apiNames[apiKey],
-		Status:                  0,
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             elapsedTime,
-		Path:                    summary,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: _protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Response:            item.Pair.Response.Payload.(map[string]interface{})["details"].(map[string]interface{}),
+		Url:                 fmt.Sprintf("%s%s", service, summary),
+		Method:              apiNames[apiKey],
+		Status:              0,
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         elapsedTime,
+		Summary:             summary,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
 	}
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        _protocol,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -202,49 +210,43 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	}
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	p = _protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	protoOut = _protocol
 	bodySize = 0
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
-	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
-	reqDetails := request["details"].(map[string]interface{})
-	resDetails := response["details"].(map[string]interface{})
 
-	apiKey := ApiKey(reqDetails["ApiKey"].(float64))
+	apiKey := ApiKey(request["apiKey"].(float64))
 
 	var repRequest []interface{}
 	var repResponse []interface{}
 	switch apiKey {
 	case Metadata:
-		repRequest = representMetadataRequest(reqDetails)
-		repResponse = representMetadataResponse(resDetails)
+		repRequest = representMetadataRequest(request)
+		repResponse = representMetadataResponse(response)
 		break
 	case ApiVersions:
-		repRequest = representApiVersionsRequest(reqDetails)
-		repResponse = representApiVersionsResponse(resDetails)
+		repRequest = representApiVersionsRequest(request)
+		repResponse = representApiVersionsResponse(response)
 		break
 	case Produce:
-		repRequest = representProduceRequest(reqDetails)
-		repResponse = representProduceResponse(resDetails)
+		repRequest = representProduceRequest(request)
+		repResponse = representProduceResponse(response)
 		break
 	case Fetch:
-		repRequest = representFetchRequest(reqDetails)
-		repResponse = representFetchResponse(resDetails)
+		repRequest = representFetchRequest(request)
+		repResponse = representFetchResponse(response)
 		break
 	case ListOffsets:
-		repRequest = representListOffsetsRequest(reqDetails)
-		repResponse = representListOffsetsResponse(resDetails)
+		repRequest = representListOffsetsRequest(request)
+		repResponse = representListOffsetsResponse(response)
 		break
 	case CreateTopics:
-		repRequest = representCreateTopicsRequest(reqDetails)
-		repResponse = representCreateTopicsResponse(resDetails)
+		repRequest = representCreateTopicsRequest(request)
+		repResponse = representCreateTopicsResponse(response)
 		break
 	case DeleteTopics:
-		repRequest = representDeleteTopicsRequest(reqDetails)
-		repResponse = representDeleteTopicsResponse(resDetails)
+		repRequest = representDeleteTopicsRequest(request)
+		repResponse = representDeleteTopicsResponse(response)
 		break
 	}
 
@@ -254,4 +256,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`kafka`: fmt.Sprintf(`proto.abbr == "%s"`, _protocol.Abbreviation),
+	}
+}
+
 var Dissector dissecting

tap/extensions/kafka/request.go

@@ -10,13 +10,13 @@ import (
 )
 
 type Request struct {
-	Size          int32
-	ApiKey        ApiKey
-	ApiVersion    int16
-	CorrelationID int32
-	ClientID      string
-	Payload       interface{}
-	CaptureTime   time.Time
+	Size          int32       `json:"size"`
+	ApiKey        ApiKey      `json:"apiKey"`
+	ApiVersion    int16       `json:"apiVersion"`
+	CorrelationID int32       `json:"correlationID"`
+	ClientID      string      `json:"clientID"`
+	Payload       interface{} `json:"payload"`
+	CaptureTime   time.Time   `json:"captureTime"`
 }
 
 func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (apiKey ApiKey, apiVersion int16, err error) {

tap/extensions/kafka/response.go

@@ -10,10 +10,10 @@ import (
 )
 
 type Response struct {
-	Size          int32
-	CorrelationID int32
-	Payload       interface{}
-	CaptureTime   time.Time
+	Size          int32       `json:"size"`
+	CorrelationID int32       `json:"correlationID"`
+	Payload       interface{} `json:"payload"`
+	CaptureTime   time.Time   `json:"captureTime"`
 }
 
 func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter) (err error) {

tap/extensions/redis/helpers.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 
 	"github.com/up9inc/mizu/tap/api"
 )
@@ -24,33 +25,38 @@ type RedisWrapper struct {
 	Details interface{} `json:"details"`
 }
 
-func representGeneric(generic map[string]interface{}) (representation []interface{}) {
-	details, _ := json.Marshal([]map[string]string{
+func representGeneric(generic map[string]interface{}, selectorPrefix string) (representation []interface{}) {
+	details, _ := json.Marshal([]api.TableData{
 		{
-			"name":  "Type",
-			"value": generic["type"].(string),
+			Name:     "Type",
+			Value:    generic["type"].(string),
+			Selector: fmt.Sprintf("%stype", selectorPrefix),
 		},
 		{
-			"name":  "Command",
-			"value": generic["command"].(string),
+			Name:     "Command",
+			Value:    generic["command"].(string),
+			Selector: fmt.Sprintf("%scommand", selectorPrefix),
 		},
 		{
-			"name":  "Key",
-			"value": generic["key"].(string),
+			Name:     "Key",
+			Value:    generic["key"].(string),
+			Selector: fmt.Sprintf("%skey", selectorPrefix),
 		},
 		{
-			"name":  "Value",
-			"value": generic["value"].(string),
+			Name:     "Value",
+			Value:    generic["value"].(string),
+			Selector: fmt.Sprintf("%svalue", selectorPrefix),
 		},
 		{
-			"name":  "Keyword",
-			"value": generic["keyword"].(string),
+			Name:     "Keyword",
+			Value:    generic["keyword"].(string),
+			Selector: fmt.Sprintf("%skeyword", selectorPrefix),
 		},
 	})
-	representation = append(representation, map[string]string{
-		"type":  api.TABLE,
-		"title": "Details",
-		"data":  string(details),
+	representation = append(representation, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
 	})
 
 	return

tap/extensions/redis/main.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/up9inc/mizu/tap/api"
 )
@@ -13,6 +14,7 @@ var protocol api.Protocol = api.Protocol{
 	Name:            "redis",
 	LongName:        "Redis Serialization Protocol",
 	Abbreviation:    "REDIS",
+	Macro:           "redis",
 	Version:         "3.x",
 	BackgroundColor: "#a41e11",
 	ForegroundColor: "#ffffff",
@@ -34,7 +36,7 @@ func (d dissecting) Register(extension *api.Extension) {
 }
 
 func (d dissecting) Ping() {
-	log.Printf("pong %s\n", protocol.Name)
+	log.Printf("pong %s", protocol.Name)
 }
 
 func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
@@ -57,9 +59,12 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
+	response := item.Pair.Response.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
+	resDetails := response["details"].(map[string]interface{})
+
 	service := "redis"
 	if resolvedDestination != "" {
 		service = resolvedDestination
@@ -78,45 +83,53 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	}
 
 	request["url"] = summary
-	entryBytes, _ := json.Marshal(item.Pair)
+	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
 	return &api.MizuEntry{
-		ProtocolName:            protocol.Name,
-		ProtocolLongName:        protocol.LongName,
-		ProtocolAbbreviation:    protocol.Abbreviation,
-		ProtocolVersion:         protocol.Version,
-		ProtocolBackgroundColor: protocol.BackgroundColor,
-		ProtocolForegroundColor: protocol.ForegroundColor,
-		ProtocolFontSize:        protocol.FontSize,
-		ProtocolReferenceLink:   protocol.ReferenceLink,
-		EntryId:                 entryId,
-		Entry:                   string(entryBytes),
-		Url:                     fmt.Sprintf("%s%s", service, summary),
-		Method:                  method,
-		Status:                  0,
-		RequestSenderIp:         item.ConnectionInfo.ClientIP,
-		Service:                 service,
-		Timestamp:               item.Timestamp,
-		ElapsedTime:             0,
-		Path:                    summary,
-		ResolvedSource:          resolvedSource,
-		ResolvedDestination:     resolvedDestination,
-		SourceIp:                item.ConnectionInfo.ClientIP,
-		DestinationIp:           item.ConnectionInfo.ServerIP,
-		SourcePort:              item.ConnectionInfo.ClientPort,
-		DestinationPort:         item.ConnectionInfo.ServerPort,
-		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+		Protocol: protocol,
+		Source: &api.TCP{
+			Name: resolvedSource,
+			IP:   item.ConnectionInfo.ClientIP,
+			Port: item.ConnectionInfo.ClientPort,
+		},
+		Destination: &api.TCP{
+			Name: resolvedDestination,
+			IP:   item.ConnectionInfo.ServerIP,
+			Port: item.ConnectionInfo.ServerPort,
+		},
+		Outgoing:            item.ConnectionInfo.IsOutgoing,
+		Request:             reqDetails,
+		Response:            resDetails,
+		Url:                 fmt.Sprintf("%s%s", service, summary),
+		Method:              method,
+		Status:              0,
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		StartTime:           item.Pair.Request.CaptureTime,
+		ElapsedTime:         elapsedTime,
+		Summary:             summary,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
 	}
 
 }
 
 func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	return &api.BaseEntryDetails{
-		Id:              entry.EntryId,
+		Id:              entry.Id,
 		Protocol:        protocol,
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
-		Summary:         entry.Path,
+		Summary:         entry.Summary,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
 		Timestamp:       entry.Timestamp,
@@ -133,22 +146,22 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 	}
 }
 
-func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
-	p = protocol
+func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) {
+	protoOut = protocol
 	bodySize = 0
-	var root map[string]interface{}
-	json.Unmarshal([]byte(entry.Entry), &root)
 	representation := make(map[string]interface{}, 0)
-	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
-	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
-	reqDetails := request["details"].(map[string]interface{})
-	resDetails := response["details"].(map[string]interface{})
-	repRequest := representGeneric(reqDetails)
-	repResponse := representGeneric(resDetails)
+	repRequest := representGeneric(request, `request.`)
+	repResponse := representGeneric(response, `response.`)
 	representation["request"] = repRequest
 	representation["response"] = repResponse
 	object, err = json.Marshal(representation)
 	return
 }
 
+func (d dissecting) Macros() map[string]string {
+	return map[string]string{
+		`redis`: fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation),
+	}
+}
+
 var Dissector dissecting

tap/extensions/redis/read.go

@@ -313,7 +313,7 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
 				packet.Value = fmt.Sprintf("%s]", packet.Value)
 			}
 		default:
-			msg := fmt.Sprintf("Unrecognized element in Redis array: %v\n", reflect.TypeOf(array[0]))
+			msg := fmt.Sprintf("Unrecognized element in Redis array: %v", reflect.TypeOf(array[0]))
 			err = errors.New(msg)
 			return
 		}
@@ -333,7 +333,7 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
 	case int64:
 		packet.Value = fmt.Sprintf("%d", x.(int64))
 	default:
-		msg := fmt.Sprintf("Unrecognized Redis data type: %v\n", reflect.TypeOf(x))
+		msg := fmt.Sprintf("Unrecognized Redis data type: %v", reflect.TypeOf(x))
 		err = errors.New(msg)
 		return
 	}

tap/passive_tapper.go

@@ -50,14 +50,15 @@ var tstype = flag.String("timestamp_type", "", "Type of timestamps to use")
 var promisc = flag.Bool("promisc", true, "Set promiscuous mode")
 var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to keep connections which don't transmit data")
 var pids = flag.String("pids", "", "A comma separated list of PIDs to capture their network namespaces")
+var istio = flag.Bool("istio", false, "Record decrypted traffic if the cluster configured with istio and mtls")
 
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
 type TapOpts struct {
-	HostMode bool
+	HostMode          bool
+	FilterAuthorities []string
 }
 
-var hostMode bool                                 // global
 var extensions []*api.Extension                   // global
 var filteringOptions *api.TrafficFilteringOptions // global
 
@@ -80,15 +81,18 @@ func inArrayString(arr []string, valueToCheck string) bool {
 }
 
 func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *api.TrafficFilteringOptions) {
-	hostMode = opts.HostMode
 	extensions = extensionsRef
 	filteringOptions = options
 
+	if opts.FilterAuthorities == nil {
+		opts.FilterAuthorities = []string{}
+	}
+
 	if GetMemoryProfilingEnabled() {
 		diagnose.StartMemoryProfiler(os.Getenv(MemoryProfilingDumpPath), os.Getenv(MemoryProfilingTimeIntervalSeconds))
 	}
 
-	go startPassiveTapper(outputItems)
+	go startPassiveTapper(opts, outputItems)
 }
 
 func printPeriodicStats(cleaner *Cleaner) {
@@ -131,7 +135,7 @@ func printPeriodicStats(cleaner *Cleaner) {
 	}
 }
 
-func initializePacketSources() (*source.PacketSourceManager, error) {
+func initializePacketSources(opts *TapOpts) (*source.PacketSourceManager, error) {
 	var bpffilter string
 	if len(flag.Args()) > 0 {
 		bpffilter = strings.Join(flag.Args(), " ")
@@ -146,17 +150,17 @@ func initializePacketSources() (*source.PacketSourceManager, error) {
 		BpfFilter:   bpffilter,
 	}
 
-	return source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, behaviour)
+	return source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, *istio, opts.FilterAuthorities, behaviour)
 }
 
-func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
+func startPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) {
 	streamsMap := NewTcpStreamMap()
 	go streamsMap.closeTimedoutTcpStreamChannels()
 
 	diagnose.InitializeErrorsMap(*debug, *verbose, *quiet)
 	diagnose.InitializeTapperInternalStats()
 
-	sources, err := initializePacketSources()
+	sources, err := initializePacketSources(opts)
 
 	if err != nil {
 		logger.Log.Fatal(err)
@@ -169,7 +173,7 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	}
 
 	packets := make(chan source.TcpPacketInfo)
-	assembler := NewTcpAssembler(outputItems, streamsMap)
+	assembler := NewTcpAssembler(outputItems, streamsMap, opts)
 
 	diagnose.AppStats.SetStartTime(time.Now())
 
@@ -193,7 +197,7 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	}
 
 	if err := diagnose.DumpMemoryProfile(*memprofile); err != nil {
-		logger.Log.Errorf("Error dumping memory profile %v\n", err)
+		logger.Log.Errorf("Error dumping memory profile %v", err)
 	}
 
 	assembler.waitAndDump()

tap/settings.go

@@ -18,24 +18,6 @@ const (
 	TcpStreamChannelTimeoutMsDefaultValue     = 10000
 )
 
-type globalSettings struct {
-	filterAuthorities []string
-}
-
-var gSettings = &globalSettings{
-	filterAuthorities: []string{},
-}
-
-func SetFilterAuthorities(ipAddresses []string) {
-	gSettings.filterAuthorities = ipAddresses
-}
-
-func GetFilterIPs() []string {
-	addresses := make([]string, len(gSettings.filterAuthorities))
-	copy(addresses, gSettings.filterAuthorities)
-	return addresses
-}
-
 func GetMaxBufferedPagesTotal() int {
 	valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesTotalEnvVarName))
 	if err != nil {

tap/source/envoy_discoverer.go

@@ -0,0 +1,112 @@
+package source
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+const envoyBinary = "/envoy"
+
+var numberRegex = regexp.MustCompile("[0-9]+")
+
+func discoverRelevantEnvoyPids(procfs string, clusterIps []string) ([]string, error) {
+	result := make([]string, 0)
+
+	pids, err := ioutil.ReadDir(procfs)
+
+	if err != nil {
+		return result, err
+	}
+
+	logger.Log.Infof("Starting envoy auto discoverer %v %v - scanning %v potential pids",
+		procfs, clusterIps, len(pids))
+
+	for _, pid := range pids {
+		if !pid.IsDir() {
+			continue
+		}
+
+		if !numberRegex.MatchString(pid.Name()) {
+			continue
+		}
+
+		if checkPid(procfs, pid.Name(), clusterIps) {
+			result = append(result, pid.Name())
+		}
+	}
+
+	logger.Log.Infof("Found %v relevant envoy processes - %v", len(result), result)
+
+	return result, nil
+}
+
+func checkPid(procfs string, pid string, clusterIps []string) bool {
+	execLink := fmt.Sprintf("%v/%v/exe", procfs, pid)
+	exec, err := os.Readlink(execLink)
+
+	if err != nil {
+		// Debug on purpose - it may happen due to many reasons and we only care
+		//	for it during troubleshooting
+		//
+		logger.Log.Debugf("Unable to read link %v - %v\n", execLink, err)
+		return false
+	}
+
+	if !strings.HasSuffix(exec, envoyBinary) {
+		return false
+	}
+
+	environmentFile := fmt.Sprintf("%v/%v/environ", procfs, pid)
+	clusterIp, err := readEnvironmentVariable(environmentFile, "INSTANCE_IP")
+
+	if err != nil {
+		return false
+	}
+
+	if clusterIp == "" {
+		logger.Log.Debugf("Found an envoy process without INSTANCE_IP variable %v\n", pid)
+		return false
+	}
+
+	logger.Log.Infof("Found envoy pid %v with cluster ip %v", pid, clusterIp)
+
+	for _, value := range clusterIps {
+		if value == clusterIp {
+			return true
+		}
+	}
+
+	return false
+}
+
+func readEnvironmentVariable(file string, name string) (string, error) {
+	bytes, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		logger.Log.Warningf("Error reading environment file %v - %v", file, err)
+		return "", err
+	}
+
+	envs := strings.Split(string(bytes), string([]byte{0}))
+
+	for _, env := range envs {
+		if !strings.Contains(env, "=") {
+			continue
+		}
+
+		parts := strings.Split(env, "=")
+		varName := parts[0]
+		value := parts[1]
+
+		if name == varName {
+			return value, nil
+		}
+	}
+
+	return "", nil
+}

tap/source/packet_source_manager.go

@@ -15,26 +15,63 @@ type PacketSourceManager struct {
 }
 
 func NewPacketSourceManager(procfs string, pids string, filename string, interfaceName string,
-	behaviour TcpPacketSourceBehaviour) (*PacketSourceManager, error) {
+	istio bool, clusterIps []string, behaviour TcpPacketSourceBehaviour) (*PacketSourceManager, error) {
 	sources := make([]*tcpPacketSource, 0)
-	hostSource, err := newHostPacketSource(filename, interfaceName, behaviour)
+	sources, err := createHostSource(sources, filename, interfaceName, behaviour)
 
 	if err != nil {
 		return nil, err
 	}
 
-	sources = append(sources, hostSource)
-
-	if pids != "" {
-		netnsSources := newNetnsPacketSources(procfs, pids, interfaceName, behaviour)
-		sources = append(sources, netnsSources...)
-	}
+	sources = createSourcesFromPids(sources, procfs, pids, interfaceName, behaviour)
+	sources = createSourcesFromEnvoy(sources, istio, procfs, clusterIps, interfaceName, behaviour)
 
 	return &PacketSourceManager{
 		sources: sources,
 	}, nil
 }
 
+func createHostSource(sources []*tcpPacketSource, filename string, interfaceName string,
+	behaviour TcpPacketSourceBehaviour) ([]*tcpPacketSource, error) {
+	hostSource, err := newHostPacketSource(filename, interfaceName, behaviour)
+
+	if err != nil {
+		return sources, err
+	}
+
+	return append(sources, hostSource), nil
+}
+
+func createSourcesFromPids(sources []*tcpPacketSource, procfs string, pids string,
+	interfaceName string, behaviour TcpPacketSourceBehaviour) []*tcpPacketSource {
+	if pids == "" {
+		return sources
+	}
+
+	netnsSources := newNetnsPacketSources(procfs, strings.Split(pids, ","), interfaceName, behaviour)
+	sources = append(sources, netnsSources...)
+	return sources
+}
+
+func createSourcesFromEnvoy(sources []*tcpPacketSource, istio bool, procfs string, clusterIps []string,
+	interfaceName string, behaviour TcpPacketSourceBehaviour) []*tcpPacketSource {
+	if !istio {
+		return sources
+	}
+
+	envoyPids, err := discoverRelevantEnvoyPids(procfs, clusterIps)
+
+	if err != nil {
+		logger.Log.Warningf("Unable to discover envoy pids - %v", err)
+		return sources
+	}
+
+	netnsSources := newNetnsPacketSources(procfs, envoyPids, interfaceName, behaviour)
+	sources = append(sources, netnsSources...)
+
+	return sources
+}
+
 func newHostPacketSource(filename string, interfaceName string,
 	behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) {
 	var name string
@@ -54,11 +91,11 @@ func newHostPacketSource(filename string, interfaceName string,
 	return source, nil
 }
 
-func newNetnsPacketSources(procfs string, pids string, interfaceName string,
+func newNetnsPacketSources(procfs string, pids []string, interfaceName string,
 	behaviour TcpPacketSourceBehaviour) []*tcpPacketSource {
 	result := make([]*tcpPacketSource, 0)
 
-	for _, pidstr := range strings.Split(pids, ",") {
+	for _, pidstr := range pids {
 		pid, err := strconv.Atoi(pidstr)
 
 		if err != nil {
@@ -100,9 +137,9 @@ func newNetnsPacketSource(pid int, nsh netns.NsHandle, interfaceName string,
 		//
 		runtime.LockOSThread()
 		defer runtime.UnlockOSThread()
-		
+
 		oldnetns, err := netns.Get()
-		
+
 		if err != nil {
 			logger.Log.Errorf("Unable to get netns of current thread %v", err)
 			errors <- err

tap/tcp_assembler.go

@@ -33,13 +33,13 @@ func (c *context) GetCaptureInfo() gopacket.CaptureInfo {
 	return c.CaptureInfo
 }
 
-func NewTcpAssembler(outputItems chan *api.OutputChannelItem, streamsMap *tcpStreamMap) *tcpAssembler {
+func NewTcpAssembler(outputItems chan *api.OutputChannelItem, streamsMap *tcpStreamMap, opts *TapOpts) *tcpAssembler {
 	var emitter api.Emitter = &api.Emitting{
 		AppStats:      &diagnose.AppStats,
 		OutputChannel: outputItems,
 	}
 
-	streamFactory := NewTcpStreamFactory(emitter, streamsMap)
+	streamFactory := NewTcpStreamFactory(emitter, streamsMap, opts)
 	streamPool := reassembly.NewStreamPool(streamFactory)
 	assembler := reassembly.NewAssembler(streamPool)
 
@@ -78,7 +78,7 @@ func (a *tcpAssembler) processPackets(dumpPacket bool, packets <-chan source.Tcp
 			if *checksum {
 				err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
 				if err != nil {
-					logger.Log.Fatalf("Failed to set network layer for checksum: %s\n", err)
+					logger.Log.Fatalf("Failed to set network layer for checksum: %s", err)
 				}
 			}
 			c := context{

tap/tcp_reader.go

@@ -66,7 +66,7 @@ func (h *tcpReader) Read(p []byte) (int, error) {
 			clientHello := tlsx.ClientHello{}
 			err := clientHello.Unmarshall(msg.bytes)
 			if err == nil {
-				logger.Log.Debugf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
+				logger.Log.Debugf("Detected TLS client hello with SNI %s", clientHello.SNI)
 				// TODO: Throws `panic: runtime error: invalid memory address or nil pointer dereference` error.
 				// numericPort, _ := strconv.Atoi(h.tcpID.DstPort)
 				// h.outboundLinkWriter.WriteOutboundLink(h.tcpID.SrcIP, h.tcpID.DstIP, numericPort, clientHello.SNI, TLSProtocol)

tap/tcp_stream_factory.go

@@ -24,6 +24,7 @@ type tcpStreamFactory struct {
 	Emitter            api.Emitter
 	streamsMap         *tcpStreamMap
 	ownIps             []string
+	opts               *TapOpts
 }
 
 type tcpStreamWrapper struct {
@@ -31,7 +32,7 @@ type tcpStreamWrapper struct {
 	createdAt time.Time
 }
 
-func NewTcpStreamFactory(emitter api.Emitter, streamsMap *tcpStreamMap) *tcpStreamFactory {
+func NewTcpStreamFactory(emitter api.Emitter, streamsMap *tcpStreamMap, opts *TapOpts) *tcpStreamFactory {
 	var ownIps []string
 
 	if localhostIPs, err := getLocalhostIPs(); err != nil {
@@ -47,6 +48,7 @@ func NewTcpStreamFactory(emitter api.Emitter, streamsMap *tcpStreamMap) *tcpStre
 		Emitter:    emitter,
 		streamsMap: streamsMap,
 		ownIps:     ownIps,
+		opts:       opts,
 	}
 }
 
@@ -139,17 +141,17 @@ func (factory *tcpStreamFactory) WaitGoRoutines() {
 }
 
 func (factory *tcpStreamFactory) getStreamProps(srcIP string, srcPort string, dstIP string, dstPort string) *streamProps {
-	if hostMode {
-		if inArrayString(gSettings.filterAuthorities, fmt.Sprintf("%s:%s", dstIP, dstPort)) {
+	if factory.opts.HostMode {
+		if inArrayString(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", dstIP, dstPort)) {
 			logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host1 %s:%s", dstIP, dstPort))
 			return &streamProps{isTapTarget: true, isOutgoing: false}
-		} else if inArrayString(gSettings.filterAuthorities, dstIP) {
+		} else if inArrayString(factory.opts.FilterAuthorities, dstIP) {
 			logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host2 %s", dstIP))
 			return &streamProps{isTapTarget: true, isOutgoing: false}
-		} else if inArrayString(gSettings.filterAuthorities, fmt.Sprintf("%s:%s", srcIP, srcPort)) {
+		} else if inArrayString(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", srcIP, srcPort)) {
 			logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host3 %s:%s", srcIP, srcPort))
 			return &streamProps{isTapTarget: true, isOutgoing: true}
-		} else if inArrayString(gSettings.filterAuthorities, srcIP) {
+		} else if inArrayString(factory.opts.FilterAuthorities, srcIP) {
 			logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host4 %s", srcIP))
 			return &streamProps{isTapTarget: true, isOutgoing: true}
 		}

tap/tcp_streams_map.go

@@ -46,7 +46,7 @@ func (streamMap *tcpStreamMap) closeTimedoutTcpStreamChannels() {
 				if !stream.isClosed && time.Now().After(streamWrapper.createdAt.Add(tcpStreamChannelTimeout)) {
 					stream.Close()
 					diagnose.AppStats.IncDroppedTcpStreams()
-					logger.Log.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d\n",
+					logger.Log.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d",
 						diagnose.AppStats.DroppedTcpStreams, runtime.NumGoroutine(), tcpStreamChannelTimeout/1000000)
 				}
 			} else {

tap/tester/tester.go

@@ -33,7 +33,7 @@ func loadExtensions() ([]*tapApi.Extension, error) {
 			continue
 		}
 
-		logger.Log.Infof("Loading extension: %s\n", filename)
+		logger.Log.Infof("Loading extension: %s", filename)
 
 		extension := &tapApi.Extension{
 			Path: path.Join(extensionsDir, filename),
@@ -55,7 +55,7 @@ func loadExtensions() ([]*tapApi.Extension, error) {
 		dissector, ok := symDissector.(tapApi.Dissector)
 
 		if !ok {
-			return nil, errors.Errorf("Symbol Dissector type error: %v %T\n", file, symDissector)
+			return nil, errors.Errorf("Symbol Dissector type error: %v %T", file, symDissector)
 		}
 
 		dissector.Register(extension)
@@ -68,7 +68,7 @@ func loadExtensions() ([]*tapApi.Extension, error) {
 	})
 
 	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v\n", extension)
+		logger.Log.Infof("Extension Properties: %+v", extension)
 	}
 
 	return extensions, nil
@@ -92,7 +92,7 @@ func internalRun() error {
 
 	tap.StartPassiveTapper(&opts, outputItems, extenssions, &tapOpts)
 
-	logger.Log.Infof("Tapping, press enter to exit...\n")
+	logger.Log.Infof("Tapping, press enter to exit...")
 	reader := bufio.NewReader(os.Stdin)
 	reader.ReadLine()
 	return nil
@@ -104,9 +104,9 @@ func main() {
 	if err != nil {
 		switch err := err.(type) {
 		case *errors.Error:
-			logger.Log.Errorf("Error: %v\n", err.ErrorStack())
+			logger.Log.Errorf("Error: %v", err.ErrorStack())
 		default:
-			logger.Log.Errorf("Error: %v\n", err)
+			logger.Log.Errorf("Error: %v", err)
 		}
 
 		os.Exit(1)

ui/.snyk

@@ -133,3 +133,6 @@ ignore:
   SNYK-JS-WS-1296835:
     - '*':
         reason: Non given
+  SNYK-JS-JSONSCHEMA-1920922:
+    - '*':
+        reason: Non given

ui/package-lock.json

@@ -1769,6 +1769,33 @@
         }
       }
     },
+    "@mapbox/rehype-prism": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/@mapbox/rehype-prism/-/rehype-prism-0.7.0.tgz",
+      "integrity": "sha512-zSG46selA6v+3THhCatTyOt9DuTzxTIVTxTbcj15kFmxPDtjzZ5VoFVCLZfjWFouYa9PiXxcbMLLhJoVzCxh9w==",
+      "requires": {
+        "hast-util-to-string": "^1.0.4",
+        "refractor": "^3.4.0",
+        "unist-util-visit": "^2.0.3"
+      },
+      "dependencies": {
+        "prismjs": {
+          "version": "1.24.1",
+          "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.24.1.tgz",
+          "integrity": "sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow=="
+        },
+        "refractor": {
+          "version": "3.4.0",
+          "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.4.0.tgz",
+          "integrity": "sha512-dBeD02lC5eytm9Gld2Mx0cMcnR+zhSnsTfPpWqFaMgUMJfC9A6bcN3Br/NaXrnBJcuxnLFR90k1jrkaSyV8umg==",
+          "requires": {
+            "hastscript": "^6.0.0",
+            "parse-entities": "^2.0.0",
+            "prismjs": "~1.24.0"
+          }
+        }
+      }
+    },
     "@material-ui/core": {
       "version": "4.12.2",
       "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.2.tgz",
@@ -1788,6 +1815,14 @@
         "react-transition-group": "^4.4.0"
       }
     },
+    "@material-ui/icons": {
+      "version": "4.11.2",
+      "resolved": "https://registry.npmjs.org/@material-ui/icons/-/icons-4.11.2.tgz",
+      "integrity": "sha512-fQNsKX2TxBmqIGJCSi3tGTO/gZ+eJgWmMJkgDiOfyNaunNaxcklJQFaFogYcFl0qFuaEz1qaXYXboa/bUXVSOQ==",
+      "requires": {
+        "@babel/runtime": "^7.4.4"
+      }
+    },
     "@material-ui/lab": {
       "version": "4.0.0-alpha.60",
       "resolved": "https://registry.npmjs.org/@material-ui/lab/-/lab-4.0.0-alpha.60.tgz",
@@ -2406,6 +2441,11 @@
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
       "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA=="
     },
+    "@types/parse5": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@types/parse5/-/parse5-6.0.1.tgz",
+      "integrity": "sha512-ARATsLdrGPUnaBvxLhUlnltcMgn7pQG312S8ccdYlnyijabrX9RN/KN/iGj9Am96CoW8e/K9628BA7Bv4XHdrA=="
+    },
     "@types/prettier": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.2.3.tgz",
@@ -2629,6 +2669,26 @@
         "eslint-visitor-keys": "^2.0.0"
       }
     },
+    "@uiw/react-textarea-code-editor": {
+      "version": "1.4.12",
+      "resolved": "https://registry.npmjs.org/@uiw/react-textarea-code-editor/-/react-textarea-code-editor-1.4.12.tgz",
+      "integrity": "sha512-op0aIRxX8hLi8OLwm/23dQ2X4o2xHUoK2pLr1DWBFgNbIh4L+RM5zByNFjV8mHfi/NvE6mQLB6LtMd1qaor5MQ==",
+      "requires": {
+        "@babel/runtime": "7.15.4",
+        "@mapbox/rehype-prism": "0.7.0",
+        "rehype": "12.0.0"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.15.4",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.15.4.tgz",
+          "integrity": "sha512-99catp6bHCaxr4sJ/DbTGgHS4+Rs2RVd2g7iOap6SLGPDknRK9ztKNsE/Fg6QhSeh1FGE5f6gHGQmvvn3I3xhw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
+      }
+    },
     "@webassemblyjs/ast": {
       "version": "1.9.0",
       "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz",
@@ -3689,6 +3749,11 @@
       "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz",
       "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ=="
     },
+    "bail": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.1.tgz",
+      "integrity": "sha512-d5FoTAr2S5DSUPKl85WNm2yUwsINN8eidIdIwsOge2t33DaOfOdSmmsI11jMN3GmALCXaw+Y6HMVHDzePshFAA=="
+    },
     "balanced-match": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -4179,6 +4244,11 @@
       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
+    "ccount": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.0.tgz",
+      "integrity": "sha512-VOR0NWFYX65n9gELQdcpqsie5L5ihBXuZGAgaPEp/U7IOSjnPMEH6geE+2f6lcekaNEfWzAHS45mPvSo5bqsUA=="
+    },
     "chalk": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -4199,6 +4269,11 @@
       "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz",
       "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw=="
     },
+    "character-entities-html4": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.0.0.tgz",
+      "integrity": "sha512-dwT2xh5ZhUAjyP96k57ilMKoTQyASaw9IAMR9U5c1lCu2RUni6O6jxfpUEdO2RcPT6TJFvr8pqsbami4Jk+2oA=="
+    },
     "character-entities-legacy": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz",
@@ -7534,11 +7609,121 @@
         "minimalistic-assert": "^1.0.1"
       }
     },
+    "hast-util-from-parse5": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-7.1.0.tgz",
+      "integrity": "sha512-m8yhANIAccpU4K6+121KpPP55sSl9/samzQSQGpb0mTExcNh2WlvjtMwSWFhg6uqD4Rr6Nfa8N6TMypQM51rzQ==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "@types/parse5": "^6.0.0",
+        "@types/unist": "^2.0.0",
+        "hastscript": "^7.0.0",
+        "property-information": "^6.0.0",
+        "vfile": "^5.0.0",
+        "vfile-location": "^4.0.0",
+        "web-namespaces": "^2.0.0"
+      },
+      "dependencies": {
+        "comma-separated-tokens": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz",
+          "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg=="
+        },
+        "hast-util-parse-selector": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.0.tgz",
+          "integrity": "sha512-AyjlI2pTAZEOeu7GeBPZhROx0RHBnydkQIXlhnFzDi0qfXTmGUWoCYZtomHbrdrheV4VFUlPcfJ6LMF5T6sQzg==",
+          "requires": {
+            "@types/hast": "^2.0.0"
+          }
+        },
+        "hastscript": {
+          "version": "7.0.2",
+          "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-7.0.2.tgz",
+          "integrity": "sha512-uA8ooUY4ipaBvKcMuPehTAB/YfFLSSzCwFSwT6ltJbocFUKH/GDHLN+tflq7lSRf9H86uOuxOFkh1KgIy3Gg2g==",
+          "requires": {
+            "@types/hast": "^2.0.0",
+            "comma-separated-tokens": "^2.0.0",
+            "hast-util-parse-selector": "^3.0.0",
+            "property-information": "^6.0.0",
+            "space-separated-tokens": "^2.0.0"
+          }
+        },
+        "property-information": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz",
+          "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg=="
+        },
+        "space-separated-tokens": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz",
+          "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw=="
+        }
+      }
+    },
+    "hast-util-is-element": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-2.1.1.tgz",
+      "integrity": "sha512-ag0fiZfRWsPiR1udvnSbaazJLGv8qd8E+/e3rW8rUZhbKG4HNJmFL4QkEceN+22BgE+uozXY30z/s+2dL6Z++g==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "@types/unist": "^2.0.0"
+      }
+    },
     "hast-util-parse-selector": {
       "version": "2.2.5",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz",
       "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ=="
     },
+    "hast-util-to-html": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-8.0.2.tgz",
+      "integrity": "sha512-ipLhUTMyyJi9F/LXaNDG9BrRdshP6obCfmUZYbE/+T639IdzqAOkKN4DyrEyID0gbb+rsC3PKf0XlviZwzomhw==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "ccount": "^2.0.0",
+        "comma-separated-tokens": "^2.0.0",
+        "hast-util-is-element": "^2.0.0",
+        "hast-util-whitespace": "^2.0.0",
+        "html-void-elements": "^2.0.0",
+        "property-information": "^6.0.0",
+        "space-separated-tokens": "^2.0.0",
+        "stringify-entities": "^4.0.0",
+        "unist-util-is": "^5.0.0"
+      },
+      "dependencies": {
+        "comma-separated-tokens": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz",
+          "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg=="
+        },
+        "property-information": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz",
+          "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg=="
+        },
+        "space-separated-tokens": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz",
+          "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw=="
+        },
+        "unist-util-is": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.1.1.tgz",
+          "integrity": "sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ=="
+        }
+      }
+    },
+    "hast-util-to-string": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-1.0.4.tgz",
+      "integrity": "sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w=="
+    },
+    "hast-util-whitespace": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.0.tgz",
+      "integrity": "sha512-Pkw+xBHuV6xFeJprJe2BBEoDV+AvQySaz3pPDRUs5PNZEMQjpXJJueqrpcHIXxnWTcAGi/UOCgVShlkY6kLoqg=="
+    },
     "hastscript": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz",
@@ -7671,6 +7856,11 @@
         "terser": "^4.6.3"
       }
     },
+    "html-void-elements": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-2.0.0.tgz",
+      "integrity": "sha512-4OYzQQsBt0G9bJ/nM9/DDsjm4+fVdzAaPJJcWk5QwA3GIAPxQEeOR0rsI8HbDHQz5Gta8pVvGnnTNSbZVEVvkQ=="
+    },
     "html-webpack-plugin": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-4.5.0.tgz",
@@ -13454,9 +13644,9 @@
       }
     },
     "react-scrollable-feed-virtualized": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.3.tgz",
-      "integrity": "sha512-M9WgJKr57jCyWKNCksc3oi+xhtO0YbL9d7Ll8Sdc5ZWOIstNvdNbNX0k4Nq6kXUVaHCJ9qE8omdSI/CxT3MLAQ=="
+      "version": "1.4.8",
+      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.8.tgz",
+      "integrity": "sha512-zsSO/9QB+4V6HEk39lxeMEUA6JFSZjfV4stw7RF17+vZdlVhyATsTBCzsj8hZywY4F29cBfH+3/GKrMhwmhAsw=="
     },
     "react-syntax-highlighter": {
       "version": "15.4.3",
@@ -13470,6 +13660,14 @@
         "refractor": "^3.2.0"
       }
     },
+    "react-toastify": {
+      "version": "8.0.3",
+      "resolved": "https://registry.npmjs.org/react-toastify/-/react-toastify-8.0.3.tgz",
+      "integrity": "sha512-rv3koC7f9lKKSkdpYgo/TGzgWlrB/aaiUInF1DyV7BpiM4kyTs+uhu6/r8XDMtBY2FOIHK+FlK3Iv7OzpA/tCA==",
+      "requires": {
+        "clsx": "^1.1.1"
+      }
+    },
     "react-transition-group": {
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz",
@@ -13693,6 +13891,38 @@
         }
       }
     },
+    "rehype": {
+      "version": "12.0.0",
+      "resolved": "https://registry.npmjs.org/rehype/-/rehype-12.0.0.tgz",
+      "integrity": "sha512-gZcttmf9R5IYHb8AlI1rlmWqXS1yX0rSB/S5ZGJs8atfYZy2DobvH3Ic/gSzB+HL/+oOHPtBguw1TprfhxXBgQ==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "rehype-parse": "^8.0.0",
+        "rehype-stringify": "^9.0.0",
+        "unified": "^10.0.0"
+      }
+    },
+    "rehype-parse": {
+      "version": "8.0.3",
+      "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-8.0.3.tgz",
+      "integrity": "sha512-RGw0CVt+0S6KdvpE8bbP2Db9WXclQcIX7A0ufM3QFqAhTo/ddJMQrrI2j3cijlRPZlGK8R3pRgC8U5HyV76IDw==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "hast-util-from-parse5": "^7.0.0",
+        "parse5": "^6.0.0",
+        "unified": "^10.0.0"
+      }
+    },
+    "rehype-stringify": {
+      "version": "9.0.2",
+      "resolved": "https://registry.npmjs.org/rehype-stringify/-/rehype-stringify-9.0.2.tgz",
+      "integrity": "sha512-BuVA6lAEYtOpXO2xuHLohAzz8UNoQAxAqYRqh4QEEtU39Co+P1JBZhw6wXA9hMWp+JLcmrxWH8+UKcNSr443Fw==",
+      "requires": {
+        "@types/hast": "^2.0.0",
+        "hast-util-to-html": "^8.0.0",
+        "unified": "^10.0.0"
+      }
+    },
     "relateurl": {
       "version": "0.2.7",
       "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz",
@@ -15342,6 +15572,22 @@
         }
       }
     },
+    "stringify-entities": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.1.tgz",
+      "integrity": "sha512-gmMQxKXPWIO3NXNSPyWNhlYcBNGpPA/487D+9dLPnU4xBnIrnHdr8cv5rGJOS/1BRxEXRb7uKwg7BA36IWV7xg==",
+      "requires": {
+        "character-entities-html4": "^2.0.0",
+        "character-entities-legacy": "^2.0.0"
+      },
+      "dependencies": {
+        "character-entities-legacy": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-2.0.0.tgz",
+          "integrity": "sha512-YwaEtEvWLpFa6Wh3uVLrvirA/ahr9fki/NUd/Bd4OR6EdJ8D22hovYQEOUCBfQfcqnC4IAMGMsHXY1eXgL4ZZA=="
+        }
+      }
+    },
     "stringify-object": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz",
@@ -15877,6 +16123,11 @@
       "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
       "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM="
     },
+    "trough": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/trough/-/trough-2.0.2.tgz",
+      "integrity": "sha512-FnHq5sTMxC0sk957wHDzRnemFnNBvt/gSY99HzK8F7UP5WAbvP70yX5bd7CjEQkN+TjdxwI7g7lJ6podqrG2/w=="
+    },
     "true-case-path": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz",
@@ -16037,6 +16288,32 @@
       "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz",
       "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg=="
     },
+    "unified": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.0.tgz",
+      "integrity": "sha512-4U3ru/BRXYYhKbwXV6lU6bufLikoAavTwev89H5UxY8enDFaAT2VXmIXYNm6hb5oHPng/EXr77PVyDFcptbk5g==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "bail": "^2.0.0",
+        "extend": "^3.0.0",
+        "is-buffer": "^2.0.0",
+        "is-plain-obj": "^4.0.0",
+        "trough": "^2.0.0",
+        "vfile": "^5.0.0"
+      },
+      "dependencies": {
+        "is-buffer": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
+          "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ=="
+        },
+        "is-plain-obj": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.0.0.tgz",
+          "integrity": "sha512-NXRbBtUdBioI73y/HmOhogw/U5msYPC9DAtGkJXeFcFWSFZw0mCUsPxk/snTuJHzNKA8kLBK4rH97RMB1BfCXw=="
+        }
+      }
+    },
     "union-value": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
@@ -16082,6 +16359,38 @@
         "crypto-random-string": "^1.0.0"
       }
     },
+    "unist-util-is": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz",
+      "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg=="
+    },
+    "unist-util-stringify-position": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.0.tgz",
+      "integrity": "sha512-SdfAl8fsDclywZpfMDTVDxA2V7LjtRDTOFd44wUJamgl6OlVngsqWjxvermMYf60elWHbxhuRCZml7AnuXCaSA==",
+      "requires": {
+        "@types/unist": "^2.0.0"
+      }
+    },
+    "unist-util-visit": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz",
+      "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "unist-util-is": "^4.0.0",
+        "unist-util-visit-parents": "^3.0.0"
+      }
+    },
+    "unist-util-visit-parents": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz",
+      "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "unist-util-is": "^4.0.0"
+      }
+    },
     "universalify": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
@@ -16306,6 +16615,42 @@
         "extsprintf": "^1.2.0"
       }
     },
+    "vfile": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.1.0.tgz",
+      "integrity": "sha512-4o7/DJjEaFPYSh0ckv5kcYkJTHQgCKdL8ozMM1jLAxO9ox95IzveDPXCZp08HamdWq8JXTkClDvfAKaeLQeKtg==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "is-buffer": "^2.0.0",
+        "unist-util-stringify-position": "^3.0.0",
+        "vfile-message": "^3.0.0"
+      },
+      "dependencies": {
+        "is-buffer": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
+          "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ=="
+        }
+      }
+    },
+    "vfile-location": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-4.0.1.tgz",
+      "integrity": "sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "vfile": "^5.0.0"
+      }
+    },
+    "vfile-message": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-3.0.2.tgz",
+      "integrity": "sha512-UUjZYIOg9lDRwwiBAuezLIsu9KlXntdxwG+nXnjuQAHvBpcX3x0eN8h+I7TkY5nkCXj+cWVp4ZqebtGBvok8ww==",
+      "requires": {
+        "@types/unist": "^2.0.0",
+        "unist-util-stringify-position": "^3.0.0"
+      }
+    },
     "vm-browserify": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz",
@@ -16590,6 +16935,11 @@
         "minimalistic-assert": "^1.0.0"
       }
     },
+    "web-namespaces": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.0.tgz",
+      "integrity": "sha512-dE7ELZRVWh0ceQsRgkjLgsAvwTuv3kcjSY/hLjqL0llleUlQBDjE9JkB9FCBY5F2mnFEwiyJoowl8+NVGHe8dw=="
+    },
     "web-vitals": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-1.1.1.tgz",

ui/package.json

@@ -4,6 +4,7 @@
   "private": true,
   "dependencies": {
     "@material-ui/core": "^4.11.3",
+    "@material-ui/icons": "^4.11.2",
     "@material-ui/lab": "^4.0.0-alpha.60",
     "@testing-library/jest-dom": "^5.11.10",
     "@testing-library/react": "^11.2.6",
@@ -12,6 +13,7 @@
     "@types/node": "^12.20.10",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.3",
+    "@uiw/react-textarea-code-editor": "^1.4.12",
     "axios": "^0.21.1",
     "jsonpath": "^1.1.1",
     "node-sass": "^5.0.0",
@@ -21,8 +23,9 @@
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
-    "react-scrollable-feed-virtualized": "^1.4.3",
+    "react-scrollable-feed-virtualized": "^1.4.8",
     "react-syntax-highlighter": "^15.4.3",
+    "react-toastify": "^8.0.3",
     "typescript": "^4.2.4",
     "web-vitals": "^1.1.1"
   },

ui/src/components/EntriesList.tsx

@@ -1,138 +1,44 @@
-import {EntryItem} from "./EntryListItem/EntryListItem";
-import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
+import React, {useRef} from "react";
 import styles from './style/EntriesList.module.sass';
-import spinner from './assets/spinner.svg';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
-import {StatusType} from "./Filters";
-import Api from "../helpers/api";
 import down from "./assets/downImg.svg";
 
 interface EntriesListProps {
     entries: any[];
-    setEntries: (entries: any[]) => void;
-    focusedEntryId: string;
-    setFocusedEntryId: (id: string) => void;
-    connectionOpen: boolean;
-    noMoreDataTop: boolean;
-    setNoMoreDataTop: (flag: boolean) => void;
-    noMoreDataBottom: boolean;
-    setNoMoreDataBottom: (flag: boolean) => void;
-    methodsFilter: Array<string>;
-    statusFilter: Array<string>;
-    pathFilter: string;
-    serviceFilter: string;
     listEntryREF: any;
-    onScrollEvent: (isAtBottom:boolean) => void;
-    scrollableList: boolean;
+    onSnapBrokenEvent: () => void;
+    isSnappedToBottom: boolean;
+    setIsSnappedToBottom: any;
+    queriedCurrent: number;
+    queriedTotal: number;
+    startTime: number;
 }
 
-enum FetchOperator {
-    LT = "lt",
-    GT = "gt"
-}
-
-const api = new Api();
-
-export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter, serviceFilter, listEntryREF, onScrollEvent, scrollableList}) => {
-
-    const [loadMoreTop, setLoadMoreTop] = useState(false);
-    const [isLoadingTop, setIsLoadingTop] = useState(false);
+export const EntriesList: React.FC<EntriesListProps> = ({entries, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, queriedTotal, startTime}) => {
 
     const scrollableRef = useRef(null);
 
-    useEffect(() => {
-        const list = document.getElementById('list').firstElementChild;
-        list.addEventListener('scroll', (e) => {
-            const el: any = e.target;
-            if(el.scrollTop === 0) {
-                setLoadMoreTop(true);
-            } else {
-                setLoadMoreTop(false);
-            }
-        });
-    }, []);
-
-    const filterEntries = useCallback((entry) => {
-        if(methodsFilter.length > 0 && !methodsFilter.includes(entry.method.toLowerCase())) return;
-        if(pathFilter && entry.path?.toLowerCase()?.indexOf(pathFilter) === -1) return;
-        if(serviceFilter && entry.service?.toLowerCase()?.indexOf(serviceFilter) === -1) return;
-        if(statusFilter.includes(StatusType.SUCCESS) && entry.statusCode >= 400) return;
-        if(statusFilter.includes(StatusType.ERROR) && entry.statusCode < 400) return;
-        return entry;
-    },[methodsFilter, pathFilter, statusFilter, serviceFilter])
-
-    const filteredEntries = useMemo(() => {
-        return entries.filter(filterEntries);
-    },[entries, filterEntries])
-
-    const getOldEntries = useCallback(async () => {
-        setIsLoadingTop(true);
-        const data = await api.fetchEntries(FetchOperator.LT, entries[0].timestamp);
-        setLoadMoreTop(false);
-
-        let scrollTo;
-        if(data.length === 0) {
-            setNoMoreDataTop(true);
-            scrollTo = document.getElementById("noMoreDataTop");
-        } else {
-            scrollTo = document.getElementById(filteredEntries?.[0]?.id);
-        }
-        setIsLoadingTop(false);
-        const newEntries = [...data, ...entries];
-        setEntries(newEntries);
-
-        if(scrollTo) {
-            scrollTo.scrollIntoView();
-        }
-    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, filteredEntries, setNoMoreDataTop])
-
-    useEffect(() => {
-        if(!loadMoreTop || connectionOpen || noMoreDataTop) return;
-        getOldEntries();
-    }, [loadMoreTop, connectionOpen, noMoreDataTop, getOldEntries]);
-
-    const getNewEntries = async () => {
-        const data = await api.fetchEntries(FetchOperator.GT, entries[entries.length - 1].timestamp);
-        let scrollTo;
-        if(data.length === 0) {
-            setNoMoreDataBottom(true);
-        }
-        scrollTo = document.getElementById(filteredEntries?.[filteredEntries.length -1]?.id);
-        let newEntries = [...entries, ...data];
-        setEntries(newEntries);
-        if(scrollTo) {
-            scrollTo.scrollIntoView({behavior: "smooth"});
-        }
-    }
-
     return <>
             <div className={styles.list}>
                 <div id="list" ref={listEntryREF} className={styles.list}>
-                    {isLoadingTop && <div className={styles.spinnerContainer}>
-                        <img alt="spinner" src={spinner} style={{height: 25}}/>
-                    </div>}
-                    <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onScroll={(isAtBottom) => onScrollEvent(isAtBottom)}>
-                        {noMoreDataTop && !connectionOpen && <div id="noMoreDataTop" className={styles.noMoreDataAvailable}>No more data available</div>}
-                        {filteredEntries.map(entry => <EntryItem key={entry.id}
-                                                        entry={entry}
-                                                        setFocusedEntryId={setFocusedEntryId}
-                                                        isSelected={focusedEntryId === entry.id}
-                                                        style={{}}/>)}
+                    <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onSnapBroken={onSnapBrokenEvent}>
+                        {false /* TODO: why there is a need for something here (not necessarily false)? */}
+                        {entries}
                     </ScrollableFeedVirtualized>
-                    {!connectionOpen && !noMoreDataBottom && <div className={styles.fetchButtonContainer}>
-                        <div className={styles.styledButton} onClick={() => getNewEntries()}>Fetch more entries</div>
-                    </div>}
                     <button type="button"
-                        className={`${styles.btnLive} ${scrollableList ? styles.showButton : styles.hideButton}`}
-                        onClick={(_) => scrollableRef.current.jumpToBottom()}>
+                        className={`${styles.btnLive} ${isSnappedToBottom ? styles.hideButton : styles.showButton}`}
+                        onClick={(_) => {
+                            scrollableRef.current.jumpToBottom();
+                            setIsSnappedToBottom(true);
+                        }}>
                         <img alt="down" src={down} />
                     </button>
                 </div>
 
-                {entries?.length > 0 && <div className={styles.footer}>
-                    <div><b>{filteredEntries?.length !== entries.length && `${filteredEntries?.length} / `} {entries?.length}</b> requests</div>
-                    <div>Started listening at <span style={{marginRight: 5, fontWeight: 600, fontSize: 13}}>{new Date(+entries[0].timestamp)?.toLocaleString()}</span></div>
-                </div>}
+                <div className={styles.footer}>
+                    <div>Displaying <b>{entries?.length}</b> results (queried <b>{queriedCurrent}</b>/<b>{queriedTotal}</b>)</div>
+                    {startTime !== 0 && <div>Started listening at <span style={{marginRight: 5, fontWeight: 600, fontSize: 13}}>{new Date(startTime).toLocaleString()}</span></div>}
+                </div>
             </div>
     </>;
 };

ui/src/components/EntryDetailed.tsx

@@ -3,7 +3,7 @@ import EntryViewer from "./EntryDetailed/EntryViewer";
 import {makeStyles} from "@material-ui/core";
 import Protocol from "./UI/Protocol"
 import StatusCode from "./UI/StatusCode";
-import {EndpointPath} from "./UI/EndpointPath";
+import {Summary} from "./UI/Summary";
 
 const useStyles = makeStyles(() => ({
     entryTitle: {
@@ -28,50 +28,79 @@ const useStyles = makeStyles(() => ({
 
 interface EntryDetailedProps {
     entryData: any
+    updateQuery: any
 }
 
 export const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`;
 
-const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime}) => {
+const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updateQuery}) => {
     const classes = useStyles();
-    const {response} = JSON.parse(data.entry);
+    const response = data.response;
 
 
     return <div className={classes.entryTitle}>
-        <Protocol protocol={protocol} horizontal={true}/>
+        <Protocol protocol={protocol} horizontal={true} updateQuery={null}/>
         <div style={{right: "30px", position: "absolute", display: "flex"}}>
-            {response.payload && <div style={{margin: "0 18px", opacity: 0.5}}>{formatSize(bodySize)}</div>}
-            {response.payload && <div style={{marginRight: 18, opacity: 0.5}}>{Math.round(elapsedTime)}ms</div>}
+            {response && <div
+                className="queryable"
+                style={{margin: "0 18px", opacity: 0.5}}
+                onClick={() => {
+                    updateQuery(`response.bodySize == ${bodySize}`)
+                }}
+            >
+                {formatSize(bodySize)}
+            </div>}
+            {response && <div
+                className="queryable"
+                style={{marginRight: 18, opacity: 0.5}}
+                onClick={() => {
+                    updateQuery(`elapsedTime >= ${elapsedTime}`)
+                }}
+            >
+                {Math.round(elapsedTime)}ms
+            </div>}
         </div>
     </div>;
 };
 
-const EntrySummary: React.FC<any> = ({data}) => {
+const EntrySummary: React.FC<any> = ({data, updateQuery}) => {
     const classes = useStyles();
 
-    const {response, request} = JSON.parse(data.entry);
+    const response = data.response;
 
     return <div className={classes.entrySummary}>
-        {response?.payload && response.payload?.details && "status" in response.payload.details && <div style={{marginRight: 8}}>
-            <StatusCode statusCode={response.payload.details.status}/>
+        {response && "status" in response && <div style={{marginRight: 8}}>
+            <StatusCode statusCode={response.status} updateQuery={updateQuery}/>
         </div>}
         <div style={{flexGrow: 1, overflow: 'hidden'}}>
-            <EndpointPath method={request?.payload.method} path={request?.payload.url}/>
+            <Summary method={data.method} summary={data.summary} updateQuery={updateQuery}/>
         </div>
     </div>;
 };
 
-export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData}) => {
+export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData, updateQuery}) => {
     return <>
         <EntryTitle
             protocol={entryData.protocol}
             data={entryData.data}
             bodySize={entryData.bodySize}
             elapsedTime={entryData.data.elapsedTime}
+            updateQuery={updateQuery}
         />
-        {entryData.data && <EntrySummary data={entryData.data}/>}
+        {entryData.data && <EntrySummary data={entryData.data} updateQuery={updateQuery}/>}
         <>
-            {entryData.data && <EntryViewer representation={entryData.representation} isRulesEnabled={entryData.isRulesEnabled} rulesMatched={entryData.rulesMatched} contractStatus={entryData.data.contractStatus} requestReason={entryData.data.contractRequestReason} responseReason={entryData.data.contractResponseReason} contractContent={entryData.data.contractContent} elapsedTime={entryData.data.elapsedTime} color={entryData.protocol.backgroundColor}/>}
+            {entryData.data && <EntryViewer
+                representation={entryData.representation}
+                isRulesEnabled={entryData.isRulesEnabled}
+                rulesMatched={entryData.rulesMatched}
+                contractStatus={entryData.data.contractStatus}
+                requestReason={entryData.data.contractRequestReason}
+                responseReason={entryData.data.contractResponseReason}
+                contractContent={entryData.data.contractContent}
+                elapsedTime={entryData.data.elapsedTime}
+                color={entryData.protocol.backgroundColor}
+                updateQuery={updateQuery}
+            />}
         </>
     </>
 };

ui/src/components/EntryDetailed/EntrySections.tsx

@@ -9,11 +9,29 @@ import ProtobufDecoder from "protobuf-decoder";
 interface EntryViewLineProps {
     label: string;
     value: number | string;
+    updateQuery: any;
+    selector: string;
+    overrideQueryValue?: string;
 }
 
-const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value}) => {
-    return (label && value && <tr className={styles.dataLine}>
-                <td className={styles.dataKey}>{label}</td>
+const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value, updateQuery, selector, overrideQueryValue}) => {
+    return (label && <tr className={styles.dataLine}>
+                <td
+                    className={`queryable ${styles.dataKey}`}
+                    onClick={() => {
+                        if (!selector) {
+                            return
+                        } else if (overrideQueryValue) {
+                            updateQuery(`${selector} == ${overrideQueryValue}`)
+                        } else if (typeof(value) === "string") {
+                            updateQuery(`${selector} == "${JSON.stringify(value).slice(1, -1)}"`)
+                        } else {
+                            updateQuery(`${selector} == ${value}`)
+                        }
+                    }}
+                >
+                    {label}
+                </td>
                 <td>
                     <FancyTextDisplay
                         className={styles.dataValue}
@@ -62,15 +80,19 @@ export const EntrySectionContainer: React.FC<EntrySectionContainerProps> = ({tit
 interface EntryBodySectionProps {
     content: any,
     color: string,
+    updateQuery: any,
     encoding?: string,
     contentType?: string,
+    selector?: string,
 }
 
 export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
     color,
+    updateQuery,
     content,
     encoding,
     contentType,
+    selector,
 }) => {
     const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes
     const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...]
@@ -107,8 +129,8 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
         {content && content?.length > 0 && <EntrySectionContainer title='Body' color={color}>
             <table>
                 <tbody>
-                    <EntryViewLine label={'Mime type'} value={contentType}/>
-                    <EntryViewLine label={'Encoding'} value={encoding}/>
+                    <EntryViewLine label={'Mime type'} value={contentType} updateQuery={updateQuery} selector={selector} overrideQueryValue={`r".*"`}/>
+                    <EntryViewLine label={'Encoding'} value={encoding} updateQuery={updateQuery} selector={selector} overrideQueryValue={`r".*"`}/>
                 </tbody>
             </table>
 
@@ -132,17 +154,23 @@ interface EntrySectionProps {
     title: string,
     color: string,
     arrayToIterate: any[],
+    updateQuery: any,
 }
 
-export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, arrayToIterate}) => {
+export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, arrayToIterate, updateQuery}) => {
     return <React.Fragment>
         {
             arrayToIterate && arrayToIterate.length > 0 ?
                 <EntrySectionContainer title={title} color={color}>
                     <table>
                         <tbody>
-                            {arrayToIterate.map(({name, value}, index) => <EntryViewLine key={index} label={name}
-                                                                                            value={value}/>)}
+                            {arrayToIterate.map(({name, value, selector}, index) => <EntryViewLine
+                                key={index}
+                                label={name}
+                                value={value}
+                                updateQuery={updateQuery}
+                                selector={selector}
+                            />)}
                         </tbody>
                     </table>
                 </EntrySectionContainer> : <span/>

ui/src/components/EntryDetailed/EntryViewer.tsx

@@ -8,7 +8,7 @@ enum SectionTypes {
     SectionBody = "body",
 }
 
-const SectionsRepresentation: React.FC<any> = ({data, color}) => {
+const SectionsRepresentation: React.FC<any> = ({data, color, updateQuery}) => {
     const sections = []
 
     if (data) {
@@ -16,12 +16,12 @@ const SectionsRepresentation: React.FC<any> = ({data, color}) => {
             switch (row.type) {
                 case SectionTypes.SectionTable:
                     sections.push(
-                        <EntryTableSection key={i} title={row.title} color={color} arrayToIterate={JSON.parse(row.data)}/>
+                        <EntryTableSection key={i} title={row.title} color={color} arrayToIterate={JSON.parse(row.data)} updateQuery={updateQuery}/>
                     )
                     break;
                 case SectionTypes.SectionBody:
                     sections.push(
-                        <EntryBodySection key={i} color={color} content={row.data} encoding={row.encoding} contentType={row.mime_type}/>
+                        <EntryBodySection key={i} color={color} content={row.data} updateQuery={updateQuery} encoding={row.encoding} contentType={row.mimeType} selector={row.selector}/>
                     )
                     break;
                 default:
@@ -33,7 +33,7 @@ const SectionsRepresentation: React.FC<any> = ({data, color}) => {
     return <>{sections}</>;
 }
 
-const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => {
+const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => {
     var TABS = [
         {
             tab: 'Request'
@@ -85,10 +85,10 @@ const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rule
                 <Tabs tabs={TABS} currentTab={currentTab} color={color} onChange={setCurrentTab} leftAligned/>
             </div>
             {currentTab === TABS[0].tab && <React.Fragment>
-                <SectionsRepresentation data={request} color={color}/>
+                <SectionsRepresentation data={request} color={color} updateQuery={updateQuery}/>
             </React.Fragment>}
             {response && currentTab === TABS[responseTabIndex].tab && <React.Fragment>
-                <SectionsRepresentation data={response} color={color}/>
+                <SectionsRepresentation data={response} color={color} updateQuery={updateQuery}/>
             </React.Fragment>}
             {isRulesEnabled && currentTab === TABS[rulesTabIndex].tab && <React.Fragment>
                 <EntryTablePolicySection title={'Rule'} color={color} latency={elapsedTime} arrayToIterate={rulesMatched ? rulesMatched : []}/>
@@ -110,9 +110,10 @@ interface Props {
     contractContent: string;
     color: string;
     elapsedTime: number;
+    updateQuery: any;
 }
 
-const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => {
+const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => {
     return <AutoRepresentation
         representation={representation}
         isRulesEnabled={isRulesEnabled}
@@ -123,6 +124,7 @@ const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatc
         contractContent={contractContent}
         elapsedTime={elapsedTime}
         color={color}
+        updateQuery={updateQuery}
     />
 };
 

ui/src/components/EntryListItem/EntryListItem.module.sass

@@ -84,7 +84,14 @@
   padding: 4px
   padding-left: 12px
 
-.port
+.tcpInfo
   font-size: 12px
   color: $secondary-font-color
-  margin: 5px
+  margin-top: 5px
+  margin-bottom: 5px
+
+.port
+  margin-right: 5px
+
+.ip
+  margin-left: 5px

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -1,8 +1,8 @@
-import React from "react";
+import React, {useState} from "react";
 import styles from './EntryListItem.module.sass';
 import StatusCode, {getClassification, StatusCodeClassification} from "../UI/StatusCode";
 import Protocol, {ProtocolInterface} from "../UI/Protocol"
-import {EndpointPath} from "../UI/EndpointPath";
+import {Summary} from "../UI/Summary";
 import ingoingIconSuccess from "../assets/ingoing-traffic-success.svg"
 import ingoingIconFailure from "../assets/ingoing-traffic-failure.svg"
 import ingoingIconNeutral from "../assets/ingoing-traffic-neutral.svg"
@@ -15,7 +15,7 @@ interface Entry {
     method?: string,
     summary: string,
     service: string,
-    id: string,
+    id: number,
     statusCode?: number;
     url?: string;
     timestamp: Date;
@@ -38,11 +38,14 @@ interface Rules {
 interface EntryProps {
     entry: Entry;
     setFocusedEntryId: (id: string) => void;
-    isSelected?: boolean;
     style: object;
+    updateQuery: any;
 }
 
-export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSelected, style}) => {
+export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, style, updateQuery}) => {
+
+    const [isSelected, setIsSelected] = useState(false);
+
     const classification = getClassification(entry.statusCode)
     const numberOfRules = entry.rules.numberOfRules
     let ingoingIcon;
@@ -115,10 +118,13 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
 
     return <>
         <div
-            id={entry.id}
+            id={entry.id.toString()}
             className={`${styles.row}
             ${isSelected && !rule && !contractEnabled ? styles.rowSelected : additionalRulesProperties}`}
-            onClick={() => setFocusedEntryId(entry.id)}
+            onClick={() => {
+                setIsSelected(!isSelected);
+                setFocusedEntryId(entry.id.toString());
+            }}
             style={{
                 border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid",
                 position: "absolute",
@@ -127,14 +133,26 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
                 width: "calc(100% - 25px)",
             }}
         >
-            <Protocol protocol={entry.protocol} horizontal={false}/>
+            <Protocol
+                protocol={entry.protocol}
+                horizontal={false}
+                updateQuery={updateQuery}
+            />
             {((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0) && <div>
-                <StatusCode statusCode={entry.statusCode}/>
+                <StatusCode statusCode={entry.statusCode} updateQuery={updateQuery}/>
             </div>}
             <div className={styles.endpointServiceContainer}>
-                <EndpointPath method={entry.method} path={entry.summary}/>
+                <Summary method={entry.method} summary={entry.summary} updateQuery={updateQuery}/>
                 <div className={styles.service}>
-                    <span title="Service Name">{entry.service}</span>
+                    <span
+                        title="Service Name"
+                        className="queryable"
+                        onClick={() => {
+                            updateQuery(`service == "${entry.service}"`)
+                        }}
+                    >
+                        {entry.service}
+                    </span>
                 </div>
             </div>
             {
@@ -152,17 +170,73 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
                 : ""
             }
             <div className={styles.separatorRight}>
-                <span className={styles.port} title="Source Port">{entry.sourcePort}</span>
+                <span
+                    className={`queryable ${styles.tcpInfo} ${styles.ip}`}
+                    title="Source IP"
+                    onClick={() => {
+                        updateQuery(`src.ip == "${entry.sourceIp}"`)
+                    }}
+                >
+                    {entry.sourceIp}
+                </span>
+                <span className={`${styles.tcpInfo}`}>:</span>
+                <span
+                    className={`queryable ${styles.tcpInfo} ${styles.port}`}
+                    title="Source Port"
+                    onClick={() => {
+                        updateQuery(`src.port == "${entry.sourcePort}"`)
+                    }}
+                >
+                    {entry.sourcePort}
+                </span>
                 {entry.isOutgoing ?
-                    <img src={outgoingIcon} alt="Ingoing traffic" title="Ingoing"/>
+                    <img
+                        src={outgoingIcon}
+                        alt="Ingoing traffic"
+                        title="Ingoing"
+                        onClick={() => {
+                            updateQuery(`outgoing == true`)
+                        }}
+                    />
                     :
-                    <img src={ingoingIcon} alt="Outgoing traffic" title="Outgoing"/>
+                    <img
+                        src={ingoingIcon}
+                        alt="Outgoing traffic"
+                        title="Outgoing"
+                        onClick={() => {
+                            updateQuery(`outgoing == false`)
+                        }}
+                    />
                 }
-                <span className={styles.port} title="Destination Port">{entry.destinationPort}</span>
+                <span
+                    className={`queryable ${styles.tcpInfo} ${styles.ip}`}
+                    title="Destination IP"
+                    onClick={() => {
+                        updateQuery(`dst.ip == "${entry.destinationIp}"`)
+                    }}
+                >
+                    {entry.destinationIp}
+                </span>
+                <span className={`${styles.tcpInfo}`}>:</span>
+                <span
+                    className={`queryable ${styles.tcpInfo} ${styles.port}`}
+                    title="Destination Port"
+                    onClick={() => {
+                        updateQuery(`dst.port == "${entry.destinationPort}"`)
+                    }}
+                >
+                    {entry.destinationPort}
+                </span>
             </div>
             <div className={styles.timestamp}>
-                <span title="Timestamp">
-                    {new Date(+entry.timestamp)?.toLocaleString()}
+                <span
+                    title="Timestamp"
+                    className="queryable"
+                    onClick={() => {
+                        updateQuery(`timestamp >= datetime("${new Date(+entry.timestamp)?.toLocaleString("en-US", {timeZone: 'UTC' })}")`)
+                    }}
+                >
+                    {new Date(+entry.timestamp)?.toLocaleString("en-US")}
                 </span>
             </div>
         </div>

ui/src/components/Filters.tsx

@@ -1,137 +1,326 @@
-import React from "react";
+import React, {useRef, useState} from "react";
 import styles from './style/Filters.module.sass';
-import {FilterSelect} from "./UI/FilterSelect";
-import {TextField} from "@material-ui/core";
-import {ALL_KEY} from "./UI/Select";
+import {Button, Grid, Modal, Box, Typography, Backdrop, Fade, Divider} from "@material-ui/core";
+import CodeEditor from '@uiw/react-textarea-code-editor';
+import MenuBookIcon from '@material-ui/icons/MenuBook';
+import {SyntaxHighlighter} from "./UI/SyntaxHighlighter/index";
+import filterUIExample1 from "./assets/filter-ui-example-1.png"
+import filterUIExample2 from "./assets/filter-ui-example-2.png"
+import variables from '../variables.module.scss';
 
 interface FiltersProps {
-    methodsFilter: Array<string>;
-    setMethodsFilter: (methods: Array<string>) => void;
-    statusFilter: Array<string>;
-    setStatusFilter: (methods: Array<string>) => void;
-    pathFilter: string
-    setPathFilter: (val: string) => void;
-    serviceFilter: string
-    setServiceFilter: (val: string) => void;
+    query: string
+    setQuery: any
+    backgroundColor: string
+    ws: any
+    openWebSocket: (query: string) => void;
 }
 
-export const Filters: React.FC<FiltersProps> = ({methodsFilter, setMethodsFilter, statusFilter, setStatusFilter, pathFilter, setPathFilter, serviceFilter, setServiceFilter}) => {
-
+export const Filters: React.FC<FiltersProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
     return <div className={styles.container}>
-        <MethodFilter methodsFilter={methodsFilter} setMethodsFilter={setMethodsFilter}/>
-        <StatusTypesFilter statusFilter={statusFilter} setStatusFilter={setStatusFilter}/>
-        <ServiceFilter serviceFilter={serviceFilter} setServiceFilter={setServiceFilter}/>
-        <PathFilter pathFilter={pathFilter} setPathFilter={setPathFilter}/>
+        <QueryForm
+            query={query}
+            setQuery={setQuery}
+            backgroundColor={backgroundColor}
+            ws={ws}
+            openWebSocket={openWebSocket}
+        />
     </div>;
 };
 
-const _toUpperCase = v => v.toUpperCase();
+interface QueryFormProps {
+    query: string
+    setQuery: any
+    backgroundColor: string
+    ws: any
+    openWebSocket: (query: string) => void;
+}
 
-const FilterContainer: React.FC = ({children}) => {
-    return <div className={styles.filterContainer}>
-        {children}
-    </div>;
+const style = {
+    position: 'absolute',
+    top: '10%',
+    left: '50%',
+    transform: 'translate(-50%, 0%)',
+    width: '80vw',
+    bgcolor: 'background.paper',
+    borderRadius: '5px',
+    boxShadow: 24,
+    p: 4,
+    color: '#000',
 };
 
-enum HTTPMethod {
-    GET = "get",
-    PUT = "put",
-    POST = "post",
-    DELETE = "delete",
-    OPTIONS="options",
-    PATCH = "patch"
-}
+export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
 
-interface MethodFilterProps {
-    methodsFilter: Array<string>;
-    setMethodsFilter: (methods: Array<string>) => void;
-}
+    const formRef = useRef<HTMLFormElement>(null);
 
-const MethodFilter: React.FC<MethodFilterProps> = ({methodsFilter, setMethodsFilter}) => {
+    const [openModal, setOpenModal] = useState(false);
 
-    const methodClicked = (val) => {
-        if(val === ALL_KEY) {
-            setMethodsFilter([]);
-            return;
-        }
-        if(methodsFilter.includes(val)) {
-            setMethodsFilter(methodsFilter.filter(method => method !== val))
-        } else {
-            setMethodsFilter([...methodsFilter, val]);
-        }
+    const handleOpenModal = () => setOpenModal(true);
+    const handleCloseModal = () => setOpenModal(false);
+
+    const handleChange = async (e) => {
+        setQuery(e.target.value);
     }
 
-    return <FilterContainer>
-        <FilterSelect
-            items={Object.values(HTTPMethod)}
-            allowMultiple={true}
-            value={methodsFilter}
-            onChange={(val) => methodClicked(val)}
-            transformDisplay={_toUpperCase}
-            label={"Methods"}
-        />
-    </FilterContainer>;
-};
-
-export enum StatusType {
-    SUCCESS = "success",
-    ERROR = "error"
-}
-
-interface StatusTypesFilterProps {
-    statusFilter: Array<string>;
-    setStatusFilter: (methods: Array<string>) => void;
-}
-
-const StatusTypesFilter: React.FC<StatusTypesFilterProps> = ({statusFilter, setStatusFilter}) => {
-
-    const statusClicked = (val) => {
-        if(val === ALL_KEY) {
-            setStatusFilter([]);
-            return;
-        }
-        setStatusFilter([val]);
+    const handleSubmit = (e) => {
+        ws.close()
+        openWebSocket(query)
+        e.preventDefault();
     }
 
-    return <FilterContainer>
-        <FilterSelect
-            items={Object.values(StatusType)}
-            allowMultiple={true}
-            value={statusFilter}
-            onChange={(val) => statusClicked(val)}
-            transformDisplay={_toUpperCase}
-            label="Status"
-        />
-    </FilterContainer>;
-};
+    return <>
+        <form
+            ref={formRef}
+            onSubmit={handleSubmit}
+            style={{
+                width: '100%',
+            }}
+        >
+            <Grid container spacing={2}>
+                <Grid
+                    item
+                    xs={8}
+                    style={{
+                        maxHeight: '25vh',
+                        overflowY: 'auto',
+                    }}
+                >
+                    <label>
+                        <CodeEditor
+                            value={query}
+                            language="py"
+                            placeholder="Mizu Filter Syntax"
+                            onChange={handleChange}
+                            padding={8}
+                            style={{
+                                fontSize: 14,
+                                backgroundColor: `${backgroundColor}`,
+                                fontFamily: 'ui-monospace,SFMono-Regular,SF Mono,Consolas,Liberation Mono,Menlo,monospace',
+                            }}
+                        />
+                    </label>
+                </Grid>
+                <Grid item xs={4}>
+                    <Button
+                        type="submit"
+                        variant="contained"
+                        style={{
+                            margin: "2px 0px 0px 0px",
+                            backgroundColor: variables.blueColor,
+                            fontWeight: 600,
+                            borderRadius: "4px",
+                            color: "#fff",
+                            textTransform: "none",
+                        }}
+                    >
+                        Apply
+                    </Button>
+                    <Button
+                        title="Open Filtering Guide (Cheatsheet)"
+                        variant="contained"
+                        color="primary"
+                        style={{
+                            margin: "2px 0px 0px 10px",
+                            minWidth: "26px",
+                            backgroundColor: variables.blueColor,
+                            fontWeight: 600,
+                            borderRadius: "4px",
+                            color: "#fff",
+                            textTransform: "none",
+                        }}
+                        onClick={handleOpenModal}
+                    >
+                        <MenuBookIcon fontSize="inherit"></MenuBookIcon>
+                    </Button>
+                </Grid>
+            </Grid>
+        </form>
 
-interface PathFilterProps {
-    pathFilter: string;
-    setPathFilter: (val: string) => void;
+        <Modal
+            aria-labelledby="transition-modal-title"
+            aria-describedby="transition-modal-description"
+            open={openModal}
+            onClose={handleCloseModal}
+            closeAfterTransition
+            BackdropComponent={Backdrop}
+            BackdropProps={{
+                timeout: 500,
+            }}
+            style={{overflow: 'auto'}}
+        >
+            <Fade in={openModal}>
+                <Box sx={style}>
+                    <Typography id="modal-modal-title" variant="h5" component="h2" style={{textAlign: 'center'}}>
+                        Filtering Guide (Cheatsheet)
+                    </Typography>
+                    <Typography id="modal-modal-description">
+                        <p>Mizu has a rich filtering syntax that let's you query the results both flexibly and efficiently.</p>
+                        <p>Here are some examples that you can try;</p>
+                    </Typography>
+                    <Grid container>
+                        <Grid item xs style={{margin: "10px"}}>
+                            <Typography id="modal-modal-description">
+                                This is a simple query that matches to HTTP packets with request path "/catalogue":
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and request.path == "/catalogue"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The same query can be negated for HTTP path and written like this:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and request.path != "/catalogue"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The syntax supports regular expressions. Here is a query that matches the HTTP requests that send JSON to a server:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and request.headers["Accept"] == r"application/json.*"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                Here is another query that matches HTTP responses with status code 4xx:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and response.status == r"4.*"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The same exact query can be as integer comparison:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and response.status >= 400`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                The results can be queried based on their timestamps:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`timestamp < datetime("10/28/2021, 9:13:02 PM")`}
+                                language="python"
+                            />
+                        </Grid>
+                        <Divider className={styles.divider1} orientation="vertical" flexItem />
+                        <Grid item xs style={{margin: "10px"}}>
+                            <Typography id="modal-modal-description">
+                                Since Mizu supports various protocols like gRPC, AMQP, Kafka and Redis. It's possible to write complex queries that match multiple protocols like this:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`(http and request.method == "PUT") or (amqp and request.queue.startsWith("test"))\n or (kafka and response.payload.errorCode == 2) or (redis and request.key == "example")\n or (grpc and request.headers[":path"] == r".*foo.*")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                By clicking the UI elements in both left-pane and right-pane, you can automatically select a field and update the query:
+                            </Typography>
+                            <img
+                                src={filterUIExample1}
+                                width={600}
+                                alt="Clicking to UI elements (left-pane)"
+                                title="Clicking to UI elements (left-pane)"
+                            />
+                            <Typography id="modal-modal-description">
+                                Such that; clicking this in left-pane, would append the query below:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`and service == "http://carts.sock-shop"`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                Another queriable UI element example, this time from the right-pane:
+                            </Typography>
+                            <img
+                                src={filterUIExample2}
+                                width={300}
+                                alt="Clicking to UI elements (right-pane)"
+                                title="Clicking to UI elements (right-pane)"
+                            />
+                            <Typography id="modal-modal-description">
+                                A query that compares one selector to another is also a valid query:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`http and (request.query["x"] == response.headers["y"]\n or response.content.text.contains(request.query["x"]))`}
+                                language="python"
+                            />
+                        </Grid>
+                        <Divider className={styles.divider2} orientation="vertical" flexItem />
+                        <Grid item xs style={{margin: "10px"}}>
+                            <Typography id="modal-modal-description">
+                                There are a few helper methods included the in the filter language* to help building queries more easily.
+                            </Typography>
+                            <br></br>
+                            <Typography id="modal-modal-description">
+                                true if the given selector's value starts with the string:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`request.path.startsWith("something")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                true if the given selector's value ends with the string:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`request.path.endsWith("something")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                true if the given selector's value contains the string:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`request.path.contains("something")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                returns the UNIX timestamp which is the equivalent of the time that's provided by the string. Invalid input evaluates to false:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`timestamp >= datetime("10/19/2021, 6:29:02 PM")`}
+                                language="python"
+                            />
+                            <Typography id="modal-modal-description">
+                                limits the number of records that are streamed back as a result of a query. Always evaluates to true:
+                            </Typography>
+                            <SyntaxHighlighter
+                                isWrapped={false}
+                                showLineNumbers={false}
+                                code={`and limit(100)`}
+                                language="python"
+                            />
+                        </Grid>
+                    </Grid>
+                    <br></br>
+                    <Typography id="modal-modal-description" style={{fontSize: 12, fontStyle: 'italic'}}>
+                        *The filtering functionality is provided through <b>Basenine</b> database server. Please refer to <a href="https://github.com/up9inc/basenine/wiki/BFL-Syntax-Reference"><b>BFL Syntax Reference</b></a> for more information.
+                    </Typography>
+                </Box>
+            </Fade>
+        </Modal>
+    </>
 }
-
-const PathFilter: React.FC<PathFilterProps> = ({pathFilter, setPathFilter}) => {
-
-    return <FilterContainer>
-        <div className={styles.filterLabel}>Path</div>
-        <div>
-            <TextField value={pathFilter} variant="outlined" className={styles.filterText} style={{minWidth: '150px'}} onChange={(e: any) => setPathFilter(e.target.value)}/>
-        </div>
-    </FilterContainer>;
-};
-
-interface ServiceFilterProps {
-    serviceFilter: string;
-    setServiceFilter: (val: string) => void;
-}
-
-const ServiceFilter: React.FC<ServiceFilterProps> = ({serviceFilter, setServiceFilter}) => {
-
-    return <FilterContainer>
-        <div className={styles.filterLabel}>Service</div>
-        <div>
-            <TextField value={serviceFilter} variant="outlined" className={styles.filterText} style={{minWidth: '150px'}} onChange={(e: any) => setServiceFilter(e.target.value)}/>
-        </div>
-    </FilterContainer>;
-};
-

ui/src/components/TrafficPage.tsx

@@ -1,6 +1,7 @@
 import React, {useEffect, useRef, useState} from "react";
 import {Filters} from "./Filters";
 import {EntriesList} from "./EntriesList";
+import {EntryItem} from "./EntryListItem/EntryListItem";
 import {makeStyles} from "@material-ui/core";
 import "./style/TrafficPage.sass";
 import styles from './style/EntriesList.module.sass';
@@ -10,6 +11,8 @@ import pauseIcon from './assets/pause.svg';
 import variables from '../variables.module.scss';
 import {StatusBar} from "./UI/StatusBar";
 import Api, {MizuWebsocketURL} from "../helpers/api";
+import { ToastContainer, toast } from 'react-toastify';
+import 'react-toastify/dist/ReactToastify.css';
 
 const useLayoutStyles = makeStyles(() => ({
     details: {
@@ -34,7 +37,6 @@ const useLayoutStyles = makeStyles(() => ({
 enum ConnectionStatus {
     Closed,
     Connected,
-    Paused
 }
 
 interface TrafficPageProps {
@@ -49,28 +51,64 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     const classes = useLayoutStyles();
 
     const [entries, setEntries] = useState([] as any);
+    const [entriesBuffer, setEntriesBuffer] = useState([] as any);
     const [focusedEntryId, setFocusedEntryId] = useState(null);
     const [selectedEntryData, setSelectedEntryData] = useState(null);
     const [connection, setConnection] = useState(ConnectionStatus.Closed);
-    const [noMoreDataTop, setNoMoreDataTop] = useState(false);
-    const [noMoreDataBottom, setNoMoreDataBottom] = useState(false);
-
-    const [methodsFilter, setMethodsFilter] = useState([]);
-    const [statusFilter, setStatusFilter] = useState([]);
-    const [pathFilter, setPathFilter] = useState("");
-    const [serviceFilter, setServiceFilter] = useState("");
 
     const [tappingStatus, setTappingStatus] = useState(null);
 
-    const [disableScrollList, setDisableScrollList] = useState(false);
+    const [isSnappedToBottom, setIsSnappedToBottom] = useState(true);
+
+    const [query, setQuery] = useState("");
+    const [queryBackgroundColor, setQueryBackgroundColor] = useState("#f5f5f5");
+    const [addition, updateQuery] = useState("");
+
+    const [queriedCurrent, setQueriedCurrent] = useState(0);
+    const [queriedTotal, setQueriedTotal] = useState(0);
+
+    const [startTime, setStartTime] = useState(0);
+
+    useEffect(() => {
+        (async function() {
+            if (!query) {
+                setQueryBackgroundColor("#f5f5f5")
+            } else {
+                const data = await api.validateQuery(query);
+                if (!data) {
+                    return;
+                }
+                if (data.valid) {
+                    setQueryBackgroundColor("#d2fad2");
+                } else {
+                    setQueryBackgroundColor("#fad6dc");
+                }
+            }
+        })();
+    }, [query]);
+
+    useEffect(() => {
+        if (query) {
+            setQuery(`${query} and ${addition}`);
+        } else {
+            setQuery(addition);
+        }
+        // eslint-disable-next-line
+    }, [addition]);
 
     const ws = useRef(null);
 
     const listEntry = useRef(null);
 
-    const openWebSocket = () => {
+    const openWebSocket = (query) => {
+        setFocusedEntryId(null);
+        setEntries([]);
+        setEntriesBuffer([]);
         ws.current = new WebSocket(MizuWebsocketURL);
-        ws.current.onopen = () => setConnection(ConnectionStatus.Connected);
+        ws.current.onopen = () => {
+            ws.current.send(query)
+            setConnection(ConnectionStatus.Connected);
+        }
         ws.current.onclose = () => setConnection(ConnectionStatus.Closed);
     }
 
@@ -80,19 +118,18 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
             const message = JSON.parse(e.data);
             switch (message.messageType) {
                 case "entry":
-                    const entry = message.data
-                    if (connection === ConnectionStatus.Paused) {
-                        setNoMoreDataBottom(false)
-                        return;
-                    }
-                    if (!focusedEntryId) setFocusedEntryId(entry.id)
-                    let newEntries = [...entries];
-                    setEntries([...newEntries, entry])
-                    if(listEntry.current) {
-                        if(isScrollable(listEntry.current.firstChild)) {
-                            setDisableScrollList(true)
-                        }
-                    }
+                    const entry = message.data;
+                    if (!focusedEntryId) setFocusedEntryId(entry.id.toString());
+                    setEntriesBuffer([
+                        ...entriesBuffer,
+                        <EntryItem
+                            key={entry.id}
+                            entry={entry}
+                            setFocusedEntryId={setFocusedEntryId}
+                            style={{}}
+                            updateQuery={updateQuery}
+                        />
+                    ]);
                     break
                 case "status":
                     setTappingStatus(message.tappingStatus);
@@ -103,6 +140,26 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                 case "outboundLink":
                     onTLSDetected(message.Data.DstIP);
                     break;
+                case "toast":
+                    toast[message.data.type](message.data.text, {
+                        position: "bottom-right",
+                        theme: "colored",
+                        autoClose: message.data.autoClose,
+                        hideProgressBar: false,
+                        closeOnClick: true,
+                        pauseOnHover: true,
+                        draggable: true,
+                        progress: undefined,
+                    });
+                    break;
+                case "queryMetadata":
+                    setQueriedCurrent(message.data.current);
+                    setQueriedTotal(message.data.total);
+                    setEntries(entriesBuffer);
+                    break;
+                case "startTime":
+                    setStartTime(message.data);
+                    break;
                 default:
                     console.error(`unsupported websocket message type, Got: ${message.messageType}`)
             }
@@ -111,7 +168,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     useEffect(() => {
         (async () => {
-            openWebSocket();
+            openWebSocket("rlimit(100)");
             try{
                 const tapStatusResponse = await api.tapStatus();
                 setTappingStatus(tapStatusResponse);
@@ -133,20 +190,33 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                 const entryData = await api.getEntry(focusedEntryId);
                 setSelectedEntryData(entryData);
             } catch (error) {
+                toast[error.response.data.type](`Entry[${focusedEntryId}]: ${error.response.data.msg}`, {
+                    position: "bottom-right",
+                    theme: "colored",
+                    autoClose: error.response.data.autoClose,
+                    hideProgressBar: false,
+                    closeOnClick: true,
+                    pauseOnHover: true,
+                    draggable: true,
+                    progress: undefined,
+                });
                 console.error(error);
             }
         })()
     }, [focusedEntryId])
 
     const toggleConnection = () => {
-        setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected);
+        if (connection === ConnectionStatus.Connected) {
+            ws.current.close();
+        } else {
+            openWebSocket(query);
+            setConnection(ConnectionStatus.Connected);
+        }
     }
 
     const getConnectionStatusClass = (isContainer) => {
         const container = isContainer ? "Container" : "";
         switch (connection) {
-            case ConnectionStatus.Paused:
-                return "orangeIndicator" + container;
             case ConnectionStatus.Connected:
                 return "greenIndicator" + container;
             default:
@@ -156,8 +226,6 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     const getConnectionTitle = () => {
         switch (connection) {
-            case ConnectionStatus.Paused:
-                return "traffic paused";
             case ConnectionStatus.Connected:
                 return "connected, waiting for traffic"
             default:
@@ -165,19 +233,17 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
         }
     }
 
-    const onScrollEvent = (isAtBottom) => {
-        isAtBottom ? setDisableScrollList(false) : setDisableScrollList(true)
+    const onSnapBrokenEvent = () => {
+        setIsSnappedToBottom(false)
     }
 
-    const isScrollable = (element) => {
-        return element.scrollHeight > element.clientHeight;
-    };
-
     return (
         <div className="TrafficPage">
             <div className="TrafficPageHeader">
-                {connection !== ConnectionStatus.Closed && <img style={{cursor: "pointer", marginRight: 15, height: 30}} alt="pause"
-                     src={connection === ConnectionStatus.Connected ? pauseIcon : playIcon} onClick={toggleConnection}/>}
+                <img className="playPauseIcon" style={{visibility: connection === ConnectionStatus.Connected ? "visible" : "hidden"}} alt="pause"
+                    src={pauseIcon} onClick={toggleConnection}/>
+                <img className="playPauseIcon" style={{position: "absolute", visibility: connection === ConnectionStatus.Connected ? "hidden" : "visible"}} alt="play"
+                    src={playIcon} onClick={toggleConnection}/>
                 <div className="connectionText">
                     {getConnectionTitle()}
                     <div className={"indicatorContainer " + getConnectionStatusClass(true)}>
@@ -185,42 +251,44 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     </div>
                 </div>
             </div>
-            {entries.length > 0 && <div className="TrafficPage-Container">
+            {<div className="TrafficPage-Container">
                 <div className="TrafficPage-ListContainer">
-                    <Filters methodsFilter={methodsFilter}
-                                setMethodsFilter={setMethodsFilter}
-                                statusFilter={statusFilter}
-                                setStatusFilter={setStatusFilter}
-                                pathFilter={pathFilter}
-                                setPathFilter={setPathFilter}
-                                serviceFilter={serviceFilter}
-                                setServiceFilter={setServiceFilter}
+                    <Filters
+                        query={query}
+                        setQuery={setQuery}
+                        backgroundColor={queryBackgroundColor}
+                        ws={ws.current}
+                        openWebSocket={openWebSocket}
                     />
                     <div className={styles.container}>
-                        <EntriesList entries={entries}
-                                        setEntries={setEntries}
-                                        focusedEntryId={focusedEntryId}
-                                        setFocusedEntryId={setFocusedEntryId}
-                                        connectionOpen={connection === ConnectionStatus.Connected}
-                                        noMoreDataBottom={noMoreDataBottom}
-                                        setNoMoreDataBottom={setNoMoreDataBottom}
-                                        noMoreDataTop={noMoreDataTop}
-                                        setNoMoreDataTop={setNoMoreDataTop}
-                                        methodsFilter={methodsFilter}
-                                        statusFilter={statusFilter}
-                                        pathFilter={pathFilter}
-                                        serviceFilter={serviceFilter}
-                                        listEntryREF={listEntry}
-                                        onScrollEvent={onScrollEvent}
-                                        scrollableList={disableScrollList}
+                        <EntriesList
+                            entries={entries}
+                            listEntryREF={listEntry}
+                            onSnapBrokenEvent={onSnapBrokenEvent}
+                            isSnappedToBottom={isSnappedToBottom}
+                            setIsSnappedToBottom={setIsSnappedToBottom}
+                            queriedCurrent={queriedCurrent}
+                            queriedTotal={queriedTotal}
+                            startTime={startTime}
                         />
                     </div>
                 </div>
                 <div className={classes.details}>
-                    {selectedEntryData && <EntryDetailed entryData={selectedEntryData}/>}
+                    {selectedEntryData && <EntryDetailed entryData={selectedEntryData} updateQuery={updateQuery}/>}
                 </div>
             </div>}
             {tappingStatus?.pods != null && <StatusBar tappingStatus={tappingStatus}/>}
+            <ToastContainer
+                position="bottom-right"
+                autoClose={5000}
+                hideProgressBar={false}
+                newestOnTop={false}
+                closeOnClick
+                rtl={false}
+                pauseOnFocusLoss
+                draggable
+                pauseOnHover
+            />
         </div>
     )
 };

ui/src/components/UI/EndpointPath.tsx

@@ -1,15 +0,0 @@
-import miscStyles from "./style/misc.module.sass";
-import React from "react";
-import styles from './style/EndpointPath.module.sass';
-
-interface EndpointPathProps {
-    method: string,
-    path: string
-}
-
-export const EndpointPath: React.FC<EndpointPathProps> = ({method, path}) => {
-    return <div className={styles.container}>
-        {method && <span title="Method" className={`${miscStyles.protocol} ${miscStyles.method}`}>{method}</span>}
-        {path && <div title="Summary" className={styles.path}>{path}</div>}
-    </div>
-};

ui/src/components/UI/FancyTextDisplay.tsx

@@ -17,7 +17,7 @@ interface Props {
 const FancyTextDisplay: React.FC<Props> = ({text, className, isPossibleToCopy = true, applyTextEllipsis = true, flipped = false, useTooltip= false, displayIconOnMouseOver = false, buttonOnly = false}) => {
     const [showCopiedNotification, setCopied] = useState(false);
     const [showTooltip, setShowTooltip] = useState(false);
-    const displayText = text || '';
+    text = String(text);
 
     const onCopy = () => {
         setCopied(true)
@@ -33,12 +33,12 @@ const FancyTextDisplay: React.FC<Props> = ({text, className, isPossibleToCopy =
         return () => clearTimeout(timer);
     }, [showCopiedNotification]);
 
-    const textElement = <span className={'FancyTextDisplay-Text'}>{displayText}</span>;
+    const textElement = <span className={'FancyTextDisplay-Text'}>{text}</span>;
 
-    const copyButton = isPossibleToCopy && displayText ? <CopyToClipboard text={displayText} onCopy={onCopy}>
+    const copyButton = isPossibleToCopy && text ? <CopyToClipboard text={text} onCopy={onCopy}>
                     <span
                         className={`FancyTextDisplay-Icon`}
-                        title={`Copy "${displayText}" value to clipboard`}
+                        title={`Copy "${text}" value to clipboard`}
                     >
                         <img src={duplicateImg} alt="Duplicate full value"/>
                         {showCopiedNotification && <span className={'FancyTextDisplay-CopyNotifier'}>Copied</span>}
@@ -48,14 +48,14 @@ const FancyTextDisplay: React.FC<Props> = ({text, className, isPossibleToCopy =
     return (
         <p
             className={`FancyTextDisplay-Container ${className ? className : ''} ${displayIconOnMouseOver ? 'displayIconOnMouseOver ' : ''} ${applyTextEllipsis ? ' FancyTextDisplay-ContainerEllipsis' : ''}`}
-            title={displayText.toString()}
+            title={text}
             onMouseOver={ e => setShowTooltip(true)}
             onMouseLeave={ e => setShowTooltip(false)}
         >
             {!buttonOnly && flipped && textElement}
             {copyButton}
             {!buttonOnly && !flipped && textElement}
-            {useTooltip && showTooltip && <span className={'FancyTextDisplay-CopyNotifier'}>{displayText}</span>}
+            {useTooltip && showTooltip && <span className={'FancyTextDisplay-CopyNotifier'}>{text}</span>}
         </p>
     );
 };

ui/src/components/UI/FilterSelect.tsx

@@ -1,28 +0,0 @@
-import React from "react";
-import { MenuItem } from '@material-ui/core';
-import style from './style/FilterSelect.module.sass';
-import { Select, SelectProps } from "./Select";
-
-interface FilterSelectProps extends SelectProps {
-    items: string[];
-    value: string | string[];
-    onChange: (string) => void;
-    label?: string;
-    allowMultiple?: boolean;
-    transformDisplay?: (string) => string;
-}
-
-export const FilterSelect: React.FC<FilterSelectProps> = ({items, value, onChange, label, allowMultiple= false, transformDisplay}) => {
-    return <Select
-        value={value}
-        multiple={allowMultiple}
-        label={label}
-        onChange={onChange}
-        transformDisplay={transformDisplay}
-        labelOnTop={true}
-        labelClassName={style.SelectLabel}
-        trimItemsWhenMultiple={true}
-    >
-        {items?.map(item => <MenuItem key={item} value={item}><span className='uppercase'>{item}</span></MenuItem>)}
-    </Select>
-};

ui/src/components/UI/Protocol.tsx

@@ -4,7 +4,8 @@ import styles from './style/Protocol.module.sass';
 export interface ProtocolInterface {
     name: string
     longName: string
-    abbreviation: string
+    abbr: string
+    macro: string
     backgroundColor: string
     foregroundColor: string
     fontSize: number
@@ -16,9 +17,10 @@ export interface ProtocolInterface {
 interface ProtocolProps {
     protocol: ProtocolInterface
     horizontal: boolean
+    updateQuery: any
 }
 
-const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal}) => {
+const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal, updateQuery}) => {
     if (horizontal) {
         return <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
             <span
@@ -28,25 +30,26 @@ const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal}) => {
                     color: protocol.foregroundColor,
                     fontSize: 13,
                 }}
-                title={protocol.abbreviation}
+                title={protocol.abbr}
             >
                 {protocol.longName}
             </span>
         </a>
     } else {
-        return <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
-            <span
-                className={`${styles.base} ${styles.vertical}`}
-                style={{
-                    backgroundColor: protocol.backgroundColor,
-                    color: protocol.foregroundColor,
-                    fontSize: protocol.fontSize,
-                }}
-                title={protocol.longName}
-            >
-                {protocol.abbreviation}
-            </span>
-        </a>
+        return <span
+            className={`${styles.base} ${styles.vertical}`}
+            style={{
+                backgroundColor: protocol.backgroundColor,
+                color: protocol.foregroundColor,
+                fontSize: protocol.fontSize,
+            }}
+            title={protocol.longName}
+            onClick={() => {
+                updateQuery(protocol.macro)
+            }}
+        >
+            {protocol.abbr}
+        </span>
     }
 };