Adding additional error handling to api server watch (#311)

* Watch the tapper pod after starting it

* Improve the logic in `watchTapperPod` method

.github/workflows/pr_validation.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.16'
+          go-version: '1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -33,7 +33,7 @@ jobs:
       - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.16'
+          go-version: '1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2

.gitignore

@@ -26,3 +26,6 @@ build
 
 # Environment variables
 .env
+
+# pprof
+pprof/*

Dockerfile

@@ -39,7 +39,7 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
-COPY build_extensions.sh ..
+COPY devops/build_extensions.sh ..
 RUN cd .. && /bin/bash build_extensions.sh
 
 FROM alpine:3.13.5

Makefile

@@ -44,11 +44,11 @@ push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
 push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
-	./build-push-featurebranch.sh
+	devops/build-push-featurebranch.sh
 
 build-docker-ci: ## Build agent docker image for CI.
 	@echo "building docker image for ci"
-	./build-agent-ci.sh
+	devops/build-agent-ci.sh
 
 push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
@@ -73,7 +73,7 @@ clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
 extensions:
-	./build_extensions.sh
+	devops/build_extensions.sh
 
 test-cli:
 	@echo "running cli tests"; cd cli && $(MAKE) test

README.md

@@ -46,7 +46,7 @@ While `mizu`most often works out of the box, you can influence its behavior:
 1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
 2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
 
-For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
+For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
 
 
 ## How to Run
@@ -143,7 +143,7 @@ Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
 User-agent filtering (like health checks) - can be configured using command-line options:
 
 ```shell
-$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
+$ mizu tap "^ca.*" --set tap.ignored-user-agents=kube-probe --set tap.ignored-user-agents=prometheus
 +carts-66c77f5fbb-fq65r
 +catalogue-5f4cb7cf5-7zrmn
 Web interface is now available at http://localhost:8899
@@ -152,12 +152,12 @@ Web interface is now available at http://localhost:8899
 ```
 Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
 
-### API Rules validation
+### Traffic validation rules
 
-This feature allows you to define set of simple rules, and test the API against them.
+This feature allows you to define set of simple rules, and test the traffic against them.
 Such validation may test response for specific JSON fields, headers, etc.
 
-Please see [API RULES](docs/POLICY_RULES.md) page for more details and syntax.
+Please see [TRAFFIC RULES](docs/POLICY_RULES.md) page for more details and syntax.
 
 
 ## How to Run local UI

TESTING.md

@@ -1,15 +0,0 @@
-![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
-# TESTING
-Testing guidelines for Mizu project 
-
-## Unit-tests
-* TBD 
-* TBD
-* TBD
-
-
-
-## System tests
-* TBD 
-* TBD
-* TBD

acceptanceTests/logs_test.go

@@ -0,0 +1,196 @@
+package acceptanceTests
+
+import (
+	"archive/zip"
+	"os/exec"
+	"testing"
+)
+
+func TestLogs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	logsCmdArgs := getDefaultLogsCommandArgs()
+
+	logsCmd := exec.Command(cliPath, logsCmdArgs...)
+	t.Logf("running command: %v", logsCmd.String())
+
+	if err := logsCmd.Start(); err != nil {
+		t.Errorf("failed to start logs command, err: %v", err)
+		return
+	}
+
+	if err := logsCmd.Wait(); err != nil {
+		t.Errorf("failed to wait logs command, err: %v", err)
+		return
+	}
+
+	logsPath, logsPathErr := getLogsPath()
+	if logsPathErr != nil {
+		t.Errorf("failed to get logs path, err: %v", logsPathErr)
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(logsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}
+
+func TestLogsPath(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	logsCmdArgs := getDefaultLogsCommandArgs()
+
+	logsPath := "../logs.zip"
+	logsCmdArgs = append(logsCmdArgs, "-f", logsPath)
+
+	logsCmd := exec.Command(cliPath, logsCmdArgs...)
+	t.Logf("running command: %v", logsCmd.String())
+
+	if err := logsCmd.Start(); err != nil {
+		t.Errorf("failed to start logs command, err: %v", err)
+		return
+	}
+
+	if err := logsCmd.Wait(); err != nil {
+		t.Errorf("failed to wait logs command, err: %v", err)
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(logsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}

acceptanceTests/tap_test.go

@@ -1,11 +1,14 @@
 package acceptanceTests
 
 import (
+	"archive/zip"
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os/exec"
+	"path"
 	"strings"
 	"testing"
 	"time"
@@ -502,10 +505,19 @@ func TestTapRedact(t *testing.T) {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{})
-		entryRequest := entry["request"].(map[string]interface{})
+		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		entryJson := data["entry"].(string)
 
-		headers :=  entryRequest["headers"].([]interface{})
+		var entry map[string]interface{}
+		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+		}
+
+		entryRequest := entry["request"].(map[string]interface{})
+		entryPayload := entryRequest["payload"].(map[string]interface{})
+		entryDetails := entryPayload["details"].(map[string]interface{})
+
+		headers :=  entryDetails["headers"].([]interface{})
 		for _, headerInterface := range headers {
 			header := headerInterface.(map[string]interface{})
 			if header["name"].(string) != "User-Agent" {
@@ -518,8 +530,8 @@ func TestTapRedact(t *testing.T) {
 			}
 		}
 
-		data := entryRequest["postData"].(map[string]interface{})
-		textDataStr := data["text"].(string)
+		postData := entryDetails["postData"].(map[string]interface{})
+		textDataStr := postData["text"].(string)
 
 		var textData map[string]string
 		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
@@ -608,10 +620,19 @@ func TestTapNoRedact(t *testing.T) {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{})
-		entryRequest := entry["request"].(map[string]interface{})
+		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		entryJson := data["entry"].(string)
 
-		headers :=  entryRequest["headers"].([]interface{})
+		var entry map[string]interface{}
+		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+		}
+
+		entryRequest := entry["request"].(map[string]interface{})
+		entryPayload := entryRequest["payload"].(map[string]interface{})
+		entryDetails := entryPayload["details"].(map[string]interface{})
+
+		headers :=  entryDetails["headers"].([]interface{})
 		for _, headerInterface := range headers {
 			header := headerInterface.(map[string]interface{})
 			if header["name"].(string) != "User-Agent" {
@@ -624,8 +645,8 @@ func TestTapNoRedact(t *testing.T) {
 			}
 		}
 
-		data := entryRequest["postData"].(map[string]interface{})
-		textDataStr := data["text"].(string)
+		postData := entryDetails["postData"].(map[string]interface{})
+		textDataStr := postData["text"].(string)
 
 		var textData map[string]string
 		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
@@ -714,11 +735,20 @@ func TestTapRegexMasking(t *testing.T) {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{})
-		entryRequest := entry["request"].(map[string]interface{})
+		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		entryJson := data["entry"].(string)
 
-		data := entryRequest["postData"].(map[string]interface{})
-		textData := data["text"].(string)
+		var entry map[string]interface{}
+		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+		}
+
+		entryRequest := entry["request"].(map[string]interface{})
+		entryPayload := entryRequest["payload"].(map[string]interface{})
+		entryDetails := entryPayload["details"].(map[string]interface{})
+
+		postData := entryDetails["postData"].(map[string]interface{})
+		textData := postData["text"].(string)
 
 		if textData != "[REDACTED]" {
 			return fmt.Errorf("unexpected result - body is not redacted")
@@ -731,3 +761,105 @@ func TestTapRegexMasking(t *testing.T) {
 		return
 	}
 }
+
+func TestTapDumpLogs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--set", "dump-logs=true")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	if err := cleanupCommand(tapCmd); err != nil {
+		t.Errorf("failed to cleanup tap command, err: %v", err)
+		return
+	}
+
+	mizuFolderPath, mizuPathErr := getMizuFolderPath()
+	if mizuPathErr != nil {
+		t.Errorf("failed to get mizu folder path, err: %v", mizuPathErr)
+		return
+	}
+
+	files, readErr := ioutil.ReadDir(mizuFolderPath)
+	if readErr != nil {
+		t.Errorf("failed to read mizu folder files, err: %v", readErr)
+		return
+	}
+
+	var dumpsLogsPath string
+	for _, file := range files {
+		fileName := file.Name()
+		if strings.Contains(fileName, "mizu_logs") {
+			dumpsLogsPath = path.Join(mizuFolderPath, fileName)
+			break
+		}
+	}
+
+	if dumpsLogsPath == "" {
+		t.Errorf("dump logs file not found")
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(dumpsLogsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}

acceptanceTests/testsUtils.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"strings"
 	"syscall"
 	"time"
 )
@@ -32,13 +33,22 @@ func getCliPath() (string, error) {
 	return cliPath, nil
 }
 
-func getConfigPath() (string, error) {
+func getMizuFolderPath() (string, error) {
 	home, homeDirErr := os.UserHomeDir()
 	if homeDirErr != nil {
 		return "", homeDirErr
 	}
 
-	return path.Join(home, ".mizu", "config.yaml"), nil
+	return path.Join(home, ".mizu"), nil
+}
+
+func getConfigPath() (string, error) {
+	mizuFolderPath, mizuPathError := getMizuFolderPath()
+	if mizuPathError != nil {
+		return "", mizuPathError
+	}
+
+	return path.Join(mizuFolderPath, "config.yaml"), nil
 }
 
 func getProxyUrl(namespace string, service string) string {
@@ -72,6 +82,13 @@ func getDefaultTapCommandArgsWithRegex(regex string) []string {
 	return append([]string{tapCommand, regex}, defaultCmdArgs...)
 }
 
+func getDefaultLogsCommandArgs() []string {
+	logsCommand := "logs"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{logsCommand}, defaultCmdArgs...)
+}
+
 func getDefaultTapNamespace() []string {
 	return []string{"-n", "mizu-tests"}
 }
@@ -193,3 +210,33 @@ func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
 
 	return pods, nil
 }
+
+func getLogsPath() (string, error) {
+	dir, filePathErr := os.Getwd()
+	if filePathErr != nil {
+		return "", filePathErr
+	}
+
+	logsPath := path.Join(dir, "mizu_logs.zip")
+	return logsPath, nil
+}
+
+func Contains(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if sliceValue == containsValue {
+			return true
+		}
+	}
+
+	return false
+}
+
+func ContainsPartOfValue(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if strings.Contains(sliceValue, containsValue) {
+			return true
+		}
+	}
+
+	return false
+}

agent/main.go

@@ -50,19 +50,21 @@ func main() {
 		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
+	filteringOptions := getTrafficFilteringOptions()
+
 	if *standaloneMode {
 		api.StartResolving(*namespace)
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions)
+		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
 
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel, getTrafficFilteringOptions())
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel, filteringOptions)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
-		// go api.StartReadingOutbound(outboundLinkOutputChannel)
 
 		hostApi(nil)
 	} else if *tapperMode {
+		rlog.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
 		if *apiServerAddress == "" {
 			panic("API server address must be provided with --api-server-address when using --tap")
 		}
@@ -73,23 +75,22 @@ func main() {
 			rlog.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
 		}
 
-		// harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions)
-		socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
+		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
+		socketConnection, _, err := websocket.DefaultDialer.Dial(*apiServerAddress, nil)
 		if err != nil {
 			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
 		}
+		rlog.Infof("Connected successfully to websocket %s", *apiServerAddress)
 
 		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
-		// go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel)
 	} else if *apiServerMode {
 		api.StartResolving(*namespace)
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel, getTrafficFilteringOptions())
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel, filteringOptions)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
 
 		hostApi(outputItemsChannel)
@@ -97,7 +98,7 @@ func main() {
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
 		filteredHarChannel := make(chan *tapApi.OutputChannelItem)
 
-		go filterItems(outputItemsChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go filterItems(outputItemsChannel, filteredHarChannel, filteringOptions)
 		go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
 		hostApi(nil)
 	}
@@ -121,7 +122,7 @@ func loadExtensions() {
 	extensionsMap = make(map[string]*tapApi.Extension)
 	for i, file := range files {
 		filename := file.Name()
-		log.Printf("Loading extension: %s\n", filename)
+		rlog.Infof("Loading extension: %s\n", filename)
 		extension := &tapApi.Extension{
 			Path: path.Join(extensionsDir, filename),
 		}
@@ -225,23 +226,23 @@ func getTapTargets() []string {
 	return tappedAddressesPerNodeDict[nodeName]
 }
 
-func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
+func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
 	if filteringOptionsJson == "" {
-		return &shared.TrafficFilteringOptions{
+		return &tapApi.TrafficFilteringOptions{
 			HealthChecksUserAgentHeaders: []string{},
 		}
 	}
-	var filteringOptions shared.TrafficFilteringOptions
+	var filteringOptions tapApi.TrafficFilteringOptions
 	err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions)
 	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the api.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
 	}
 
 	return &filteringOptions
 }
 
-func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
+func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem, filterOptions *tapApi.TrafficFilteringOptions) {
 	for message := range inChannel {
 		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
 			continue
@@ -251,10 +252,6 @@ func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *ta
 			continue
 		}
 
-		// if !filterOptions.DisableRedaction {
-		// 	sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
-		// }
-
 		outChannel <- message
 	}
 }
@@ -293,7 +290,7 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 	for messageData := range messageDataChannel {
 		marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData)
 		if err != nil {
-			rlog.Infof("error converting message to json %s, (%v,%+v)\n", err, err, err)
+			rlog.Errorf("error converting message to json %v, err: %s, (%v,%+v)", messageData, err, err, err)
 			continue
 		}
 
@@ -301,26 +298,8 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 		// and goes into the intermediate WebSocket.
 		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
 		if err != nil {
-			rlog.Infof("error sending message through socket server %s, (%v,%+v)\n", err, err, err)
+			rlog.Errorf("error sending message through socket server %v, err: %s, (%v,%+v)", messageData, err, err, err)
 			continue
 		}
 	}
 }
-
-func pipeOutboundLinksChannelToSocket(connection *websocket.Conn, outboundLinkChannel <-chan *tap.OutboundLink) {
-	for outboundLink := range outboundLinkChannel {
-		if outboundLink.SuggestedProtocol == tap.TLSProtocol {
-			marshaledData, err := models.CreateWebsocketOutboundLinkMessage(outboundLink)
-			if err != nil {
-				rlog.Infof("Error converting outbound link to json %s, (%v,%+v)", err, err, err)
-				continue
-			}
-
-			err = connection.WriteMessage(websocket.TextMessage, marshaledData)
-			if err != nil {
-				rlog.Infof("error sending outbound link message through socket server %s, (%v,%+v)", err, err, err)
-				continue
-			}
-		}
-	}
-}

agent/pkg/api/main.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/holder"
-	"net/url"
+	"mizuserver/pkg/providers"
 	"os"
 	"path"
 	"sort"
@@ -18,7 +18,6 @@ import (
 
 	"github.com/google/martian/har"
 	"github.com/romana/rlog"
-	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
@@ -59,8 +58,10 @@ func StartReadingEntries(harChannel <-chan *tapApi.OutputChannelItem, workingDir
 }
 
 func startReadingFiles(workingDir string) {
-	err := os.MkdirAll(workingDir, os.ModePerm)
-	utils.CheckErr(err)
+	if err := os.MkdirAll(workingDir, os.ModePerm); err != nil {
+		rlog.Errorf("Failed to make dir: %s, err: %v", workingDir, err)
+		return
+	}
 
 	for true {
 		dir, _ := os.Open(workingDir)
@@ -88,17 +89,6 @@ func startReadingFiles(workingDir string) {
 		decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
 		utils.CheckErr(decErr)
 
-		// for _, entry := range inputHar.Log.Entries {
-		// 	time.Sleep(time.Millisecond * 250)
-		// 	// connectionInfo := &tap.ConnectionInfo{
-		// 	// 	ClientIP:   fileInfo.Name(),
-		// 	// 	ClientPort: "",
-		// 	// 	ServerIP:   "",
-		// 	// 	ServerPort: "",
-		// 	// 	IsOutgoing: false,
-		// 	// }
-		// 	// saveHarToDb(entry, connectionInfo)
-		// }
 		rmErr := os.Remove(inputFilePath)
 		utils.CheckErr(rmErr)
 	}
@@ -110,24 +100,29 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	}
 
 	for item := range outputItems {
+		providers.EntryAdded()
+
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
 		mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
 		baseEntry := extension.Dissector.Summarize(mizuEntry)
 		mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
 		database.CreateEntry(mizuEntry)
+		if extension.Protocol.Name == "http" {
+			var pair tapApi.RequestResponsePair
+			json.Unmarshal([]byte(mizuEntry.Entry), &pair)
+			harEntry, err := utils.NewEntry(&pair)
+			if err == nil {
+				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
+				baseEntry.Rules = rules
+			}
+		}
+
 		baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)
 		BroadcastToBrowserClients(baseEntryBytes)
 	}
 }
 
-func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
-	// tcpStreamFactory will block on write to channel. Empty channel to unblock.
-	// TODO: Make write to channel optional.
-	for range outboundLinkChannel {
-	}
-}
-
 func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string) {
 	if k8sResolver != nil {
 		unresolvedSource := connectionInfo.ClientIP
@@ -150,12 +145,6 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 	return resolvedSource, resolvedDestination
 }
 
-func getServiceNameFromUrl(inputUrl string) (string, string) {
-	parsed, err := url.Parse(inputUrl)
-	utils.CheckErr(err)
-	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path
-}
-
 func CheckIsServiceIP(address string) bool {
 	if k8sResolver == nil {
 		return false

agent/pkg/controllers/entries_controller.go

@@ -3,7 +3,6 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/google/martian/har"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
@@ -13,6 +12,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/google/martian/har"
+
 	"github.com/gin-gonic/gin"
 	"github.com/romana/rlog"
 
@@ -140,11 +141,26 @@ func GetEntry(c *gin.Context) {
 
 	extension := extensionsMap[entryData.ProtocolName]
 	protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
+
+	var rules []map[string]interface{}
+	var isRulesEnabled bool
+	if entryData.ProtocolName == "http" {
+		var pair tapApi.RequestResponsePair
+		json.Unmarshal([]byte(entryData.Entry), &pair)
+		harEntry, _ := utils.NewEntry(&pair)
+		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service)
+		isRulesEnabled = _isRulesEnabled
+		inrec, _ := json.Marshal(rulesMatched)
+		json.Unmarshal(inrec, &rules)
+	}
+
 	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
 		Protocol:       protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,
 		Data:           entryData,
+		Rules:          rules,
+		IsRulesEnabled: isRulesEnabled,
 	})
 }
 

agent/pkg/database/main.go

@@ -61,7 +61,7 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *stri
 	order := OrderDesc
 	protocolNameCondition := "1 = 1"
 	if protocolName != nil {
-		protocolNameCondition = fmt.Sprintf("protocolKey = '%s'", *protocolName)
+		protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
 	}
 
 	var entries []tapApi.MizuEntry

agent/pkg/models/models.go

@@ -2,9 +2,10 @@ package models
 
 import (
 	"encoding/json"
+
 	tapApi "github.com/up9inc/mizu/tap/api"
+
 	"mizuserver/pkg/rules"
-	"mizuserver/pkg/utils"
 
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
@@ -15,15 +16,6 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
-// TODO: until we fixed the Rules feature
-//func NewApplicableRules(status bool, latency int64, number int) tapApi.ApplicableRules {
-//	ar := tapApi.ApplicableRules{}
-//	ar.Status = status
-//	ar.Latency = latency
-//	ar.NumberOfRules = number
-//	return ar
-//}
-
 type EntriesFilter struct {
 	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
 	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
@@ -105,33 +97,8 @@ type ExtendedCreator struct {
 	Source *string `json:"_source"`
 }
 
-type FullEntryWithPolicy struct {
-	RulesMatched []rules.RulesMatched `json:"rulesMatched,omitempty"`
-	Entry        har.Entry            `json:"entry"`
-	Service      string               `json:"service"`
+func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.ApplicableRules, []rules.RulesMatched, bool) {
+	resultPolicyToSend, isEnabled := rules.MatchRequestPolicy(harEntry, service)
+	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
+	return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
 }
-
-func (fewp *FullEntryWithPolicy) UnmarshalData(entry *tapApi.MizuEntry) error {
-	var pair tapApi.RequestResponsePair
-	if err := json.Unmarshal([]byte(entry.Entry), &pair); err != nil {
-		return err
-	}
-	harEntry, err := utils.NewEntry(&pair)
-	if err != nil {
-		return err
-	}
-	fewp.Entry = *harEntry
-
-	_, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service)
-	fewp.RulesMatched = resultPolicyToSend
-	fewp.Service = entry.Service
-	return nil
-}
-
-// TODO: until we fixed the Rules feature
-//func RunValidationRulesState(harEntry har.Entry, service string) tapApi.ApplicableRules {
-//	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
-//	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
-//	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
-//	return ar
-//}

agent/pkg/rules/rulesHTTP.go

@@ -1,12 +1,15 @@
 package rules
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"reflect"
 	"regexp"
 	"strings"
 
+	"github.com/romana/rlog"
+
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	jsonpath "github.com/yalp/jsonpath"
@@ -41,16 +44,19 @@ func ValidateService(serviceFromRule string, service string) bool {
 	return true
 }
 
-func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched) {
-	enforcePolicy, _ := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
-	var resultPolicyToSend []RulesMatched
+func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend []RulesMatched, isEnabled bool) {
+	enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
+	if err == nil {
+		isEnabled = true
+	}
 	for _, rule := range enforcePolicy.Rules {
 		if !ValidatePath(rule.Path, harEntry.Request.URL) || !ValidateService(rule.Service, service) {
 			continue
 		}
 		if rule.Type == "json" {
 			var bodyJsonMap interface{}
-			if err := json.Unmarshal(harEntry.Response.Content.Text, &bodyJsonMap); err != nil {
+			contentTextDecoded, _ := base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text))
+			if err := json.Unmarshal(contentTextDecoded, &bodyJsonMap); err != nil {
 				continue
 			}
 			out, err := jsonpath.Read(bodyJsonMap, rule.Key)
@@ -63,6 +69,7 @@ func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched
 				if err != nil {
 					continue
 				}
+				rlog.Info(matchValue, rule.Value)
 			} else {
 				val := fmt.Sprint(out)
 				matchValue, err = regexp.MatchString(rule.Value, val)
@@ -89,22 +96,28 @@ func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched
 			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, true, rule)
 		}
 	}
-	return len(enforcePolicy.Rules), resultPolicyToSend
+	return
 }
 
-func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64, int) {
-	if len(rulesMatched) == 0 {
-		return false, 0, 0
+func PassedValidationRules(rulesMatched []RulesMatched) (bool, int64, int) {
+	var numberOfRulesMatched = len(rulesMatched)
+	var responseTime int64 = -1
+
+	if numberOfRulesMatched == 0 {
+		return false, 0, numberOfRulesMatched
 	}
+
 	for _, rule := range rulesMatched {
 		if rule.Matched == false {
-			return false, -1, len(rulesMatched)
+			return false, responseTime, numberOfRulesMatched
+		} else {
+			if strings.ToLower(rule.Rule.Type) == "responseTime" {
+				if rule.Rule.ResponseTime < responseTime || responseTime == -1 {
+					responseTime = rule.Rule.ResponseTime
+				}
+			}
 		}
 	}
-	for _, rule := range rulesMatched {
-		if strings.ToLower(rule.Rule.Type) == "latency" {
-			return true, rule.Rule.Latency, len(rulesMatched)
-		}
-	}
-	return true, -1, len(rulesMatched)
+
+	return true, responseTime, numberOfRulesMatched
 }

agent/pkg/utils/har.go

@@ -4,16 +4,14 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"github.com/google/martian/har"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/tap/api"
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/tap"
-	"github.com/up9inc/mizu/tap/api"
 )
 
-
 // Keep it because we might want cookies in the future
 //func BuildCookies(rawCookies []interface{}) []har.Cookie {
 //	cookies := make([]har.Cookie, 0, len(rawCookies))
@@ -82,7 +80,7 @@ func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, strin
 			path = h["value"].(string)
 		}
 		if h["name"] == ":status" {
-			path = h["value"].(string)
+			status = h["value"].(string)
 		}
 	}
 
@@ -205,7 +203,7 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
 		status, err = strconv.Atoi(_status)
 		if err != nil {
-			tap.SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)", err, err, err)
+			rlog.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
 			return nil, errors.New("failed converting response status to int for HAR")
 		}
 	}
@@ -226,14 +224,13 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
 	harRequest, err := NewRequest(&pair.Request)
 	if err != nil {
-		tap.SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
+		rlog.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting request to HAR")
 	}
 
 	harResponse, err := NewResponse(&pair.Response)
 	if err != nil {
-		fmt.Printf("err: %+v\n", err)
-		tap.SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)
+		rlog.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting response to HAR")
 	}
 

agent/pkg/utils/utils.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"github.com/gin-gonic/gin"
 	"github.com/romana/rlog"
-	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -18,8 +17,8 @@ import (
 func StartServer(app *gin.Engine) {
 	signals := make(chan os.Signal, 2)
 	signal.Notify(signals,
-		os.Interrupt,  	  // this catch ctrl + c
-		syscall.SIGTSTP,  // this catch ctrl + z
+		os.Interrupt,    // this catch ctrl + c
+		syscall.SIGTSTP, // this catch ctrl + z
 	)
 
 	srv := &http.Server{
@@ -36,8 +35,9 @@ func StartServer(app *gin.Engine) {
 	}()
 
 	// Run server.
+	rlog.Infof("Starting the server...")
 	if err := app.Run(":8899"); err != nil {
-		log.Printf("Oops... Server is not running! Reason: %v", err)
+		rlog.Errorf("Server is not running! Reason: %v", err)
 	}
 }
 
@@ -54,15 +54,14 @@ func ReverseSlice(data interface{}) {
 
 func CheckErr(e error) {
 	if e != nil {
-		log.Printf("%v", e)
-		//panic(e)
+		rlog.Errorf("%v", e)
 	}
 }
 
 func SetHostname(address, newHostname string) string {
 	replacedUrl, err := url.Parse(address)
-	if err != nil{
-		log.Printf("error replacing hostname to %s in address %s, returning original %v",newHostname, address, err)
+	if err != nil {
+		rlog.Errorf("error replacing hostname to %s in address %s, returning original %v", newHostname, address, err)
 		return address
 	}
 	replacedUrl.Host = newHostname

cli/cmd/common.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -63,8 +62,8 @@ func openBrowser(url string) {
 	default:
 		err = fmt.Errorf("unsupported platform")
 	}
-	if err != nil {
-		log.Fatal(err)
-	}
 
+	if err != nil {
+		logger.Log.Errorf("error while opening browser, %v", err)
+	}
 }

cli/cmd/logs.go

@@ -32,7 +32,7 @@ var logsCmd = &cobra.Command{
 
 		logger.Log.Debugf("Using file path %s", config.Config.Logs.FilePath())
 
-		if dumpLogsErr := fsUtils.DumpLogs(kubernetesProvider, ctx, config.Config.Logs.FilePath()); dumpLogsErr != nil {
+		if dumpLogsErr := fsUtils.DumpLogs(ctx, kubernetesProvider, config.Config.Logs.FilePath()); dumpLogsErr != nil {
 			logger.Log.Errorf("Failed dump logs %v", dumpLogsErr)
 		}
 

cli/cmd/tap.go

@@ -2,11 +2,13 @@ package cmd
 
 import (
 	"errors"
+	"fmt"
+	"os"
+
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"os"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
@@ -65,5 +67,8 @@ func init() {
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
-	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")
+	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
+
+	tapCmd.Flags().String(configStructs.EnforcePolicyFileDeprecated, defaultTapConfig.EnforcePolicyFileDeprecated, "Yaml file with policy rules")
+	tapCmd.Flags().MarkDeprecated(configStructs.EnforcePolicyFileDeprecated, fmt.Sprintf("Use --%s instead", configStructs.EnforcePolicyFile))
 }

cli/cmd/tapRunner.go

@@ -8,6 +8,10 @@ import (
 	"strings"
 	"time"
 
+	"gopkg.in/yaml.v3"
+	core "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
@@ -21,9 +25,7 @@ import (
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
-	yaml "gopkg.in/yaml.v3"
-	core "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
+	"github.com/up9inc/mizu/tap/api"
 )
 
 const (
@@ -35,7 +37,6 @@ type tapState struct {
 	apiServerService         *core.Service
 	currentlyTappedPods      []core.Pod
 	mizuServiceAccountExists bool
-	doNotRemoveConfigMap     bool
 }
 
 var state tapState
@@ -46,14 +47,23 @@ func RunMizuTap() {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
 		return
 	}
+
 	var mizuValidationRules string
-	if config.Config.Tap.EnforcePolicyFile != "" {
-		mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
+	if config.Config.Tap.EnforcePolicyFile != "" || config.Config.Tap.EnforcePolicyFileDeprecated != "" {
+		var trafficValidation string
+		if config.Config.Tap.EnforcePolicyFile != "" {
+			trafficValidation = config.Config.Tap.EnforcePolicyFile
+		} else {
+			trafficValidation = config.Config.Tap.EnforcePolicyFileDeprecated
+		}
+
+		mizuValidationRules, err = readValidationRules(trafficValidation)
 		if err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
 			return
 		}
 	}
+
 	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
 		logger.Log.Error(err)
@@ -66,7 +76,7 @@ func RunMizuTap() {
 	targetNamespaces := getNamespaces(kubernetesProvider)
 
 	if config.Config.IsNsRestrictedMode() {
-		if len(targetNamespaces) != 1 || !mizu.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
+		if len(targetNamespaces) != 1 || !shared.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
 			logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
 				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName)
 			return
@@ -74,7 +84,7 @@ func RunMizuTap() {
 	}
 
 	var namespacesStr string
-	if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+	if !shared.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
 		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
 	} else {
 		namespacesStr = "all namespaces"
@@ -89,7 +99,7 @@ func RunMizuTap() {
 
 	if len(state.currentlyTappedPods) == 0 {
 		var suggestionStr string
-		if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+		if !shared.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
 			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
 		}
 		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
@@ -99,18 +109,17 @@ func RunMizuTap() {
 		return
 	}
 
-	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
-
 	defer finishMizuExecution(kubernetesProvider)
-	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
+	if err := createMizuResources(ctx, kubernetesProvider, mizuApiFilteringOptions, mizuValidationRules); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		return
 	}
 
-	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
-	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel)
+	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel, mizuApiFilteringOptions)
+	go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel)
+	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel, mizuApiFilteringOptions)
 
-	//block until exit signal or error
+	// block until exit signal or error
 	waitForFinish(ctx, cancel)
 }
 
@@ -123,7 +132,7 @@ func readValidationRules(file string) (string, error) {
 	return string(newContent), nil
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error {
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions, mizuValidationRules string) error {
 	if !config.Config.IsNsRestrictedMode() {
 		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 			return err
@@ -134,15 +143,8 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
 		return err
 	}
 
-	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
-		return err
-	}
-
 	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
 		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
-		state.doNotRemoveConfigMap = true
-	} else if mizuValidationRules == "" {
-		state.doNotRemoveConfigMap = true
 	}
 
 	return nil
@@ -158,7 +160,7 @@ func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	return err
 }
 
-func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
+func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
 	var err error
 
 	state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
@@ -199,13 +201,13 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	return nil
 }
 
-func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
-	var compiledRegexSlice []*shared.SerializableRegexp
+func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
+	var compiledRegexSlice []*api.SerializableRegexp
 
 	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
-		compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
+		compiledRegexSlice = make([]*api.SerializableRegexp, 0)
 		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
-			compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
+			compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr)
 			if err != nil {
 				return nil, err
 			}
@@ -213,14 +215,16 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
 		}
 	}
 
-	return &shared.TrafficFilteringOptions{
+	return &api.TrafficFilteringOptions{
 		PlainTextMaskingRegexes:      compiledRegexSlice,
 		HealthChecksUserAgentHeaders: config.Config.Tap.HealthChecksUserAgentHeaders,
 		DisableRedaction:             config.Config.Tap.DisableRedaction,
 	}, nil
 }
 
-func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
+func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
+	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
+
 	if len(nodeToTappedPodIPMap) > 0 {
 		var serviceAccountName string
 		if state.mizuServiceAccountExists {
@@ -240,6 +244,7 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 			serviceAccountName,
 			config.Config.Tap.TapperResources,
 			config.Config.ImagePullPolicy(),
+			mizuApiFilteringOptions,
 		); err != nil {
 			return err
 		}
@@ -257,75 +262,108 @@ func finishMizuExecution(kubernetesProvider *kubernetes.Provider) {
 	telemetry.ReportAPICalls()
 	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
 	defer cancel()
-	dumpLogsIfNeeded(kubernetesProvider, removalCtx)
-	cleanUpMizuResources(kubernetesProvider, removalCtx, cancel)
+	dumpLogsIfNeeded(removalCtx, kubernetesProvider)
+	cleanUpMizuResources(removalCtx, cancel, kubernetesProvider)
 }
 
-func dumpLogsIfNeeded(kubernetesProvider *kubernetes.Provider, removalCtx context.Context) {
+func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
 	if !config.Config.DumpLogs {
 		return
 	}
 	mizuDir := mizu.GetMizuFolderPath()
 	filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
-	if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
+	if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
 		logger.Log.Errorf("Failed dump logs %v", err)
 	}
 }
 
-func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider, removalCtx context.Context, cancel context.CancelFunc) {
+func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
 	logger.Log.Infof("\nRemoving mizu resources\n")
 
-	if !config.Config.IsNsRestrictedMode() {
-		if err := kubernetesProvider.RemoveNamespace(removalCtx, config.Config.MizuResourcesNamespace); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-			return
-		}
+	var leftoverResources []string
+
+	if config.Config.IsNsRestrictedMode() {
+		leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider)
 	} else {
-		if err := kubernetesProvider.RemovePod(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-		}
-
-		if err := kubernetesProvider.RemoveService(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-		}
-
-		if err := kubernetesProvider.RemoveDaemonSet(removalCtx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-		}
-
-		if !state.doNotRemoveConfigMap {
-			if err := kubernetesProvider.RemoveConfigMap(removalCtx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
-				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-			}
-		}
-
+		leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider)
 	}
 
-	if state.mizuServiceAccountExists {
-		if !config.Config.IsNsRestrictedMode() {
-			if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
-				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err)))
-				return
-			}
-		} else {
-			if err := kubernetesProvider.RemoveServicAccount(removalCtx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
-				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-				return
-			}
-
-			if err := kubernetesProvider.RemoveRole(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
-				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-			}
-
-			if err := kubernetesProvider.RemoveRoleBinding(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
-				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-			}
+	if len(leftoverResources) > 0 {
+		errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", logger.GetLogFilePath())
+		for _, resource := range leftoverResources {
+			errMsg += "\n- " + resource
 		}
+		logger.Log.Errorf(uiUtils.Error, errMsg)
+	}
+}
+
+func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) []string {
+	leftoverResources := make([]string, 0)
+
+	if err := kubernetesProvider.RemovePod(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+		resourceDesc := fmt.Sprintf("Pod %s in namespace %s", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
 	}
 
-	if !config.Config.IsNsRestrictedMode() {
-		waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider)
+	if err := kubernetesProvider.RemoveService(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+		resourceDesc := fmt.Sprintf("Service %s in namespace %s", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
 	}
+
+	if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
+		resourceDesc := fmt.Sprintf("DaemonSet %s in namespace %s", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
+	if err := kubernetesProvider.RemoveConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
+		resourceDesc := fmt.Sprintf("ConfigMap %s in namespace %s", mizu.ConfigMapName, config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
+	if err := kubernetesProvider.RemoveServicAccount(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
+		resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
+	if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
+		resourceDesc := fmt.Sprintf("Role %s in namespace %s", mizu.RoleName, config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
+	if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
+		resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", mizu.RoleBindingName, config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
+	return leftoverResources
+}
+
+func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) []string {
+	leftoverResources := make([]string, 0)
+
+	if err := kubernetesProvider.RemoveNamespace(ctx, config.Config.MizuResourcesNamespace); err != nil {
+		resourceDesc := fmt.Sprintf("Namespace %s", config.Config.MizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	} else {
+		defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider)
+	}
+
+	if err := kubernetesProvider.RemoveClusterRole(ctx, mizu.ClusterRoleName); err != nil {
+		resourceDesc := fmt.Sprintf("ClusterRole %s", mizu.ClusterRoleName)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
+	if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, mizu.ClusterRoleBindingName); err != nil {
+		resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", mizu.ClusterRoleBindingName)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
+	return leftoverResources
+}
+
+func handleDeletionError(err error, resourceDesc string, leftoverResources *[]string) {
+	logger.Log.Debugf("Error removing %s: %v", resourceDesc, errormessage.FormatError(err))
+	*leftoverResources = append(*leftoverResources, resourceDesc)
 }
 
 func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
@@ -346,7 +384,7 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k
 	}
 }
 
-func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) {
+func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc, mizuApiFilteringOptions *api.TrafficFilteringOptions) {
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, config.Config.Tap.PodRegex())
 
 	restartTappers := func() {
@@ -365,13 +403,8 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 			logger.Log.Debugf("[Error] failed update tapped pods %v", err)
 		}
 
-		nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
-			cancel()
-		}
-		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
+		if err := updateMizuTappers(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating tappers: %v", errormessage.FormatError(err)))
 			cancel()
 		}
 	}
@@ -379,13 +412,28 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 	for {
 		select {
-		case pod := <-added:
+		case pod, ok := <-added:
+			if !ok {
+				added = nil
+				continue
+			}
+
 			logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-		case pod := <-removed:
+		case pod, ok := <-removed:
+			if !ok {
+				removed = nil
+				continue
+			}
+
 			logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-		case pod := <-modified:
+		case pod, ok := <-modified:
+			if !ok {
+				modified = nil
+				continue
+			}
+
 			logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
@@ -396,8 +444,12 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 			if pod.Status.PodIP != "" {
 				restartTappersDebouncer.SetOn()
 			}
+		case err, ok := <-errorChan:
+			if !ok {
+				errorChan = nil
+				continue
+			}
 
-		case err := <-errorChan:
 			logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
 			restartTappersDebouncer.Cancel()
 			// TODO: Does this also perform cleanup?
@@ -470,39 +522,68 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
 	return missingPods
 }
 
-func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, mizuApiFilteringOptions *api.TrafficFilteringOptions) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
 	isPodReady := false
 	timeAfter := time.After(25 * time.Second)
 	for {
 		select {
-		case <-ctx.Done():
-			logger.Log.Debugf("Watching API Server pod loop, ctx done")
-			return
-		case <-added:
+		case _, ok := <-added:
+			if !ok {
+				added = nil
+				continue
+			}
+
 			logger.Log.Debugf("Watching API Server pod loop, added")
-			continue
-		case <-removed:
+		case _, ok := <-removed:
+			if !ok {
+				removed = nil
+				continue
+			}
+
 			logger.Log.Infof("%s removed", mizu.ApiServerPodName)
 			cancel()
 			return
-		case modifiedPod := <-modified:
-			if modifiedPod == nil {
-				logger.Log.Debugf("Watching API Server pod loop, modifiedPod with nil")
+		case modifiedPod, ok := <-modified:
+			if !ok {
+				modified = nil
 				continue
 			}
+
 			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
+
+			if modifiedPod.Status.Phase == core.PodPending {
+				if modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
+					logger.Log.Debugf("Wasn't able to deploy the API server. Reason: \"%s\"", modifiedPod.Status.Conditions[0].Message)
+					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server, for more info check logs at %s", logger.GetLogFilePath()))
+					cancel()
+					break
+				}
+
+				if len(modifiedPod.Status.ContainerStatuses) > 0 && modifiedPod.Status.ContainerStatuses[0].State.Waiting != nil && modifiedPod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImagePull" {
+					logger.Log.Debugf("Wasn't able to deploy the API server. (ErrImagePull) Reason: \"%s\"", modifiedPod.Status.ContainerStatuses[0].State.Waiting.Message)
+					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server: failed to pull the image, for more info check logs at %v", logger.GetLogFilePath()))
+					cancel()
+					break
+				}
+			}
+
 			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 				isPodReady = true
 				go startProxyReportErrorIfAny(kubernetesProvider, cancel)
 
 				url := GetApiServerUrl()
 				if err := apiserver.Provider.InitAndTestConnection(url); err != nil {
-					logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs")
+					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", logger.GetLogFilePath()))
 					cancel()
 					break
 				}
+				if err := updateMizuTappers(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
+					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating tappers: %v", errormessage.FormatError(err)))
+					cancel()
+				}
+
 				logger.Log.Infof("Mizu is available at %s\n", url)
 				openBrowser(url)
 				requestForAnalysisIfNeeded()
@@ -510,14 +591,89 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
 				}
 			}
+		case err, ok := <-errorChan:
+			if !ok {
+				errorChan = nil
+				continue
+			}
+
+			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace, error: %v", config.Config.MizuResourcesNamespace, err)
+			cancel()
+
 		case <-timeAfter:
 			if !isPodReady {
 				logger.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time")
 				cancel()
 			}
-		case <-errorChan:
-			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace", config.Config.MizuResourcesNamespace)
+		case <-ctx.Done():
+			logger.Log.Debugf("Watching API Server pod loop, ctx done")
+			return
+		}
+	}
+}
+
+func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", mizu.TapperDaemonSetName))
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
+	var prevPodPhase core.PodPhase
+	for {
+		select {
+		case addedPod, ok := <-added:
+			if !ok {
+				added = nil
+				continue
+			}
+
+			logger.Log.Debugf("Tapper is created [%s]", addedPod.Name)
+		case removedPod, ok := <-removed:
+			if !ok {
+				removed = nil
+				continue
+			}
+
+			logger.Log.Debugf("Tapper is removed [%s]", removedPod.Name)
+		case modifiedPod, ok := <-modified:
+			if !ok {
+				modified = nil
+				continue
+			}
+
+			if modifiedPod.Status.Phase == core.PodPending && modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
+				logger.Log.Infof(uiUtils.Red, "Wasn't able to deploy the tapper %s. Reason: \"%s\"", modifiedPod.Name, modifiedPod.Status.Conditions[0].Message)
+				cancel()
+				break
+			}
+
+			podStatus := modifiedPod.Status
+			if podStatus.Phase == core.PodPending && prevPodPhase == podStatus.Phase {
+				logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
+				continue
+			}
+			prevPodPhase = podStatus.Phase
+
+			if podStatus.Phase == core.PodRunning {
+				state := podStatus.ContainerStatuses[0].State
+				if state.Terminated != nil {
+					switch state.Terminated.Reason {
+					case "OOMKilled":
+						logger.Log.Infof(uiUtils.Red, "Tapper %s was terminated (reason: OOMKilled). You should consider increasing machine resources.", modifiedPod.Name)
+					}
+				}
+			}
+
+			logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
+		case err, ok := <-errorChan:
+			if !ok {
+				errorChan = nil
+				continue
+			}
+
+			logger.Log.Debugf("[Error] Error in mizu tapper watch, err: %v", err)
 			cancel()
+
+		case <-ctx.Done():
+			logger.Log.Debugf("Watching tapper pod loop, ctx done")
+			return
 		}
 	}
 }
@@ -563,7 +719,7 @@ func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
 	if config.Config.Tap.AllNamespaces {
 		return []string{mizu.K8sAllNamespaces}
 	} else if len(config.Config.Tap.Namespaces) > 0 {
-		return mizu.Unique(config.Config.Tap.Namespaces)
+		return shared.Unique(config.Config.Tap.Namespaces)
 	} else {
 		return []string{kubernetesProvider.CurrentNamespace()}
 	}

cli/cmd/viewRunner.go

@@ -47,7 +47,7 @@ func runMizuView() {
 	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
 
 	if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil {
-		logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs")
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", logger.GetLogFilePath()))
 		return
 	}
 

cli/config/config.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/up9inc/mizu/cli/logger"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/shared"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -89,7 +89,7 @@ func initFlag(f *pflag.Flag) {
 	configElemValue := reflect.ValueOf(&Config).Elem()
 
 	var flagPath []string
-	if mizu.Contains([]string{ConfigFilePathCommandName}, f.Name) {
+	if shared.Contains([]string{ConfigFilePathCommandName}, f.Name) {
 		flagPath = []string{f.Name}
 	} else {
 		flagPath = []string{cmdName, f.Name}

cli/config/configStructs/tapConfig.go

@@ -16,7 +16,8 @@ const (
 	DisableRedactionTapName       = "no-redact"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	DryRunTapName                 = "dry-run"
-	EnforcePolicyFile             = "test-rules"
+	EnforcePolicyFile             = "traffic-validation-file"
+	EnforcePolicyFileDeprecated   = "test-rules"
 )
 
 type TapConfig struct {
@@ -32,7 +33,8 @@ type TapConfig struct {
 	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
 	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
 	DryRun                       bool      `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile            string    `yaml:"test-rules"`
+	EnforcePolicyFile            string    `yaml:"traffic-validation-file"`
+	EnforcePolicyFileDeprecated  string    `yaml:"test-rules,omitempty" readonly:""`
 	ApiServerResources           Resources `yaml:"api-server-resources"`
 	TapperResources              Resources `yaml:"tapper-resources"`
 }

cli/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0
+	github.com/up9inc/mizu/tap/api v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
@@ -19,3 +20,5 @@ require (
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
+
+replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api

cli/kubernetes/provider.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap/api"
 	core "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -150,7 +151,7 @@ type ApiServerOptions struct {
 	PodImage                string
 	ServiceAccountName      string
 	IsNamespaceRestricted   bool
-	MizuApiFilteringOptions *shared.TrafficFilteringOptions
+	MizuApiFilteringOptions *api.TrafficFilteringOptions
 	MaxEntriesDBSizeBytes   int64
 	Resources               configStructs.Resources
 	ImagePullPolicy         core.PullPolicy
@@ -267,67 +268,21 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 	return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
 }
 
-func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, serviceAccountName string) (bool, error) {
-	serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{})
-	return provider.doesResourceExist(serviceAccount, err)
-}
-
-func (provider *Provider) DoesConfigMapExist(ctx context.Context, namespace string, name string) (bool, error) {
-	resource, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
 func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) {
 	resource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
 	return provider.doesResourceExist(resource, err)
 }
 
-func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) {
-	resource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
-func (provider *Provider) DoesClusterRoleExist(ctx context.Context, name string) (bool, error) {
-	resource, err := provider.clientSet.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
-func (provider *Provider) DoesClusterRoleBindingExist(ctx context.Context, name string) (bool, error) {
-	resource, err := provider.clientSet.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
-func (provider *Provider) DoesRoleExist(ctx context.Context, namespace string, name string) (bool, error) {
-	resource, err := provider.clientSet.RbacV1().Roles(namespace).Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
-func (provider *Provider) DoesRoleBindingExist(ctx context.Context, namespace string, name string) (bool, error) {
-	resource, err := provider.clientSet.RbacV1().RoleBindings(namespace).Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
-func (provider *Provider) DoesPodExist(ctx context.Context, namespace string, name string) (bool, error) {
-	resource, err := provider.clientSet.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
-func (provider *Provider) DoesDaemonSetExist(ctx context.Context, namespace string, name string) (bool, error) {
-	resource, err := provider.clientSet.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(resource, err)
-}
-
 func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) {
-	var statusError *k8serrors.StatusError
-	if errors.As(err, &statusError) {
-		// expected behavior when resource does not exist
-		if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound {
-			return false, nil
-		}
+	// Getting NotFound error is the expected behavior when a resource does not exist.
+	if k8serrors.IsNotFound(err) {
+		return false, nil
 	}
+
 	if err != nil {
 		return false, err
 	}
+
 	return resource != nil, nil
 }
 
@@ -440,115 +395,63 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context,
 }
 
 func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error {
-	if isFound, err := provider.DoesNamespaceExist(ctx, name); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
-}
-
-func (provider *Provider) RemoveNonNamespacedResources(ctx context.Context, clusterRoleName string, clusterRoleBindingName string) error {
-	if err := provider.RemoveClusterRole(ctx, clusterRoleName); err != nil {
-		return err
-	}
-
-	if err := provider.RemoveClusterRoleBinding(ctx, clusterRoleBindingName); err != nil {
-		return err
-	}
-
-	return nil
+	err := provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error {
-	if isFound, err := provider.DoesClusterRoleExist(ctx, name); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
+	err := provider.clientSet.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error {
-	if isFound, err := provider.DoesClusterRoleBindingExist(ctx, name); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
+	err := provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveRoleBinding(ctx context.Context, namespace string, name string) error {
-	if isFound, err := provider.DoesRoleBindingExist(ctx, namespace, name); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	err := provider.clientSet.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveRole(ctx context.Context, namespace string, name string) error {
-	if isFound, err := provider.DoesRoleExist(ctx, namespace, name); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	err := provider.clientSet.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveServicAccount(ctx context.Context, namespace string, name string) error {
-	if isFound, err := provider.DoesServiceAccountExist(ctx, namespace, name); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
-	if isFound, err := provider.DoesPodExist(ctx, namespace, podName); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
+	err := provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error {
-	if isFound, err := provider.DoesConfigMapExist(ctx, namespace, configMapName); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{})
+	err := provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
-	if isFound, err := provider.DoesServicesExist(ctx, namespace, serviceName); err != nil {
-		return err
-	} else if !isFound {
-		return nil
-	}
-
-	return provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{})
+	err := provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
 }
 
 func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
-	if isFound, err := provider.DoesDaemonSetExist(ctx, namespace, daemonSetName); err != nil {
-		return err
-	} else if !isFound {
+	err := provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
+}
+
+func (provider *Provider) handleRemovalError(err error) error {
+	// Ignore NotFound - There is nothing to delete.
+	// Ignore Forbidden - Assume that a user could not have created the resource in the first place.
+	if k8serrors.IsNotFound(err) || k8serrors.IsForbidden(err) {
 		return nil
 	}
 
-	return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
+	return err
 }
 
 func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string) error {
@@ -575,7 +478,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources configStructs.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
 	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodIPMap) == 0 {
@@ -587,6 +490,11 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		return err
 	}
 
+	marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions)
+	if err != nil {
+		return err
+	}
+
 	mizuCmd := []string{
 		"./mizuagent",
 		"-i", "any",
@@ -604,6 +512,8 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
 		applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)),
+		applyconfcore.EnvVar().WithName(shared.GoGCEnvVar).WithValue("12800"),
+		applyconfcore.EnvVar().WithName(shared.MizuFilteringOptionsEnvVar).WithValue(string(marshaledFilteringOptions)),
 	)
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.NodeNameEnvVar).WithValueFrom(
@@ -723,7 +633,7 @@ func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, r
 	return matchingPods, nil
 }
 
-func (provider *Provider) GetPodLogs(namespace string, podName string, ctx context.Context) (string, error) {
+func (provider *Provider) GetPodLogs(ctx context.Context, namespace string, podName string) (string, error) {
 	podLogOpts := core.PodLogOptions{}
 	req := provider.clientSet.CoreV1().Pods(namespace).GetLogs(podName, &podLogOpts)
 	podLogs, err := req.Stream(ctx)
@@ -739,7 +649,7 @@ func (provider *Provider) GetPodLogs(namespace string, podName string, ctx conte
 	return str, nil
 }
 
-func (provider *Provider) GetNamespaceEvents(namespace string, ctx context.Context) (string, error) {
+func (provider *Provider) GetNamespaceEvents(ctx context.Context, namespace string) (string, error) {
 	eventsOpts := metav1.ListOptions{}
 	eventList, err := provider.clientSet.CoreV1().Events(namespace).List(ctx, eventsOpts)
 	if err != nil {

cli/mizu/controlSocket.go

@@ -1,42 +0,0 @@
-package mizu
-
-import (
-	"encoding/json"
-	"github.com/gorilla/websocket"
-	"github.com/up9inc/mizu/shared"
-	core "k8s.io/api/core/v1"
-	"time"
-)
-
-type ControlSocket struct {
-	connection *websocket.Conn
-}
-
-func CreateControlSocket(socketServerAddress string) (*ControlSocket, error) {
-	connection, err := shared.ConnectToSocketServer(socketServerAddress, 30, 2 * time.Second, true)
-	if err != nil {
-		return nil, err
-	} else {
-		return &ControlSocket{connection: connection}, nil
-	}
-}
-
-func (controlSocket *ControlSocket) SendNewTappedPodsListMessage(pods []core.Pod) error {
-	podInfos := make([]shared.PodInfo, 0)
-	for _, pod := range pods {
-		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
-	}
-	tapStatus := shared.TapStatus{Pods: podInfos}
-	socketMessage := shared.CreateWebSocketStatusMessage(tapStatus)
-
-	jsonMessage, err := json.Marshal(socketMessage)
-	if err != nil {
-		return err
-	}
-	err = controlSocket.connection.WriteMessage(websocket.TextMessage, jsonMessage)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -12,7 +12,7 @@ import (
 	"regexp"
 )
 
-func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error {
+func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath string) error {
 	podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
 	pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{config.Config.MizuResourcesNamespace})
 	if err != nil {
@@ -32,7 +32,7 @@ func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath strin
 	defer zipWriter.Close()
 
 	for _, pod := range pods {
-		logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx)
+		logs, err := provider.GetPodLogs(ctx, pod.Namespace, pod.Name)
 		if err != nil {
 			logger.Log.Errorf("Failed to get logs, %v", err)
 			continue
@@ -47,7 +47,7 @@ func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath strin
 		}
 	}
 
-	events, err := provider.GetNamespaceEvents(config.Config.MizuResourcesNamespace, ctx)
+	events, err := provider.GetNamespaceEvents(ctx, config.Config.MizuResourcesNamespace)
 	if err != nil {
 		logger.Log.Debugf("Failed to get k8b events, %v", err)
 	} else {

cli/mizu/version/versionCheck.go

@@ -8,6 +8,7 @@ import (
 	"github.com/up9inc/mizu/cli/mizu"
 	"io/ioutil"
 	"net/http"
+	"runtime"
 	"time"
 
 	"github.com/google/go-github/v37/github"
@@ -76,7 +77,7 @@ func CheckNewerVersion(versionChan chan string) {
 	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
 
 	if gitHubVersionSemVer.GreaterThan(currentSemVer) {
-		versionChan <- fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL)
+		versionChan <- fmt.Sprintf("Update available! %v -> %v (curl -Lo mizu %v/mizu_%s_amd64 && chmod 755 mizu)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL, runtime.GOOS)
 	} else {
 		versionChan <- ""
 	}

docs/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at mizu@up9.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

docs/CONTRIBUTING.md

@@ -1,18 +1,20 @@
-![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
-# CONTRIBUTE
+![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
+
+# Contributing to Mizu
+
 We welcome code contributions from the community.
 Please read and follow the guidelines below.
 
 ## Communication
+
 * Before starting work on a major feature, please reach out to us via [GitHub](https://github.com/up9inc/mizu), [Slack](https://join.slack.com/share/zt-u6bbs3pg-X1zhQOXOH0yEoqILgH~csw), [email](mailto:mizu@up9.com), etc. We will make sure no one else is already working on it. A _major feature_ is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior
 * Small patches and bug fixes don't need prior communication.
 
-## Contribution requirements
+## Contribution Requirements
+
 * Code style - most of the code is written in Go, please follow [these guidelines](https://golang.org/doc/effective_go)
-* Go-tools compatible (`go get`, `go test`, etc)
-* Unit-test coverage can’t go down ..
+* Go-tools compatible (`go get`, `go test`, etc.)
+* Code coverage for unit tests must not decrease.
 * Code must be usefully commented. Not only for developers on the project, but also for external users of these packages
 * When reviewing PRs, you are encouraged to use Golang's [code review comments page](https://github.com/golang/go/wiki/CodeReviewComments)
-
-
-
+* Project follows [Google JSON Style Guide](https://google.github.io/styleguide/jsoncstyleguide.xml) for the REST APIs that are provided.

docs/PERMISSIONS.md

@@ -1,4 +1,4 @@
-![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
 # Kubernetes permissions for MIZU  
 
 This document describes in details all permissions required for full and correct operation of Mizu

docs/POLICY_RULES.md

@@ -1,37 +1,34 @@
 
-# API rules validation
+# Traffic validation rules
 
-This feature allows you to define set of simple rules, and test the API against them.
+This feature allows you to define set of simple rules, and test the traffic against them.
 Such validation may test response for specific JSON fields, headers, etc.
 
 ## Examples
 
-
-Example 1: HTTP request (REST API call) that didn’t pass validation is highlighted in red
+Example 1: HTTP request (REST API call) that didn't pass validation is highlighted in red
 
 ![Simple UI](../assets/validation-example1.png)
 
 - - -
 
-
 Example 2: Details pane shows the validation rule details and whether it passed or failed
 
 ![Simple UI](../assets/validation-example2.png)
 
 
 ## How to use
+
 To use this feature - create simple rules file (see details below) and pass this file as parameter to `mizu tap` command. For example, if rules are stored in file named `rules.yaml` — run the following command:
 
-
 ```shell
-mizu tap --test-rules rules.yaml PODNAME
+mizu tap --traffic-validation-file rules.yaml
 ```
 
 
-
 ## Rules file structure
 
-The structure of the test-rules-file is:
+The structure of the traffic-validation-file is:
 
 * `name`: string, name of the rule
 * `type`: string, type of the rule, must be `json` or `header` or `latency`
@@ -62,6 +59,7 @@ rules:
   service: "carts.*"
 ```
 
+
 ### Explanation:
 
 * First rule `holy-in-name-property`:
@@ -74,5 +72,4 @@ rules:
 
 * Third rule `latency-test`:
 
-  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1` will be marked as failure, marked as success otherwise.
-
+  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1ms` will be marked as failure, marked as success otherwise.

docs/TESTING.md

@@ -0,0 +1,33 @@
+![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
+# Testing guidelines
+
+## Generic guidelines
+* Use "[testing](https://pkg.go.dev/testing)" package
+* Write [Table-driven tests using subtests](https://go.dev/blog/subtests)
+* Use cleanup in test/subtest in order to clean up resources
+* Name the test func "Test<tested_func_name><tested_case>"
+
+## Unit tests
+* Position the test file inside the folder of the tested package
+* In case of internal func testing
+  * Name the test file "<tested_file_name>_internal_test.go"
+  * Name the test package same as the package being tested
+  * Example - [Config](../cli/config/config_internal_test.go)
+* In case of exported func testing
+  * Name the test file "<tested_file_name>_test.go"
+  * Name the test package "<tested_package>_test"
+  * Example - [Slice Utils](../cli/mizu/sliceUtils_test.go)
+* Make sure to run test coverage to make sure you covered all the cases and lines in the func  
+  
+## Acceptance tests
+* Position the test file inside the [acceptance tests folder](../acceptanceTests)
+* Name the file "<tested_command>_test.go"
+* Name the package "acceptanceTests"
+* Do not run as part of the short tests
+* Use/Create generic test utils func in acceptanceTests/testsUtils
+* Don't use sleep inside the tests - active check 
+* Running acceptance tests locally
+  * Switch to the branch that is being tested
+  * Run acceptanceTests/setup.sh
+  * Run tests (make acceptance-test)
+* Example - [Tap](../acceptanceTests/tap_test.go)

shared/consts.go

@@ -8,4 +8,5 @@ const (
 	MaxEntriesDBSizeBytesEnvVar      = "MAX_ENTRIES_DB_BYTES"
 	RulePolicyPath                   = "/app/enforce-policy/"
 	RulePolicyFileName               = "enforce-policy.yaml"
+	GoGCEnvVar                       = "GOGC"
 )

shared/models.go

@@ -1,8 +1,8 @@
 package shared
 
 import (
-	"fmt"
 	"io/ioutil"
+	"log"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -74,12 +74,6 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 	}
 }
 
-type TrafficFilteringOptions struct {
-	HealthChecksUserAgentHeaders []string
-	PlainTextMaskingRegexes      []*SerializableRegexp
-	DisableRedaction             bool
-}
-
 type VersionResponse struct {
 	SemVer string `json:"semver"`
 }
@@ -89,25 +83,33 @@ type RulesPolicy struct {
 }
 
 type RulePolicy struct {
-	Type    string `yaml:"type"`
-	Service string `yaml:"service"`
-	Path    string `yaml:"path"`
-	Method  string `yaml:"method"`
-	Key     string `yaml:"key"`
-	Value   string `yaml:"value"`
-	Latency int64  `yaml:"latency"`
-	Name    string `yaml:"name"`
+	Type         string `yaml:"type"`
+	Service      string `yaml:"service"`
+	Path         string `yaml:"path"`
+	Method       string `yaml:"method"`
+	Key          string `yaml:"key"`
+	Value        string `yaml:"value"`
+	ResponseTime int64  `yaml:"response-time"`
+	Name         string `yaml:"name"`
+}
+
+type RulesMatched struct {
+	Matched bool       `json:"matched"`
+	Rule    RulePolicy `json:"rule"`
 }
 
 func (r *RulePolicy) validateType() bool {
-	permitedTypes := []string{"json", "header", "latency"}
+	permitedTypes := []string{"json", "header", "slo"}
 	_, found := Find(permitedTypes, r.Type)
 	if !found {
-		fmt.Printf("\nRule with name %s will be ignored. Err: only json, header and latency types are supported on rule definition.\n", r.Name)
+		log.Printf("Error: %s. ", r.Name)
+		log.Printf("Only json, header and slo types are supported on rule definition. This rule will be ignored\n")
+		found = false
 	}
-	if strings.ToLower(r.Type) == "latency" {
-		if r.Latency == 0 {
-			fmt.Printf("\nRule with name %s will be ignored. Err: when type=latency, the field Latency should be specified and have a value >= 1\n\n", r.Name)
+	if strings.ToLower(r.Type) == "slo" {
+		if r.ResponseTime <= 0 {
+			log.Printf("Error: %s. ", r.Name)
+			log.Printf("When type=slo, the field response-time should be specified and have a value >= 1\n\n")
 			found = false
 		}
 	}
@@ -125,10 +127,6 @@ func (rules *RulesPolicy) ValidateRulesPolicy() []int {
 	return invalidIndex
 }
 
-func (rules *RulesPolicy) RemoveRule(idx int) {
-	rules.Rules = append(rules.Rules[:idx], rules.Rules[idx+1:]...)
-}
-
 func Find(slice []string, val string) (int, bool) {
 	for i, item := range slice {
 		if item == val {
@@ -149,10 +147,15 @@ func DecodeEnforcePolicy(path string) (RulesPolicy, error) {
 		return enforcePolicy, err
 	}
 	invalidIndex := enforcePolicy.ValidateRulesPolicy()
+	var k = 0
 	if len(invalidIndex) != 0 {
-		for i := range invalidIndex {
-			enforcePolicy.RemoveRule(invalidIndex[i])
+		for i, rule := range enforcePolicy.Rules {
+			if !ContainsInt(invalidIndex, i) {
+				enforcePolicy.Rules[k] = rule
+				k++
+			}
 		}
+		enforcePolicy.Rules = enforcePolicy.Rules[:k]
 	}
 	return enforcePolicy, nil
 }

shared/sliceUtils.go

@@ -1,4 +1,4 @@
-package mizu
+package shared
 
 func Contains(slice []string, containsValue string) bool {
 	for _, sliceValue := range slice {
@@ -10,6 +10,16 @@ func Contains(slice []string, containsValue string) bool {
 	return false
 }
 
+func ContainsInt(slice []int, containsValue int) bool {
+	for _, sliceValue := range slice {
+		if sliceValue == containsValue {
+			return true
+		}
+	}
+	return false
+}
+
+
 func Unique(slice []string) []string {
 	keys := make(map[string]bool)
 	var list []string

shared/sliceUtils_test.go

@@ -1,8 +1,8 @@
-package mizu_test
+package shared_test
 
 import (
 	"fmt"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/shared"
 	"reflect"
 	"testing"
 )
@@ -21,7 +21,7 @@ func TestContainsExists(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			actual := shared.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -43,7 +43,7 @@ func TestContainsNotExists(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			actual := shared.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -63,7 +63,7 @@ func TestContainsEmptySlice(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			actual := shared.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -83,7 +83,7 @@ func TestContainsNilSlice(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			actual := shared.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -102,7 +102,7 @@ func TestUniqueNoDuplicateValues(t *testing.T) {
 
 	for index, test := range tests {
 		t.Run(fmt.Sprintf("%v", index), func(t *testing.T) {
-			actual := mizu.Unique(test.Slice)
+			actual := shared.Unique(test.Slice)
 			if !reflect.DeepEqual(test.Expected, actual) {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -121,7 +121,7 @@ func TestUniqueDuplicateValues(t *testing.T) {
 
 	for index, test := range tests {
 		t.Run(fmt.Sprintf("%v", index), func(t *testing.T) {
-			actual := mizu.Unique(test.Slice)
+			actual := shared.Unique(test.Slice)
 			if !reflect.DeepEqual(test.Expected, actual) {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}

shared/socket_client.go

@@ -1,39 +0,0 @@
-package shared
-
-import (
-	"fmt"
-	"github.com/gorilla/websocket"
-	"time"
-)
-
-const (
-	DEFAULT_SOCKET_RETRIES          = 3
-	DEFAULT_SOCKET_RETRY_SLEEP_TIME = time.Second * 10
-)
-
-func ConnectToSocketServer(address string, retries int, retrySleepTime time.Duration, hideTimeoutErrors bool) (*websocket.Conn, error) {
-	var err error
-	var connection *websocket.Conn
-	try := 0
-
-	// Connection to server fails if client pod is up before server.
-	// Retries solve this issue.
-	for try < retries {
-		connection, _, err = websocket.DefaultDialer.Dial(address, nil)
-		if err != nil {
-			try++
-			if !hideTimeoutErrors {
-				fmt.Printf("Failed connecting to websocket server: %s, (%v,%+v)\n", err, err, err)
-			}
-		} else {
-			break
-		}
-		time.Sleep(retrySleepTime)
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	return connection, nil
-}

tap/api/api.go

@@ -9,19 +9,19 @@ import (
 
 type Protocol struct {
 	Name            string   `json:"name"`
-	LongName        string   `json:"long_name"`
+	LongName        string   `json:"longName"`
 	Abbreviation    string   `json:"abbreviation"`
 	Version         string   `json:"version"`
-	BackgroundColor string   `json:"background_color"`
-	ForegroundColor string   `json:"foreground_color"`
-	FontSize        int8     `json:"font_size"`
-	ReferenceLink   string   `json:"reference_link"`
+	BackgroundColor string   `json:"backgroundColor"`
+	ForegroundColor string   `json:"foregroundColor"`
+	FontSize        int8     `json:"fontSize"`
+	ReferenceLink   string   `json:"referenceLink"`
 	Ports           []string `json:"ports"`
 	Priority        uint8    `json:"priority"`
 }
 
 type Extension struct {
-	Protocol   Protocol
+	Protocol   *Protocol
 	Path       string
 	Plug       *plugin.Plugin
 	Dissector  Dissector
@@ -50,8 +50,8 @@ type CounterPair struct {
 }
 
 type GenericMessage struct {
-	IsRequest   bool        `json:"is_request"`
-	CaptureTime time.Time   `json:"capture_time"`
+	IsRequest   bool        `json:"isRequest"`
+	CaptureTime time.Time   `json:"captureTime"`
 	Payload     interface{} `json:"payload"`
 }
 
@@ -72,16 +72,22 @@ type SuperTimer struct {
 	CaptureTime time.Time
 }
 
+type SuperIdentifier struct {
+	Protocol       *Protocol
+	IsClosedOthers bool
+}
+
 type Dissector interface {
 	Register(*Extension)
 	Ping()
-	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, emitter Emitter) error
+	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error
 	Analyze(item *OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *MizuEntry
 	Summarize(entry *MizuEntry) *BaseEntryDetails
 	Represent(entry *MizuEntry) (protocol Protocol, object []byte, bodySize int64, err error)
 }
 
 type Emitting struct {
+	AppStats      *AppStats
 	OutputChannel chan *OutputChannelItem
 }
 
@@ -91,57 +97,67 @@ type Emitter interface {
 
 func (e *Emitting) Emit(item *OutputChannelItem) {
 	e.OutputChannel <- item
+	e.AppStats.IncMatchedPairs()
 }
 
 type MizuEntry struct {
-	ID                  uint `gorm:"primarykey"`
-	CreatedAt           time.Time
-	UpdatedAt           time.Time
-	ProtocolName        string `json:"protocol_key" gorm:"column:protocolKey"`
-	ProtocolVersion     string `json:"protocol_version" gorm:"column:protocolVersion"`
-	Entry               string `json:"entry,omitempty" gorm:"column:entry"`
-	EntryId             string `json:"entryId" gorm:"column:entryId"`
-	Url                 string `json:"url" gorm:"column:url"`
-	Method              string `json:"method" gorm:"column:method"`
-	Status              int    `json:"status" gorm:"column:status"`
-	RequestSenderIp     string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
-	Service             string `json:"service" gorm:"column:service"`
-	Timestamp           int64  `json:"timestamp" gorm:"column:timestamp"`
-	ElapsedTime         int64  `json:"elapsedTime" gorm:"column:elapsedTime"`
-	Path                string `json:"path" gorm:"column:path"`
-	ResolvedSource      string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
-	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
-	SourceIp            string `json:"sourceIp,omitempty" gorm:"column:sourceIp"`
-	DestinationIp       string `json:"destinationIp,omitempty" gorm:"column:destinationIp"`
-	SourcePort          string `json:"sourcePort,omitempty" gorm:"column:sourcePort"`
-	DestinationPort     string `json:"destinationPort,omitempty" gorm:"column:destinationPort"`
-	IsOutgoing          bool   `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
-	EstimatedSizeBytes  int    `json:"-" gorm:"column:estimatedSizeBytes"`
+	ID                      uint `gorm:"primarykey"`
+	CreatedAt               time.Time
+	UpdatedAt               time.Time
+	ProtocolName            string `json:"protocolName" gorm:"column:protocolName"`
+	ProtocolLongName        string `json:"protocolLongName" gorm:"column:protocolLongName"`
+	ProtocolAbbreviation    string `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"`
+	ProtocolVersion         string `json:"protocolVersion" gorm:"column:protocolVersion"`
+	ProtocolBackgroundColor string `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"`
+	ProtocolForegroundColor string `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"`
+	ProtocolFontSize        int8   `json:"protocolFontSize" gorm:"column:protocolFontSize"`
+	ProtocolReferenceLink   string `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"`
+	Entry                   string `json:"entry,omitempty" gorm:"column:entry"`
+	EntryId                 string `json:"entryId" gorm:"column:entryId"`
+	Url                     string `json:"url" gorm:"column:url"`
+	Method                  string `json:"method" gorm:"column:method"`
+	Status                  int    `json:"status" gorm:"column:status"`
+	RequestSenderIp         string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
+	Service                 string `json:"service" gorm:"column:service"`
+	Timestamp               int64  `json:"timestamp" gorm:"column:timestamp"`
+	ElapsedTime             int64  `json:"elapsedTime" gorm:"column:elapsedTime"`
+	Path                    string `json:"path" gorm:"column:path"`
+	ResolvedSource          string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
+	ResolvedDestination     string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
+	SourceIp                string `json:"sourceIp,omitempty" gorm:"column:sourceIp"`
+	DestinationIp           string `json:"destinationIp,omitempty" gorm:"column:destinationIp"`
+	SourcePort              string `json:"sourcePort,omitempty" gorm:"column:sourcePort"`
+	DestinationPort         string `json:"destinationPort,omitempty" gorm:"column:destinationPort"`
+	IsOutgoing              bool   `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
+	EstimatedSizeBytes      int    `json:"-" gorm:"column:estimatedSizeBytes"`
 }
 
 type MizuEntryWrapper struct {
-	Protocol       Protocol  `json:"protocol"`
-	Representation string    `json:"representation"`
-	BodySize       int64     `json:"bodySize"`
-	Data           MizuEntry `json:"data"`
+	Protocol       Protocol                 `json:"protocol"`
+	Representation string                   `json:"representation"`
+	BodySize       int64                    `json:"bodySize"`
+	Data           MizuEntry                `json:"data"`
+	Rules          []map[string]interface{} `json:"rulesMatched,omitempty"`
+	IsRulesEnabled bool                     `json:"isRulesEnabled"`
 }
 
 type BaseEntryDetails struct {
 	Id              string          `json:"id,omitempty"`
 	Protocol        Protocol        `json:"protocol,omitempty"`
 	Url             string          `json:"url,omitempty"`
-	RequestSenderIp string          `json:"request_sender_ip,omitempty"`
+	RequestSenderIp string          `json:"requestSenderIp,omitempty"`
 	Service         string          `json:"service,omitempty"`
+	Path            string          `json:"path,omitempty"`
 	Summary         string          `json:"summary,omitempty"`
-	StatusCode      int             `json:"status_code"`
+	StatusCode      int             `json:"statusCode"`
 	Method          string          `json:"method,omitempty"`
 	Timestamp       int64           `json:"timestamp,omitempty"`
-	SourceIp        string          `json:"source_ip,omitempty"`
-	DestinationIp   string          `json:"destination_ip,omitempty"`
-	SourcePort      string          `json:"source_port,omitempty"`
-	DestinationPort string          `json:"destination_port,omitempty"`
+	SourceIp        string          `json:"sourceIp,omitempty"`
+	DestinationIp   string          `json:"destinationIp,omitempty"`
+	SourcePort      string          `json:"sourcePort,omitempty"`
+	DestinationPort string          `json:"destinationPort,omitempty"`
 	IsOutgoing      bool            `json:"isOutgoing,omitempty"`
-	Latency         int64           `json:"latency,omitempty"`
+	Latency         int64           `json:"latency"`
 	Rules           ApplicableRules `json:"rules,omitempty"`
 }
 
@@ -156,17 +172,26 @@ type DataUnmarshaler interface {
 }
 
 func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
-	entryUrl := entry.Url
-	service := entry.Service
+	bed.Protocol = Protocol{
+		Name:            entry.ProtocolName,
+		LongName:        entry.ProtocolLongName,
+		Abbreviation:    entry.ProtocolAbbreviation,
+		Version:         entry.ProtocolVersion,
+		BackgroundColor: entry.ProtocolBackgroundColor,
+		ForegroundColor: entry.ProtocolForegroundColor,
+		FontSize:        entry.ProtocolFontSize,
+		ReferenceLink:   entry.ProtocolReferenceLink,
+	}
 	bed.Id = entry.EntryId
-	bed.Url = entryUrl
-	bed.Service = service
+	bed.Url = entry.Url
+	bed.Service = entry.Service
 	bed.Summary = entry.Path
 	bed.StatusCode = entry.Status
 	bed.Method = entry.Method
 	bed.Timestamp = entry.Timestamp
 	bed.RequestSenderIp = entry.RequestSenderIp
 	bed.IsOutgoing = entry.IsOutgoing
+	bed.Latency = entry.ElapsedTime
 	return nil
 }
 

tap/api/options.go

@@ -0,0 +1,7 @@
+package api
+
+type TrafficFilteringOptions struct {
+	HealthChecksUserAgentHeaders []string
+	PlainTextMaskingRegexes      []*SerializableRegexp
+	DisableRedaction             bool
+}

tap/api/serializable_regexp.go

@@ -1,4 +1,4 @@
-package shared
+package api
 
 import "regexp"
 

tap/api/stats_tracker.go

@@ -0,0 +1,70 @@
+package api
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type AppStats struct {
+	StartTime                   time.Time `json:"-"`
+	ProcessedBytes              uint64    `json:"processedBytes"`
+	PacketsCount                uint64    `json:"packetsCount"`
+	TcpPacketsCount             uint64    `json:"tcpPacketsCount"`
+	ReassembledTcpPayloadsCount uint64    `json:"reassembledTcpPayloadsCount"`
+	TlsConnectionsCount         uint64    `json:"tlsConnectionsCount"`
+	MatchedPairs                uint64    `json:"matchedPairs"`
+	DroppedTcpStreams           uint64    `json:"droppedTcpStreams"`
+}
+
+func (as *AppStats) IncMatchedPairs() {
+	atomic.AddUint64(&as.MatchedPairs, 1)
+}
+
+func (as *AppStats) IncDroppedTcpStreams() {
+	atomic.AddUint64(&as.DroppedTcpStreams, 1)
+}
+
+func (as *AppStats) IncPacketsCount() uint64 {
+	atomic.AddUint64(&as.PacketsCount, 1)
+	return as.PacketsCount
+}
+
+func (as *AppStats) IncTcpPacketsCount() {
+	atomic.AddUint64(&as.TcpPacketsCount, 1)
+}
+
+func (as *AppStats) IncReassembledTcpPayloadsCount() {
+	atomic.AddUint64(&as.ReassembledTcpPayloadsCount, 1)
+}
+
+func (as *AppStats) IncTlsConnectionsCount() {
+	atomic.AddUint64(&as.TlsConnectionsCount, 1)
+}
+
+func (as *AppStats) UpdateProcessedBytes(size uint64) {
+	atomic.AddUint64(&as.ProcessedBytes, size)
+}
+
+func (as *AppStats) SetStartTime(startTime time.Time) {
+	as.StartTime = startTime
+}
+
+func (as *AppStats) DumpStats() *AppStats {
+	currentAppStats := &AppStats{StartTime: as.StartTime}
+
+	currentAppStats.ProcessedBytes = resetUint64(&as.ProcessedBytes)
+	currentAppStats.PacketsCount = resetUint64(&as.PacketsCount)
+	currentAppStats.TcpPacketsCount = resetUint64(&as.TcpPacketsCount)
+	currentAppStats.ReassembledTcpPayloadsCount = resetUint64(&as.ReassembledTcpPayloadsCount)
+	currentAppStats.TlsConnectionsCount = resetUint64(&as.TlsConnectionsCount)
+	currentAppStats.MatchedPairs = resetUint64(&as.MatchedPairs)
+	currentAppStats.DroppedTcpStreams = resetUint64(&as.DroppedTcpStreams)
+
+	return currentAppStats
+}
+
+func resetUint64(ref *uint64) (val uint64) {
+	val = atomic.LoadUint64(ref)
+	atomic.StoreUint64(ref, 0)
+	return
+}

tap/extensions/amqp/main.go

@@ -32,7 +32,7 @@ func init() {
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
-	extension.Protocol = protocol
+	extension.Protocol = &protocol
 }
 
 func (d dissecting) Ping() {
@@ -41,7 +41,7 @@ func (d dissecting) Ping() {
 
 const amqpRequest string = "amqp_request"
 
-func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter) error {
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	r := AmqpReader{b}
 
 	var remaining int
@@ -78,13 +78,14 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	var lastMethodFrameMessage Message
 
 	for {
+		if superIdentifier.Protocol != nil && superIdentifier.Protocol != &protocol {
+			return errors.New("Identified by another protocol")
+		}
+
 		frame, err := r.ReadFrame()
 		if err == io.EOF {
 			// We must read until we see an EOF... very important!
 			return errors.New("AMQP EOF")
-		} else if err != nil {
-			// TODO: Causes ignoring some methods. Return only in case of a certain error. But what?
-			return err
 		}
 
 		switch f := frame.(type) {
@@ -101,6 +102,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			case *BasicDeliver:
 				eventBasicDeliver.Properties = header.Properties
 			default:
+				frame = nil
 			}
 
 		case *BodyFrame:
@@ -110,11 +112,15 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			switch lastMethodFrameMessage.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Body = f.Body
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventBasicPublish, amqpRequest, basicMethodMap[40], connectionInfo, superTimer.CaptureTime, emitter)
 			case *BasicDeliver:
 				eventBasicDeliver.Body = f.Body
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventBasicDeliver, amqpRequest, basicMethodMap[60], connectionInfo, superTimer.CaptureTime, emitter)
 			default:
+				body = nil
+				frame = nil
 			}
 
 		case *MethodFrame:
@@ -134,6 +140,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventQueueBind, amqpRequest, queueMethodMap[20], connectionInfo, superTimer.CaptureTime, emitter)
 
 			case *BasicConsume:
@@ -146,6 +153,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 					NoWait:      m.NoWait,
 					Arguments:   m.Arguments,
 				}
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventBasicConsume, amqpRequest, basicMethodMap[20], connectionInfo, superTimer.CaptureTime, emitter)
 
 			case *BasicDeliver:
@@ -165,6 +173,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventQueueDeclare, amqpRequest, queueMethodMap[10], connectionInfo, superTimer.CaptureTime, emitter)
 
 			case *ExchangeDeclare:
@@ -178,6 +187,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventExchangeDeclare, amqpRequest, exchangeMethodMap[10], connectionInfo, superTimer.CaptureTime, emitter)
 
 			case *ConnectionStart:
@@ -188,6 +198,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 					Mechanisms:       m.Mechanisms,
 					Locales:          m.Locales,
 				}
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventConnectionStart, amqpRequest, connectionMethodMap[10], connectionInfo, superTimer.CaptureTime, emitter)
 
 			case *ConnectionClose:
@@ -197,9 +208,11 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 					ClassId:   m.ClassId,
 					MethodId:  m.MethodId,
 				}
+				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventConnectionClose, amqpRequest, connectionMethodMap[50], connectionInfo, superTimer.CaptureTime, emitter)
 
 			default:
+				frame = nil
 
 			}
 
@@ -254,25 +267,31 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	request["url"] = summary
 	entryBytes, _ := json.Marshal(item.Pair)
 	return &api.MizuEntry{
-		ProtocolName:        protocol.Name,
-		ProtocolVersion:     protocol.Version,
-		EntryId:             entryId,
-		Entry:               string(entryBytes),
-		Url:                 fmt.Sprintf("%s%s", service, summary),
-		Method:              request["method"].(string),
-		Status:              0,
-		RequestSenderIp:     item.ConnectionInfo.ClientIP,
-		Service:             service,
-		Timestamp:           item.Timestamp,
-		ElapsedTime:         0,
-		Path:                summary,
-		ResolvedSource:      resolvedSource,
-		ResolvedDestination: resolvedDestination,
-		SourceIp:            item.ConnectionInfo.ClientIP,
-		DestinationIp:       item.ConnectionInfo.ServerIP,
-		SourcePort:          item.ConnectionInfo.ClientPort,
-		DestinationPort:     item.ConnectionInfo.ServerPort,
-		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
+		ProtocolName:            protocol.Name,
+		ProtocolLongName:        protocol.LongName,
+		ProtocolAbbreviation:    protocol.Abbreviation,
+		ProtocolVersion:         protocol.Version,
+		ProtocolBackgroundColor: protocol.BackgroundColor,
+		ProtocolForegroundColor: protocol.ForegroundColor,
+		ProtocolFontSize:        protocol.FontSize,
+		ProtocolReferenceLink:   protocol.ReferenceLink,
+		EntryId:                 entryId,
+		Entry:                   string(entryBytes),
+		Url:                     fmt.Sprintf("%s%s", service, summary),
+		Method:                  request["method"].(string),
+		Status:                  0,
+		RequestSenderIp:         item.ConnectionInfo.ClientIP,
+		Service:                 service,
+		Timestamp:               item.Timestamp,
+		ElapsedTime:             0,
+		Path:                    summary,
+		ResolvedSource:          resolvedSource,
+		ResolvedDestination:     resolvedDestination,
+		SourceIp:                item.ConnectionInfo.ClientIP,
+		DestinationIp:           item.ConnectionInfo.ServerIP,
+		SourcePort:              item.ConnectionInfo.ClientPort,
+		DestinationPort:         item.ConnectionInfo.ServerPort,
+		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
 	}
 
 }
@@ -293,7 +312,7 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 		SourcePort:      entry.SourcePort,
 		DestinationPort: entry.DestinationPort,
 		IsOutgoing:      entry.IsOutgoing,
-		Latency:         0,
+		Latency:         entry.ElapsedTime,
 		Rules: api.ApplicableRules{
 			Latency: 0,
 			Status:  false,

tap/extensions/amqp/read.go

@@ -54,7 +54,7 @@ func (r *AmqpReader) ReadFrame() (frame frame, err error) {
 	channel := binary.BigEndian.Uint16(scratch[1:3])
 	size := binary.BigEndian.Uint32(scratch[3:7])
 
-	if size > 1000000 {
+	if size > 1000000*16 {
 		return nil, ErrMaxSize
 	}
 
@@ -352,6 +352,10 @@ func (r *AmqpReader) parseHeaderFrame(channel uint16, size uint32) (frame frame,
 		return
 	}
 
+	if hf.Size > 512 {
+		return nil, ErrMaxHeaderFrameSize
+	}
+
 	var flags uint16
 
 	if err = binary.Read(r.R, binary.BigEndian, &flags); err != nil {
@@ -439,6 +443,7 @@ func (r *AmqpReader) parseBodyFrame(channel uint16, size uint32) (frame frame, e
 	}
 
 	if _, err = io.ReadFull(r.R, bf.Body); err != nil {
+		bf.Body = nil
 		return nil, err
 	}
 

tap/extensions/amqp/spec091.go

@@ -10,7 +10,6 @@ package main
 
 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 )
 
@@ -19,32 +18,35 @@ import (
 // ErrCredentials.  The text of the error is likely more interesting than
 // these constants.
 const (
-	frameMethod        = 1
-	frameHeader        = 2
-	frameBody          = 3
-	frameHeartbeat     = 8
-	frameMinSize       = 4096
-	frameEnd           = 206
-	replySuccess       = 200
-	ContentTooLarge    = 311
-	NoRoute            = 312
-	NoConsumers        = 313
-	ConnectionForced   = 320
-	InvalidPath        = 402
-	AccessRefused      = 403
-	NotFound           = 404
-	ResourceLocked     = 405
-	PreconditionFailed = 406
-	FrameError         = 501
-	SyntaxError        = 502
-	CommandInvalid     = 503
-	ChannelError       = 504
-	UnexpectedFrame    = 505
-	ResourceError      = 506
-	NotAllowed         = 530
-	NotImplemented     = 540
-	InternalError      = 541
-	MaxSizeError       = 551
+	frameMethod                 = 1
+	frameHeader                 = 2
+	frameBody                   = 3
+	frameHeartbeat              = 8
+	frameMinSize                = 4096
+	frameEnd                    = 206
+	replySuccess                = 200
+	ContentTooLarge             = 311
+	NoRoute                     = 312
+	NoConsumers                 = 313
+	ConnectionForced            = 320
+	InvalidPath                 = 402
+	AccessRefused               = 403
+	NotFound                    = 404
+	ResourceLocked              = 405
+	PreconditionFailed          = 406
+	FrameError                  = 501
+	SyntaxError                 = 502
+	CommandInvalid              = 503
+	ChannelError                = 504
+	UnexpectedFrame             = 505
+	ResourceError               = 506
+	NotAllowed                  = 530
+	NotImplemented              = 540
+	InternalError               = 541
+	MaxSizeError                = 551
+	MaxHeaderFrameSizeError     = 552
+	BadMethodFrameUnknownMethod = 601
+	BadMethodFrameUnknownClass  = 602
 )
 
 func isSoftExceptionCode(code int) bool {
@@ -2854,7 +2856,7 @@ func (r *AmqpReader) parseMethodFrame(channel uint16, size uint32) (f frame, err
 			mf.Method = method
 
 		default:
-			return nil, fmt.Errorf("Bad method frame, unknown method %d for class %d", mf.MethodId, mf.ClassId)
+			return nil, ErrBadMethodFrameUnknownMethod
 		}
 
 	case 20: // channel
@@ -2909,7 +2911,7 @@ func (r *AmqpReader) parseMethodFrame(channel uint16, size uint32) (f frame, err
 			mf.Method = method
 
 		default:
-			return nil, fmt.Errorf("Bad method frame, unknown method %d for class %d", mf.MethodId, mf.ClassId)
+			return nil, ErrBadMethodFrameUnknownMethod
 		}
 
 	case 40: // exchange
@@ -2980,7 +2982,7 @@ func (r *AmqpReader) parseMethodFrame(channel uint16, size uint32) (f frame, err
 			mf.Method = method
 
 		default:
-			return nil, fmt.Errorf("Bad method frame, unknown method %d for class %d", mf.MethodId, mf.ClassId)
+			return nil, ErrBadMethodFrameUnknownMethod
 		}
 
 	case 50: // queue
@@ -3067,7 +3069,7 @@ func (r *AmqpReader) parseMethodFrame(channel uint16, size uint32) (f frame, err
 			mf.Method = method
 
 		default:
-			return nil, fmt.Errorf("Bad method frame, unknown method %d for class %d", mf.MethodId, mf.ClassId)
+			return nil, ErrBadMethodFrameUnknownMethod
 		}
 
 	case 60: // basic
@@ -3218,7 +3220,7 @@ func (r *AmqpReader) parseMethodFrame(channel uint16, size uint32) (f frame, err
 			mf.Method = method
 
 		default:
-			return nil, fmt.Errorf("Bad method frame, unknown method %d for class %d", mf.MethodId, mf.ClassId)
+			return nil, ErrBadMethodFrameUnknownMethod
 		}
 
 	case 90: // tx
@@ -3273,7 +3275,7 @@ func (r *AmqpReader) parseMethodFrame(channel uint16, size uint32) (f frame, err
 			mf.Method = method
 
 		default:
-			return nil, fmt.Errorf("Bad method frame, unknown method %d for class %d", mf.MethodId, mf.ClassId)
+			return nil, ErrBadMethodFrameUnknownMethod
 		}
 
 	case 85: // confirm
@@ -3296,11 +3298,11 @@ func (r *AmqpReader) parseMethodFrame(channel uint16, size uint32) (f frame, err
 			mf.Method = method
 
 		default:
-			return nil, fmt.Errorf("Bad method frame, unknown method %d for class %d", mf.MethodId, mf.ClassId)
+			return nil, ErrBadMethodFrameUnknownMethod
 		}
 
 	default:
-		return nil, fmt.Errorf("Bad method frame, unknown class %d", mf.ClassId)
+		return nil, ErrBadMethodFrameUnknownClass
 	}
 
 	return mf, nil

tap/extensions/amqp/types.go

@@ -60,8 +60,13 @@ var (
 	// ErrFieldType is returned when writing a message containing a Go type unsupported by AMQP.
 	ErrFieldType = &Error{Code: SyntaxError, Reason: "unsupported table field type"}
 
-	// ErrClosed is returned when the channel or connection is not open
-	ErrMaxSize = &Error{Code: MaxSizeError, Reason: "an AMQP message cannot be bigger than 1MB"}
+	ErrMaxSize = &Error{Code: MaxSizeError, Reason: "an AMQP message cannot be bigger than 16MB"}
+
+	ErrMaxHeaderFrameSize = &Error{Code: MaxHeaderFrameSizeError, Reason: "an AMQP header cannot be bigger than 512 bytes"}
+
+	ErrBadMethodFrameUnknownMethod = &Error{Code: BadMethodFrameUnknownMethod, Reason: "Bad method frame, unknown method"}
+
+	ErrBadMethodFrameUnknownClass = &Error{Code: BadMethodFrameUnknownClass, Reason: "Bad method frame, unknown class"}
 )
 
 // Error captures the code and reason a channel or connection has been closed

tap/extensions/http/go.mod

@@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tap/extensions/http
 go 1.16
 
 require (
+	github.com/beevik/etree v1.1.0 // indirect
 	github.com/google/martian v2.1.0+incompatible
 	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	github.com/up9inc/mizu/tap/api v0.0.0

tap/extensions/http/go.sum

@@ -1,3 +1,5 @@
+github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
+github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=

tap/extensions/http/handlers.go

@@ -13,7 +13,14 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter) error {
+func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *api.TrafficFilteringOptions) {
+	if !options.DisableRedaction {
+		FilterSensitiveData(item, options)
+	}
+	emitter.Emit(item)
+}
+
+func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	streamID, messageHTTP1, err := grpcAssembler.readMessage()
 	if err != nil {
 		return err
@@ -64,13 +71,13 @@ func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTime
 
 	if item != nil {
 		item.Protocol = http2Protocol
-		emitter.Emit(item)
+		filterAndEmit(item, emitter, options)
 	}
 
 	return nil
 }
 
-func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter) error {
+func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	req, err := http.ReadRequest(b)
 	if err != nil {
 		// log.Println("Error reading stream:", err)
@@ -107,12 +114,12 @@ func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 			ServerPort: tcpID.DstPort,
 			IsOutgoing: true,
 		}
-		emitter.Emit(item)
+		filterAndEmit(item, emitter, options)
 	}
 	return nil
 }
 
-func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter) error {
+func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	res, err := http.ReadResponse(b, nil)
 	if err != nil {
 		// log.Println("Error reading stream:", err)
@@ -157,7 +164,7 @@ func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 			ServerPort: tcpID.SrcPort,
 			IsOutgoing: false,
 		}
-		emitter.Emit(item)
+		filterAndEmit(item, emitter, options)
 	}
 	return nil
 }

tap/extensions/http/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -52,7 +53,7 @@ func init() {
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
-	extension.Protocol = protocol
+	extension.Protocol = &protocol
 	extension.MatcherMap = reqResMatcher.openMessagesMap
 }
 
@@ -60,7 +61,7 @@ func (d dissecting) Ping() {
 	log.Printf("pong %s\n", protocol.Name)
 }
 
-func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter) error {
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	ident := fmt.Sprintf("%s->%s:%s->%s", tcpID.SrcIP, tcpID.DstIP, tcpID.SrcPort, tcpID.DstPort)
 	isHTTP2, err := checkIsHTTP2Connection(b, isClient)
 	if err != nil {
@@ -77,41 +78,46 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 		grpcAssembler = createGrpcAssembler(b)
 	}
 
-	success := false
+	dissected := false
 	for {
+		if superIdentifier.Protocol != nil && superIdentifier.Protocol != &protocol {
+			return errors.New("Identified by another protocol")
+		}
+
 		if isHTTP2 {
-			err = handleHTTP2Stream(grpcAssembler, tcpID, superTimer, emitter)
+			err = handleHTTP2Stream(grpcAssembler, tcpID, superTimer, emitter, options)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
 				rlog.Debugf("[HTTP/2] stream %s error: %s (%v,%+v)", ident, err, err, err)
 				continue
 			}
-			success = true
+			dissected = true
 		} else if isClient {
-			err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter)
+			err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
 				rlog.Debugf("[HTTP-request] stream %s Request error: %s (%v,%+v)", ident, err, err, err)
 				continue
 			}
-			success = true
+			dissected = true
 		} else {
-			err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter)
+			err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
 				rlog.Debugf("[HTTP-response], stream %s Response error: %s (%v,%+v)", ident, err, err, err)
 				continue
 			}
-			success = true
+			dissected = true
 		}
 	}
 
-	if !success {
+	if !dissected {
 		return err
 	}
+	superIdentifier.Protocol = &protocol
 	return nil
 }
 
@@ -166,25 +172,31 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
 	entryBytes, _ := json.Marshal(item.Pair)
 	return &api.MizuEntry{
-		ProtocolName:        protocol.Name,
-		ProtocolVersion:     item.Protocol.Version,
-		EntryId:             entryId,
-		Entry:               string(entryBytes),
-		Url:                 fmt.Sprintf("%s%s", service, path),
-		Method:              reqDetails["method"].(string),
-		Status:              int(resDetails["status"].(float64)),
-		RequestSenderIp:     item.ConnectionInfo.ClientIP,
-		Service:             service,
-		Timestamp:           item.Timestamp,
-		ElapsedTime:         elapsedTime,
-		Path:                path,
-		ResolvedSource:      resolvedSource,
-		ResolvedDestination: resolvedDestination,
-		SourceIp:            item.ConnectionInfo.ClientIP,
-		DestinationIp:       item.ConnectionInfo.ServerIP,
-		SourcePort:          item.ConnectionInfo.ClientPort,
-		DestinationPort:     item.ConnectionInfo.ServerPort,
-		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
+		ProtocolName:            protocol.Name,
+		ProtocolLongName:        protocol.LongName,
+		ProtocolAbbreviation:    protocol.Abbreviation,
+		ProtocolVersion:         item.Protocol.Version,
+		ProtocolBackgroundColor: protocol.BackgroundColor,
+		ProtocolForegroundColor: protocol.ForegroundColor,
+		ProtocolFontSize:        protocol.FontSize,
+		ProtocolReferenceLink:   protocol.ReferenceLink,
+		EntryId:                 entryId,
+		Entry:                   string(entryBytes),
+		Url:                     fmt.Sprintf("%s%s", service, path),
+		Method:                  reqDetails["method"].(string),
+		Status:                  int(resDetails["status"].(float64)),
+		RequestSenderIp:         item.ConnectionInfo.ClientIP,
+		Service:                 service,
+		Timestamp:               item.Timestamp,
+		ElapsedTime:             elapsedTime,
+		Path:                    path,
+		ResolvedSource:          resolvedSource,
+		ResolvedDestination:     resolvedDestination,
+		SourceIp:                item.ConnectionInfo.ClientIP,
+		DestinationIp:           item.ConnectionInfo.ServerIP,
+		SourcePort:              item.ConnectionInfo.ClientPort,
+		DestinationPort:         item.ConnectionInfo.ServerPort,
+		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
 	}
 }
 
@@ -201,6 +213,7 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 		Url:             entry.Url,
 		RequestSenderIp: entry.RequestSenderIp,
 		Service:         entry.Service,
+		Path:            entry.Path,
 		Summary:         entry.Path,
 		StatusCode:      entry.Status,
 		Method:          entry.Method,
@@ -210,7 +223,7 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 		SourcePort:      entry.SourcePort,
 		DestinationPort: entry.DestinationPort,
 		IsOutgoing:      entry.IsOutgoing,
-		Latency:         0,
+		Latency:         entry.ElapsedTime,
 		Rules: api.ApplicableRules{
 			Latency: 0,
 			Status:  false,

tap/extensions/http/sensitive_data_cleaner.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/beevik/etree"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+const maskedFieldPlaceholderValue = "[REDACTED]"
+
+//these values MUST be all lower case and contain no `-` or `_` characters
+var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
+	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
+	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
+	"zip", "zipcode", "address", "country", "firstname", "lastname",
+	"middlename", "fname", "lname", "birthdate"}
+
+func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
+	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
+	response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)
+
+	filterHeaders(&request.Header)
+	filterHeaders(&response.Header)
+	filterUrl(request.URL)
+	filterRequestBody(request, options)
+	filterResponseBody(response, options)
+}
+
+func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) {
+	contenType := getContentTypeHeaderValue(request.Header)
+	body, err := ioutil.ReadAll(request.Body)
+	if err != nil {
+		rlog.Debugf("Filtering error reading body: %v", err)
+		return
+	}
+	filteredBody, err := filterHttpBody([]byte(body), contenType, options)
+	if err == nil {
+		request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
+	} else {
+		request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
+	}
+}
+
+func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) {
+	contentType := getContentTypeHeaderValue(response.Header)
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		rlog.Debugf("Filtering error reading body: %v", err)
+		return
+	}
+	filteredBody, err := filterHttpBody([]byte(body), contentType, options)
+	if err == nil {
+		response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
+	} else {
+		response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
+	}
+}
+
+func filterHeaders(headers *http.Header) {
+	for key, _ := range *headers {
+		if strings.ToLower(key) == "cookie" {
+			headers.Del(key)
+		} else if isFieldNameSensitive(key) {
+			headers.Set(key, maskedFieldPlaceholderValue)
+		}
+	}
+}
+
+func getContentTypeHeaderValue(headers http.Header) string {
+	for key, _ := range headers {
+		if strings.ToLower(key) == "content-type" {
+			return headers.Get(key)
+		}
+	}
+	return ""
+}
+
+func isFieldNameSensitive(fieldName string) bool {
+	if fieldName == ":authority" {
+		return false
+	}
+
+	name := strings.ToLower(fieldName)
+	name = strings.ReplaceAll(name, "_", "")
+	name = strings.ReplaceAll(name, "-", "")
+	name = strings.ReplaceAll(name, " ", "")
+
+	for _, sensitiveField := range personallyIdentifiableDataFields {
+		if strings.Contains(name, sensitiveField) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) {
+	mimeType := strings.Split(contentType, ";")[0]
+	switch strings.ToLower(mimeType) {
+	case "application/json":
+		return filterJsonBody(bytes)
+	case "text/html":
+		fallthrough
+	case "application/xhtml+xml":
+		fallthrough
+	case "text/xml":
+		fallthrough
+	case "application/xml":
+		return filterXmlEtree(bytes)
+	case "text/plain":
+		if options != nil && options.PlainTextMaskingRegexes != nil {
+			return filterPlainText(bytes, options), nil
+		}
+	}
+	return bytes, nil
+}
+
+func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte {
+	for _, regex := range options.PlainTextMaskingRegexes {
+		bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
+	}
+	return bytes
+}
+
+func filterXmlEtree(bytes []byte) ([]byte, error) {
+	if !IsValidXML(bytes) {
+		return nil, errors.New("Invalid XML")
+	}
+	xmlDoc := etree.NewDocument()
+	err := xmlDoc.ReadFromBytes(bytes)
+	if err != nil {
+		return nil, err
+	} else {
+		filterXmlElement(xmlDoc.Root())
+	}
+	return xmlDoc.WriteToBytes()
+}
+
+func IsValidXML(data []byte) bool {
+	return xml.Unmarshal(data, new(interface{})) == nil
+}
+
+func filterXmlElement(element *etree.Element) {
+	for i, attribute := range element.Attr {
+		if isFieldNameSensitive(attribute.Key) {
+			element.Attr[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+	if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
+		if isFieldNameSensitive(element.Tag) {
+			element.SetText(maskedFieldPlaceholderValue)
+		}
+	} else {
+		for _, element := range element.ChildElements() {
+			filterXmlElement(element)
+		}
+	}
+}
+
+func filterJsonBody(bytes []byte) ([]byte, error) {
+	var bodyJsonMap map[string]interface{}
+	err := json.Unmarshal(bytes, &bodyJsonMap)
+	if err != nil {
+		return nil, err
+	}
+	filterJsonMap(bodyJsonMap)
+	return json.Marshal(bodyJsonMap)
+}
+
+func filterJsonMap(jsonMap map[string]interface{}) {
+	for key, value := range jsonMap {
+		// Do not replace nil values with maskedFieldPlaceholderValue
+		if value == nil {
+			continue
+		}
+
+		nestedMap, isNested := value.(map[string]interface{})
+		if isNested {
+			filterJsonMap(nestedMap)
+		} else {
+			if isFieldNameSensitive(key) {
+				jsonMap[key] = maskedFieldPlaceholderValue
+			}
+		}
+	}
+}
+
+func filterUrl(url *url.URL) {
+	if len(url.RawQuery) > 0 {
+		newQueryArgs := make([]string, 0)
+		for urlQueryParamName, urlQueryParamValues := range url.Query() {
+			newValues := urlQueryParamValues
+			if isFieldNameSensitive(urlQueryParamName) {
+				newValues = []string{maskedFieldPlaceholderValue}
+			}
+			for _, paramValue := range newValues {
+				newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
+			}
+		}
+
+		url.RawQuery = strings.Join(newQueryArgs, "&")
+	}
+}

tap/extensions/kafka/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"time"
@@ -30,7 +31,7 @@ func init() {
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
-	extension.Protocol = _protocol
+	extension.Protocol = &_protocol
 	extension.MatcherMap = reqResMatcher.openMessagesMap
 }
 
@@ -38,18 +39,24 @@ func (d dissecting) Ping() {
 	log.Printf("pong %s\n", _protocol.Name)
 }
 
-func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter) error {
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	for {
+		if superIdentifier.Protocol != nil && superIdentifier.Protocol != &_protocol {
+			return errors.New("Identified by another protocol")
+		}
+
 		if isClient {
 			_, _, err := ReadRequest(b, tcpID, superTimer)
 			if err != nil {
 				return err
 			}
+			superIdentifier.Protocol = &_protocol
 		} else {
 			err := ReadResponse(b, tcpID, superTimer, emitter)
 			if err != nil {
 				return err
 			}
+			superIdentifier.Protocol = &_protocol
 		}
 	}
 }
@@ -135,25 +142,31 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
 	entryBytes, _ := json.Marshal(item.Pair)
 	return &api.MizuEntry{
-		ProtocolName:        _protocol.Name,
-		ProtocolVersion:     _protocol.Version,
-		EntryId:             entryId,
-		Entry:               string(entryBytes),
-		Url:                 fmt.Sprintf("%s%s", service, summary),
-		Method:              apiNames[apiKey],
-		Status:              0,
-		RequestSenderIp:     item.ConnectionInfo.ClientIP,
-		Service:             service,
-		Timestamp:           item.Timestamp,
-		ElapsedTime:         elapsedTime,
-		Path:                summary,
-		ResolvedSource:      resolvedSource,
-		ResolvedDestination: resolvedDestination,
-		SourceIp:            item.ConnectionInfo.ClientIP,
-		DestinationIp:       item.ConnectionInfo.ServerIP,
-		SourcePort:          item.ConnectionInfo.ClientPort,
-		DestinationPort:     item.ConnectionInfo.ServerPort,
-		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
+		ProtocolName:            _protocol.Name,
+		ProtocolLongName:        _protocol.LongName,
+		ProtocolAbbreviation:    _protocol.Abbreviation,
+		ProtocolVersion:         _protocol.Version,
+		ProtocolBackgroundColor: _protocol.BackgroundColor,
+		ProtocolForegroundColor: _protocol.ForegroundColor,
+		ProtocolFontSize:        _protocol.FontSize,
+		ProtocolReferenceLink:   _protocol.ReferenceLink,
+		EntryId:                 entryId,
+		Entry:                   string(entryBytes),
+		Url:                     fmt.Sprintf("%s%s", service, summary),
+		Method:                  apiNames[apiKey],
+		Status:                  0,
+		RequestSenderIp:         item.ConnectionInfo.ClientIP,
+		Service:                 service,
+		Timestamp:               item.Timestamp,
+		ElapsedTime:             elapsedTime,
+		Path:                    summary,
+		ResolvedSource:          resolvedSource,
+		ResolvedDestination:     resolvedDestination,
+		SourceIp:                item.ConnectionInfo.ClientIP,
+		DestinationIp:           item.ConnectionInfo.ServerIP,
+		SourcePort:              item.ConnectionInfo.ClientPort,
+		DestinationPort:         item.ConnectionInfo.ServerPort,
+		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
 	}
 }
 
@@ -173,7 +186,7 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 		SourcePort:      entry.SourcePort,
 		DestinationPort: entry.DestinationPort,
 		IsOutgoing:      entry.IsOutgoing,
-		Latency:         0,
+		Latency:         entry.ElapsedTime,
 		Rules: api.ApplicableRules{
 			Latency: 0,
 			Status:  false,

tap/extensions/redis/errors.go

@@ -0,0 +1,14 @@
+package main
+
+//ConnectError redis connection error,such as io timeout
+type ConnectError struct {
+	Message string
+}
+
+func newConnectError(message string) *ConnectError {
+	return &ConnectError{Message: message}
+}
+
+func (e *ConnectError) Error() string {
+	return e.Message
+}

tap/extensions/redis/go.mod

@@ -0,0 +1,9 @@
+module github.com/up9inc/mizu/tap/extensions/redis
+
+go 1.16
+
+require (
+    github.com/up9inc/mizu/tap/api v0.0.0
+)
+
+replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/extensions/redis/handlers.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+func handleClientStream(tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, request *RedisPacket) error {
+	counterPair.Request++
+	ident := fmt.Sprintf(
+		"%s->%s %s->%s %d",
+		tcpID.SrcIP,
+		tcpID.DstIP,
+		tcpID.SrcPort,
+		tcpID.DstPort,
+		counterPair.Request,
+	)
+	item := reqResMatcher.registerRequest(ident, request, superTimer.CaptureTime)
+	if item != nil {
+		item.ConnectionInfo = &api.ConnectionInfo{
+			ClientIP:   tcpID.SrcIP,
+			ClientPort: tcpID.SrcPort,
+			ServerIP:   tcpID.DstIP,
+			ServerPort: tcpID.DstPort,
+			IsOutgoing: true,
+		}
+		emitter.Emit(item)
+	}
+	return nil
+}
+
+func handleServerStream(tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, response *RedisPacket) error {
+	counterPair.Response++
+	ident := fmt.Sprintf(
+		"%s->%s %s->%s %d",
+		tcpID.DstIP,
+		tcpID.SrcIP,
+		tcpID.DstPort,
+		tcpID.SrcPort,
+		counterPair.Response,
+	)
+	item := reqResMatcher.registerResponse(ident, response, superTimer.CaptureTime)
+	if item != nil {
+		item.ConnectionInfo = &api.ConnectionInfo{
+			ClientIP:   tcpID.DstIP,
+			ClientPort: tcpID.DstPort,
+			ServerIP:   tcpID.SrcIP,
+			ServerPort: tcpID.SrcPort,
+			IsOutgoing: false,
+		}
+		emitter.Emit(item)
+	}
+	return nil
+}

tap/extensions/redis/helpers.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"encoding/json"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+type RedisPayload struct {
+	Data interface{}
+}
+
+type RedisPayloader interface {
+	MarshalJSON() ([]byte, error)
+}
+
+func (h RedisPayload) MarshalJSON() ([]byte, error) {
+	return json.Marshal(h.Data)
+}
+
+type RedisWrapper struct {
+	Method  string      `json:"method"`
+	Url     string      `json:"url"`
+	Details interface{} `json:"details"`
+}
+
+func representGeneric(generic map[string]interface{}) (representation []interface{}) {
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Type",
+			"value": generic["type"].(string),
+		},
+		{
+			"name":  "Command",
+			"value": generic["command"].(string),
+		},
+		{
+			"name":  "Key",
+			"value": generic["key"].(string),
+		},
+		{
+			"name":  "Value",
+			"value": generic["value"].(string),
+		},
+		{
+			"name":  "Keyword",
+			"value": generic["keyword"].(string),
+		},
+	})
+	representation = append(representation, map[string]string{
+		"type":  api.TABLE,
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	return
+}

tap/extensions/redis/main.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+var protocol api.Protocol = api.Protocol{
+	Name:            "redis",
+	LongName:        "Redis Serialization Protocol",
+	Abbreviation:    "REDIS",
+	Version:         "3.x",
+	BackgroundColor: "#a41e11",
+	ForegroundColor: "#ffffff",
+	FontSize:        11,
+	ReferenceLink:   "https://redis.io/topics/protocol",
+	Ports:           []string{"6379"},
+	Priority:        3,
+}
+
+func init() {
+	log.Println("Initializing Redis extension...")
+}
+
+type dissecting string
+
+func (d dissecting) Register(extension *api.Extension) {
+	extension.Protocol = &protocol
+	extension.MatcherMap = reqResMatcher.openMessagesMap
+}
+
+func (d dissecting) Ping() {
+	log.Printf("pong %s\n", protocol.Name)
+}
+
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
+	is := &RedisInputStream{
+		Reader: b,
+		Buf:    make([]byte, 8192),
+	}
+	proto := NewProtocol(is)
+	for {
+		redisPacket, err := proto.Read()
+		if err != nil {
+			return err
+		}
+
+		if isClient {
+			handleClientStream(tcpID, counterPair, superTimer, emitter, redisPacket)
+		} else {
+			handleServerStream(tcpID, counterPair, superTimer, emitter, redisPacket)
+		}
+	}
+}
+
+func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+	request := item.Pair.Request.Payload.(map[string]interface{})
+	reqDetails := request["details"].(map[string]interface{})
+	service := "redis"
+	if resolvedDestination != "" {
+		service = resolvedDestination
+	} else if resolvedSource != "" {
+		service = resolvedSource
+	}
+
+	method := ""
+	if reqDetails["command"] != nil {
+		method = reqDetails["command"].(string)
+	}
+
+	summary := ""
+	if reqDetails["key"] != nil {
+		summary = reqDetails["key"].(string)
+	}
+
+	request["url"] = summary
+	entryBytes, _ := json.Marshal(item.Pair)
+	return &api.MizuEntry{
+		ProtocolName:            protocol.Name,
+		ProtocolLongName:        protocol.LongName,
+		ProtocolAbbreviation:    protocol.Abbreviation,
+		ProtocolVersion:         protocol.Version,
+		ProtocolBackgroundColor: protocol.BackgroundColor,
+		ProtocolForegroundColor: protocol.ForegroundColor,
+		ProtocolFontSize:        protocol.FontSize,
+		ProtocolReferenceLink:   protocol.ReferenceLink,
+		EntryId:                 entryId,
+		Entry:                   string(entryBytes),
+		Url:                     fmt.Sprintf("%s%s", service, summary),
+		Method:                  method,
+		Status:                  0,
+		RequestSenderIp:         item.ConnectionInfo.ClientIP,
+		Service:                 service,
+		Timestamp:               item.Timestamp,
+		ElapsedTime:             0,
+		Path:                    summary,
+		ResolvedSource:          resolvedSource,
+		ResolvedDestination:     resolvedDestination,
+		SourceIp:                item.ConnectionInfo.ClientIP,
+		DestinationIp:           item.ConnectionInfo.ServerIP,
+		SourcePort:              item.ConnectionInfo.ClientPort,
+		DestinationPort:         item.ConnectionInfo.ServerPort,
+		IsOutgoing:              item.ConnectionInfo.IsOutgoing,
+	}
+
+}
+
+func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
+	return &api.BaseEntryDetails{
+		Id:              entry.EntryId,
+		Protocol:        protocol,
+		Url:             entry.Url,
+		RequestSenderIp: entry.RequestSenderIp,
+		Service:         entry.Service,
+		Summary:         entry.Path,
+		StatusCode:      entry.Status,
+		Method:          entry.Method,
+		Timestamp:       entry.Timestamp,
+		SourceIp:        entry.SourceIp,
+		DestinationIp:   entry.DestinationIp,
+		SourcePort:      entry.SourcePort,
+		DestinationPort: entry.DestinationPort,
+		IsOutgoing:      entry.IsOutgoing,
+		Latency:         entry.ElapsedTime,
+		Rules: api.ApplicableRules{
+			Latency: 0,
+			Status:  false,
+		},
+	}
+}
+
+func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
+	p = protocol
+	bodySize = 0
+	var root map[string]interface{}
+	json.Unmarshal([]byte(entry.Entry), &root)
+	representation := make(map[string]interface{}, 0)
+	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
+	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
+	reqDetails := request["details"].(map[string]interface{})
+	resDetails := response["details"].(map[string]interface{})
+	repRequest := representGeneric(reqDetails)
+	repResponse := representGeneric(resDetails)
+	representation["request"] = repRequest
+	representation["response"] = repResponse
+	object, err = json.Marshal(representation)
+	return
+}
+
+var Dissector dissecting

tap/extensions/redis/matcher.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+var reqResMatcher = createResponseRequestMatcher() // global
+
+// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}_{incremental_counter}
+type requestResponseMatcher struct {
+	openMessagesMap *sync.Map
+}
+
+func createResponseRequestMatcher() requestResponseMatcher {
+	newMatcher := &requestResponseMatcher{openMessagesMap: &sync.Map{}}
+	return *newMatcher
+}
+
+func (matcher *requestResponseMatcher) registerRequest(ident string, request *RedisPacket, captureTime time.Time) *api.OutputChannelItem {
+	split := splitIdent(ident)
+	key := genKey(split)
+
+	requestRedisMessage := api.GenericMessage{
+		IsRequest:   true,
+		CaptureTime: captureTime,
+		Payload: RedisPayload{
+			Data: &RedisWrapper{
+				Method:  string(request.Command),
+				Url:     "",
+				Details: request,
+			},
+		},
+	}
+
+	if response, found := matcher.openMessagesMap.LoadAndDelete(key); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		responseRedisMessage := response.(*api.GenericMessage)
+		if responseRedisMessage.IsRequest {
+			return nil
+		}
+		return matcher.preparePair(&requestRedisMessage, responseRedisMessage)
+	}
+
+	matcher.openMessagesMap.Store(key, &requestRedisMessage)
+	return nil
+}
+
+func (matcher *requestResponseMatcher) registerResponse(ident string, response *RedisPacket, captureTime time.Time) *api.OutputChannelItem {
+	split := splitIdent(ident)
+	key := genKey(split)
+
+	responseRedisMessage := api.GenericMessage{
+		IsRequest:   false,
+		CaptureTime: captureTime,
+		Payload: RedisPayload{
+			Data: &RedisWrapper{
+				Method:  string(response.Command),
+				Url:     "",
+				Details: response,
+			},
+		},
+	}
+
+	if request, found := matcher.openMessagesMap.LoadAndDelete(key); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		requestRedisMessage := request.(*api.GenericMessage)
+		if !requestRedisMessage.IsRequest {
+			return nil
+		}
+		return matcher.preparePair(requestRedisMessage, &responseRedisMessage)
+	}
+
+	matcher.openMessagesMap.Store(key, &responseRedisMessage)
+	return nil
+}
+
+func (matcher *requestResponseMatcher) preparePair(requestRedisMessage *api.GenericMessage, responseRedisMessage *api.GenericMessage) *api.OutputChannelItem {
+	return &api.OutputChannelItem{
+		Protocol:       protocol,
+		Timestamp:      requestRedisMessage.CaptureTime.UnixNano() / int64(time.Millisecond),
+		ConnectionInfo: nil,
+		Pair: &api.RequestResponsePair{
+			Request:  *requestRedisMessage,
+			Response: *responseRedisMessage,
+		},
+	}
+}
+
+func splitIdent(ident string) []string {
+	ident = strings.Replace(ident, "->", " ", -1)
+	return strings.Split(ident, " ")
+}
+
+func genKey(split []string) string {
+	key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
+	return key
+}

tap/extensions/redis/read.go

@@ -0,0 +1,474 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	askPrefix         = "ASK "
+	movedPrefix       = "MOVED "
+	clusterDownPrefix = "CLUSTERDOWN "
+	busyPrefix        = "BUSY "
+	noscriptPrefix    = "NOSCRIPT "
+
+	defaultHost         = "localhost"
+	defaultPort         = 6379
+	defaultSentinelPort = 26379
+	defaultTimeout      = 5 * time.Second
+	defaultDatabase     = 2 * time.Second
+
+	dollarByte        = '$'
+	asteriskByte      = '*'
+	plusByte          = '+'
+	minusByte         = '-'
+	colonByte         = ':'
+	notApplicableByte = '0'
+
+	sentinelMasters             = "masters"
+	sentinelGetMasterAddrByName = "get-master-addr-by-name"
+	sentinelReset               = "reset"
+	sentinelSlaves              = "slaves"
+	sentinelFailOver            = "failover"
+	sentinelMonitor             = "monitor"
+	sentinelRemove              = "remove"
+	sentinelSet                 = "set"
+
+	clusterNodes            = "nodes"
+	clusterMeet             = "meet"
+	clusterReset            = "reset"
+	clusterAddSlots         = "addslots"
+	clusterDelSlots         = "delslots"
+	clusterInfo             = "info"
+	clusterGetKeysInSlot    = "getkeysinslot"
+	clusterSetSlot          = "setslot"
+	clusterSetSlotNode      = "node"
+	clusterSetSlotMigrating = "migrating"
+	clusterSetSlotImporting = "importing"
+	clusterSetSlotStable    = "stable"
+	clusterForget           = "forget"
+	clusterFlushSlot        = "flushslots"
+	clusterKeySlot          = "keyslot"
+	clusterCountKeyInSlot   = "countkeysinslot"
+	clusterSaveConfig       = "saveconfig"
+	clusterReplicate        = "replicate"
+	clusterSlaves           = "slaves"
+	clusterFailOver         = "failover"
+	clusterSlots            = "slots"
+	pubSubChannels          = "channels"
+	pubSubNumSub            = "numsub"
+	pubSubNumPat            = "numpat"
+)
+
+//intToByteArr convert int to byte array
+func intToByteArr(a int) []byte {
+	buf := make([]byte, 0)
+	return strconv.AppendInt(buf, int64(a), 10)
+}
+
+var (
+	bytesTrue  = intToByteArr(1)
+	bytesFalse = intToByteArr(0)
+	bytesTilde = []byte("~")
+
+	positiveInfinityBytes = []byte("+inf")
+	negativeInfinityBytes = []byte("-inf")
+)
+
+var (
+	sizeTable = []int{9, 99, 999, 9999, 99999, 999999, 9999999, 99999999,
+		999999999, math.MaxInt32}
+
+	digitTens = []byte{'0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1',
+		'1', '1', '1', '1', '1', '1', '1', '1', '1', '2', '2', '2', '2', '2', '2', '2', '2', '2',
+		'2', '3', '3', '3', '3', '3', '3', '3', '3', '3', '3', '4', '4', '4', '4', '4', '4', '4',
+		'4', '4', '4', '5', '5', '5', '5', '5', '5', '5', '5', '5', '5', '6', '6', '6', '6', '6',
+		'6', '6', '6', '6', '6', '7', '7', '7', '7', '7', '7', '7', '7', '7', '7', '8', '8', '8',
+		'8', '8', '8', '8', '8', '8', '8', '9', '9', '9', '9', '9', '9', '9', '9', '9', '9'}
+
+	digitOnes = []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0',
+		'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8',
+		'9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6',
+		'7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4',
+		'5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2',
+		'3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
+
+	digits = []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a',
+		'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
+		't', 'u', 'v', 'w', 'x', 'y', 'z'}
+)
+
+// receive message from redis
+type RedisInputStream struct {
+	*bufio.Reader
+	Buf   []byte
+	count int
+	limit int
+}
+
+func (r *RedisInputStream) readByte() (byte, error) {
+	err := r.ensureFill()
+	if err != nil {
+		return 0, err
+	}
+	ret := r.Buf[r.count]
+	r.count++
+	return ret, nil
+}
+
+func (r *RedisInputStream) ensureFill() error {
+	if r.count < r.limit {
+		return nil
+	}
+	var err error
+	r.limit, err = r.Read(r.Buf)
+	if err != nil {
+		return newConnectError(err.Error())
+	}
+	r.count = 0
+	if r.limit == -1 {
+		return newConnectError("Unexpected end of stream")
+	}
+	return nil
+}
+
+func (r *RedisInputStream) readLine() (string, error) {
+	buf := ""
+	for {
+		err := r.ensureFill()
+		if err != nil {
+			return "", err
+		}
+		b := r.Buf[r.count]
+		r.count++
+		if b == '\r' {
+			err := r.ensureFill()
+			if err != nil {
+				return "", err
+			}
+			c := r.Buf[r.count]
+			r.count++
+			if c == '\n' {
+				break
+			}
+			buf += string(b)
+			buf += string(c)
+		} else {
+			buf += string(b)
+		}
+	}
+	if buf == "" {
+		return "", newConnectError("It seems like server has closed the connection.")
+	}
+	return buf, nil
+}
+
+func (r *RedisInputStream) readLineBytes() ([]byte, error) {
+	err := r.ensureFill()
+	if err != nil {
+		return nil, err
+	}
+	pos := r.count
+	buf := r.Buf
+	for {
+		if pos == r.limit {
+			return r.readLineBytesSlowly()
+		}
+		p := buf[pos]
+		pos++
+		if p == '\r' {
+			if pos == r.limit {
+				return r.readLineBytesSlowly()
+			}
+			p := buf[pos]
+			pos++
+			if p == '\n' {
+				break
+			}
+		}
+	}
+	N := pos - r.count - 2
+	line := make([]byte, N)
+	j := 0
+	for i := r.count; i <= N; i++ {
+		line[j] = buf[i]
+		j++
+	}
+	r.count = pos
+	return line, nil
+}
+
+func (r *RedisInputStream) readLineBytesSlowly() ([]byte, error) {
+	buf := make([]byte, 0)
+	for {
+		err := r.ensureFill()
+		if err != nil {
+			return nil, err
+		}
+		b := r.Buf[r.count]
+		r.count++
+		if b == 'r' {
+			err := r.ensureFill()
+			if err != nil {
+				return nil, err
+			}
+			c := r.Buf[r.count]
+			r.count++
+			if c == '\n' {
+				break
+			}
+			buf = append(buf, b)
+			buf = append(buf, c)
+		} else {
+			buf = append(buf, b)
+		}
+	}
+	return buf, nil
+}
+
+func (r *RedisInputStream) readIntCrLf() (int64, error) {
+	err := r.ensureFill()
+	if err != nil {
+		return 0, err
+	}
+	buf := r.Buf
+	isNeg := false
+	if buf[r.count] == '-' {
+		isNeg = true
+	}
+	if isNeg {
+		r.count++
+	}
+	value := int64(0)
+	for {
+		err := r.ensureFill()
+		if err != nil {
+			return 0, err
+		}
+		b := buf[r.count]
+		r.count++
+		if b == '\r' {
+			err := r.ensureFill()
+			if err != nil {
+				return 0, err
+			}
+			c := buf[r.count]
+			r.count++
+			if c != '\n' {
+				return 0, newConnectError("Unexpected character!")
+			}
+			break
+		} else {
+			value = value*10 + int64(b) - int64('0')
+		}
+	}
+	if isNeg {
+		return -value, nil
+	}
+	return value, nil
+}
+
+type RedisProtocol struct {
+	is *RedisInputStream
+}
+
+func NewProtocol(is *RedisInputStream) *RedisProtocol {
+	return &RedisProtocol{
+		is: is,
+	}
+}
+
+func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
+	x, r, err := p.process()
+	if err != nil {
+		return
+	}
+	packet = &RedisPacket{}
+	packet.Type = r
+
+	switch x.(type) {
+	case []interface{}:
+		array := x.([]interface{})
+		packet.Command = RedisCommand(strings.ToUpper(string(array[0].([]uint8))))
+		if len(array) > 1 {
+			packet.Key = string(array[1].([]uint8))
+		}
+		if len(array) > 2 {
+			packet.Value = string(array[2].([]uint8))
+		}
+	case []uint8:
+		val := string(x.([]uint8))
+		if packet.Type == types[plusByte] {
+			packet.Keyword = RedisKeyword(strings.ToUpper(val))
+			if !isValidRedisKeyword(keywords, packet.Keyword) {
+				err = errors.New(fmt.Sprintf("Unrecognized keyword: %s", string(packet.Command)))
+				return
+			}
+		} else {
+			packet.Value = val
+		}
+	case string:
+		packet.Value = x.(string)
+	default:
+		msg := fmt.Sprintf("Unrecognized Redis data type: %v\n", reflect.TypeOf(x))
+		log.Printf(msg)
+		err = errors.New(msg)
+		return
+	}
+
+	if packet.Command != "" {
+		if !isValidRedisCommand(commands, packet.Command) {
+			err = errors.New(fmt.Sprintf("Unrecognized command: %s", string(packet.Command)))
+			return
+		}
+	}
+
+	return
+}
+
+func (p *RedisProtocol) process() (v interface{}, r RedisType, err error) {
+	b, err := p.is.readByte()
+	if err != nil {
+		return nil, types[notApplicableByte], newConnectError(err.Error())
+	}
+	switch b {
+	case plusByte:
+		v, err = p.processSimpleString()
+		r = types[plusByte]
+		return
+	case dollarByte:
+		v, err = p.processBulkString()
+		r = types[dollarByte]
+		return
+	case asteriskByte:
+		v, err = p.processArray()
+		r = types[asteriskByte]
+		return
+	case colonByte:
+		v, err = p.processInteger()
+		r = types[colonByte]
+		return
+	case minusByte:
+		v, err = p.processError()
+		r = types[minusByte]
+		return
+	default:
+		return nil, types[notApplicableByte], newConnectError(fmt.Sprintf("Unknown reply: %b", b))
+	}
+}
+
+func (p *RedisProtocol) processSimpleString() ([]byte, error) {
+	return p.is.readLineBytes()
+}
+
+func (p *RedisProtocol) processBulkString() ([]byte, error) {
+	l, err := p.is.readIntCrLf()
+	if err != nil {
+		return nil, newConnectError(err.Error())
+	}
+	if l == -1 {
+		return nil, nil
+	}
+	line := make([]byte, 0)
+	for {
+		err := p.is.ensureFill()
+		if err != nil {
+			return nil, err
+		}
+		b := p.is.Buf[p.is.count]
+		p.is.count++
+		if b == '\r' {
+			err := p.is.ensureFill()
+			if err != nil {
+				return nil, err
+			}
+			c := p.is.Buf[p.is.count]
+			p.is.count++
+			if c != '\n' {
+				return nil, newConnectError("Unexpected character!")
+			}
+			break
+		} else {
+			line = append(line, b)
+		}
+	}
+	return line, nil
+}
+
+func (p *RedisProtocol) processArray() ([]interface{}, error) {
+	l, err := p.is.readIntCrLf()
+	if err != nil {
+		return nil, newConnectError(err.Error())
+	}
+	if l == -1 {
+		return nil, nil
+	}
+	ret := make([]interface{}, 0)
+	for i := 0; i < int(l); i++ {
+		if obj, _, err := p.process(); err != nil {
+			ret = append(ret, err)
+		} else {
+			ret = append(ret, obj)
+		}
+	}
+	return ret, nil
+}
+
+func (p *RedisProtocol) processInteger() (int64, error) {
+	return p.is.readIntCrLf()
+}
+
+func (p *RedisProtocol) processError() (interface{}, error) {
+	msg, err := p.is.readLine()
+	if err != nil {
+		return nil, newConnectError(err.Error())
+	}
+	if strings.HasPrefix(msg, movedPrefix) {
+		host, port, slot, err := p.parseTargetHostAndSlot(msg)
+		if err != nil {
+			return nil, err
+		}
+		return fmt.Sprintf("MovedDataError: %s host: %s port: %d slot: %d", msg, host, port, slot), nil
+	} else if strings.HasPrefix(msg, askPrefix) {
+		host, port, slot, err := p.parseTargetHostAndSlot(msg)
+		if err != nil {
+			return nil, err
+		}
+		return fmt.Sprintf("AskDataError: %s host: %s port: %d slot: %d", msg, host, port, slot), nil
+	} else if strings.HasPrefix(msg, clusterDownPrefix) {
+		return fmt.Sprintf("ClusterError: %s", msg), nil
+	} else if strings.HasPrefix(msg, busyPrefix) {
+		return fmt.Sprintf("BusyError: %s", msg), nil
+	} else if strings.HasPrefix(msg, noscriptPrefix) {
+		return fmt.Sprintf("NoScriptError: %s", msg), nil
+	}
+	return fmt.Sprintf("DataError: %s", msg), nil
+}
+
+func (p *RedisProtocol) parseTargetHostAndSlot(clusterRedirectResponse string) (host string, po int, slot int, err error) {
+	arr := strings.Split(clusterRedirectResponse, " ")
+	host, port := p.extractParts(arr[2])
+	slot, err = strconv.Atoi(arr[1])
+	po, err = strconv.Atoi(port)
+	return
+}
+
+func (p *RedisProtocol) extractParts(from string) (string, string) {
+	idx := strings.LastIndex(from, ":")
+	host := from
+	if idx != -1 {
+		host = from[0:idx]
+	}
+	port := ""
+	if idx != -1 {
+		port = from[idx+1:]
+	}
+	return host, port
+}

tap/extensions/redis/structs.go

@@ -0,0 +1,290 @@
+package main
+
+type RedisType string
+type RedisCommand string
+type RedisKeyword string
+
+var types map[rune]RedisType = map[rune]RedisType{
+	plusByte:          "Simple String",
+	dollarByte:        "Bulk String",
+	asteriskByte:      "Array",
+	colonByte:         "Integer",
+	minusByte:         "Error",
+	notApplicableByte: "N/A",
+}
+
+var commands []RedisCommand = []RedisCommand{
+	"PING",
+	"SET",
+	"GET",
+	"QUIT",
+	"EXISTS",
+	"DEL",
+	"UNLINK",
+	"TYPE",
+	"FLUSHDB",
+	"KEYS",
+	"RANDOMKEY",
+	"RENAME",
+	"RENAMENX",
+	"RENAMEX",
+	"DBSIZE",
+	"EXPIRE",
+	"EXPIREAT",
+	"TTL",
+	"SELECT",
+	"MOVE",
+	"FLUSHALL",
+	"GETSET",
+	"MGET",
+	"SETNX",
+	"SETEX",
+	"MSET",
+	"MSETNX",
+	"DECRBY",
+	"DECR",
+	"INCRBY",
+	"INCR",
+	"APPEND",
+	"SUBSTR",
+	"HSET",
+	"HGET",
+	"HSETNX",
+	"HMSET",
+	"HMGET",
+	"HINCRBY",
+	"HEXISTS",
+	"HDEL",
+	"HLEN",
+	"HKEYS",
+	"HVALS",
+	"HGETALL",
+	"RPUSH",
+	"LPUSH",
+	"LLEN",
+	"LRANGE",
+	"LTRIM",
+	"LINDEX",
+	"LSET",
+	"LREM",
+	"LPOP",
+	"RPOP",
+	"RPOPLPUSH",
+	"SADD",
+	"SMEMBERS",
+	"SREM",
+	"SPOP",
+	"SMOVE",
+	"SCARD",
+	"SISMEMBER",
+	"SINTER",
+	"SINTERSTORE",
+	"SUNION",
+	"SUNIONSTORE",
+	"SDIFF",
+	"SDIFFSTORE",
+	"SRANDMEMBER",
+	"ZADD",
+	"ZRANGE",
+	"ZREM",
+	"ZINCRBY",
+	"ZRANK",
+	"ZREVRANK",
+	"ZREVRANGE",
+	"ZCARD",
+	"ZSCORE",
+	"MULTI",
+	"DISCARD",
+	"EXEC",
+	"WATCH",
+	"UNWATCH",
+	"SORT",
+	"BLPOP",
+	"BRPOP",
+	"AUTH",
+	"SUBSCRIBE",
+	"PUBLISH",
+	"UNSUBSCRIBE",
+	"PSUBSCRIBE",
+	"PUNSUBSCRIBE",
+	"PUBSUB",
+	"ZCOUNT",
+	"ZRANGEBYSCORE",
+	"ZREVRANGEBYSCORE",
+	"ZREMRANGEBYRANK",
+	"ZREMRANGEBYSCORE",
+	"ZUNIONSTORE",
+	"ZINTERSTORE",
+	"ZLEXCOUNT",
+	"ZRANGEBYLEX",
+	"ZREVRANGEBYLEX",
+	"ZREMRANGEBYLEX",
+	"SAVE",
+	"BGSAVE",
+	"BGREWRITEAOF",
+	"LASTSAVE",
+	"SHUTDOWN",
+	"INFO",
+	"MONITOR",
+	"SLAVEOF",
+	"CONFIG",
+	"STRLEN",
+	"SYNC",
+	"LPUSHX",
+	"PERSIST",
+	"RPUSHX",
+	"ECHO",
+	"LINSERT",
+	"DEBUG",
+	"BRPOPLPUSH",
+	"SETBIT",
+	"GETBIT",
+	"BITPOS",
+	"SETRANGE",
+	"GETRANGE",
+	"EVAL",
+	"EVALSHA",
+	"SCRIPT",
+	"SLOWLOG",
+	"OBJECT",
+	"BITCOUNT",
+	"BITOP",
+	"SENTINEL",
+	"DUMP",
+	"RESTORE",
+	"PEXPIRE",
+	"PEXPIREAT",
+	"PTTL",
+	"INCRBYFLOAT",
+	"PSETEX",
+	"CLIENT",
+	"TIME",
+	"MIGRATE",
+	"HINCRBYFLOAT",
+	"SCAN",
+	"HSCAN",
+	"SSCAN",
+	"ZSCAN",
+	"WAIT",
+	"CLUSTER",
+	"ASKING",
+	"PFADD",
+	"PFCOUNT",
+	"PFMERGE",
+	"READONLY",
+	"GEOADD",
+	"GEODIST",
+	"GEOHASH",
+	"GEOPOS",
+	"GEORADIUS",
+	"GEORADIUS_RO",
+	"GEORADIUSBYMEMBER",
+	"GEORADIUSBYMEMBER_RO",
+	"MODULE",
+	"BITFIELD",
+	"HSTRLEN",
+	"TOUCH",
+	"SWAPDB",
+	"MEMORY",
+	"XADD",
+	"XLEN",
+	"XDEL",
+	"XTRIM",
+	"XRANGE",
+	"XREVRANGE",
+	"XREAD",
+	"XACK",
+	"XGROUP",
+	"XREADGROUP",
+	"XPENDING",
+	"XCLAIM",
+}
+
+var keywords []RedisKeyword = []RedisKeyword{
+	"AGGREGATE",
+	"ALPHA",
+	"ASC",
+	"BY",
+	"DESC",
+	"GET",
+	"LIMIT",
+	"MESSAGE",
+	"NO",
+	"NOSORT",
+	"PMESSAGE",
+	"PSUBSCRIBE",
+	"PUNSUBSCRIBE",
+	"OK",
+	"ONE",
+	"QUEUED",
+	"SET",
+	"STORE",
+	"SUBSCRIBE",
+	"UNSUBSCRIBE",
+	"WEIGHTS",
+	"WITHSCORES",
+	"RESETSTAT",
+	"REWRITE",
+	"RESET",
+	"FLUSH",
+	"EXISTS",
+	"LOAD",
+	"KILL",
+	"LEN",
+	"REFCOUNT",
+	"ENCODING",
+	"IDLETIME",
+	"GETNAME",
+	"SETNAME",
+	"LIST",
+	"MATCH",
+	"COUNT",
+	"PING",
+	"PONG",
+	"UNLOAD",
+	"REPLACE",
+	"KEYS",
+	"PAUSE",
+	"DOCTOR",
+	"BLOCK",
+	"NOACK",
+	"STREAMS",
+	"KEY",
+	"CREATE",
+	"MKSTREAM",
+	"SETID",
+	"DESTROY",
+	"DELCONSUMER",
+	"MAXLEN",
+	"GROUP",
+	"IDLE",
+	"TIME",
+	"RETRYCOUNT",
+	"FORCE",
+}
+
+type RedisPacket struct {
+	Type    RedisType    `json:"type"`
+	Command RedisCommand `json:"command"`
+	Key     string       `json:"key"`
+	Value   string       `json:"value"`
+	Keyword RedisKeyword `json:"keyword"`
+}
+
+func isValidRedisCommand(s []RedisCommand, c RedisCommand) bool {
+	for _, v := range s {
+		if v == c {
+			return true
+		}
+	}
+	return false
+}
+
+func isValidRedisKeyword(s []RedisKeyword, c RedisKeyword) bool {
+	for _, v := range s {
+		if v == c {
+			return true
+		}
+	}
+	return false
+}

tap/passive_tapper.go

@@ -13,11 +13,14 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/signal"
 	"runtime"
+	_debug "runtime/debug"
 	"runtime/pprof"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -61,7 +64,7 @@ var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to k
 
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
-var statsTracker = StatsTracker{}
+var appStats = api.AppStats{}
 
 // global
 var stats struct {
@@ -90,9 +93,12 @@ var outputLevel int
 var errorsMap map[string]uint
 var errorsMapMutex sync.Mutex
 var nErrors uint
-var ownIps []string             // global
-var hostMode bool               // global
-var extensions []*api.Extension // global
+var ownIps []string                               // global
+var hostMode bool                                 // global
+var extensions []*api.Extension                   // global
+var filteringOptions *api.TrafficFilteringOptions // global
+
+const baseStreamChannelTimeoutMs int = 5000 * 100
 
 /* minOutputLevel: Error will be printed only if outputLevel is above this value
  * t:              key for errorsMap (counting errors)
@@ -148,17 +154,18 @@ type Context struct {
 	CaptureInfo gopacket.CaptureInfo
 }
 
-func GetStats() AppStats {
-	return statsTracker.appStats
+func GetStats() api.AppStats {
+	return appStats
 }
 
 func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 	return c.CaptureInfo
 }
 
-func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension) {
+func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *api.TrafficFilteringOptions) {
 	hostMode = opts.HostMode
 	extensions = extensionsRef
+	filteringOptions = options
 
 	if GetMemoryProfilingEnabled() {
 		startMemoryProfiler()
@@ -168,11 +175,23 @@ func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem,
 }
 
 func startMemoryProfiler() {
-	dirname := "/app/pprof"
-	rlog.Info("Profiling is on, results will be written to %s", dirname)
+	dumpPath := "/app/pprof"
+	envDumpPath := os.Getenv(MemoryProfilingDumpPath)
+	if envDumpPath != "" {
+		dumpPath = envDumpPath
+	}
+	timeInterval := 60
+	envTimeInterval := os.Getenv(MemoryProfilingTimeIntervalSeconds)
+	if envTimeInterval != "" {
+		if i, err := strconv.Atoi(envTimeInterval); err == nil {
+			timeInterval = i
+		}
+	}
+
+	rlog.Info("Profiling is on, results will be written to %s", dumpPath)
 	go func() {
-		if _, err := os.Stat(dirname); os.IsNotExist(err) {
-			if err := os.Mkdir(dirname, 0777); err != nil {
+		if _, err := os.Stat(dumpPath); os.IsNotExist(err) {
+			if err := os.Mkdir(dumpPath, 0777); err != nil {
 				log.Fatal("could not create directory for profile: ", err)
 			}
 		}
@@ -180,9 +199,9 @@ func startMemoryProfiler() {
 		for true {
 			t := time.Now()
 
-			filename := fmt.Sprintf("%s/%s__mem.prof", dirname, t.Format("15_04_05"))
+			filename := fmt.Sprintf("%s/%s__mem.prof", dumpPath, t.Format("15_04_05"))
 
-			rlog.Info("Writing memory profile to %s\n", filename)
+			rlog.Infof("Writing memory profile to %s\n", filename)
 
 			f, err := os.Create(filename)
 			if err != nil {
@@ -193,13 +212,50 @@ func startMemoryProfiler() {
 				log.Fatal("could not write memory profile: ", err)
 			}
 			_ = f.Close()
-			time.Sleep(time.Minute)
+			time.Sleep(time.Second * time.Duration(timeInterval))
 		}
 	}()
 }
 
+func closeTimedoutTcpStreamChannels() {
+	TcpStreamChannelTimeoutMs := GetTcpChannelTimeoutMs()
+	for {
+		time.Sleep(10 * time.Millisecond)
+		_debug.FreeOSMemory()
+		streams.Range(func(key interface{}, value interface{}) bool {
+			streamWrapper := value.(*tcpStreamWrapper)
+			stream := streamWrapper.stream
+			if stream.superIdentifier.Protocol == nil {
+				if !stream.isClosed && time.Now().After(streamWrapper.createdAt.Add(TcpStreamChannelTimeoutMs)) {
+					stream.Close()
+					appStats.IncDroppedTcpStreams()
+					rlog.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d\n", appStats.DroppedTcpStreams, runtime.NumGoroutine(), TcpStreamChannelTimeoutMs/1000000)
+				}
+			} else {
+				if !stream.superIdentifier.IsClosedOthers {
+					for i := range stream.clients {
+						reader := &stream.clients[i]
+						if reader.extension.Protocol != stream.superIdentifier.Protocol {
+							reader.Close()
+						}
+					}
+					for i := range stream.servers {
+						reader := &stream.servers[i]
+						if reader.extension.Protocol != stream.superIdentifier.Protocol {
+							reader.Close()
+						}
+					}
+					stream.superIdentifier.IsClosedOthers = true
+				}
+			}
+			return true
+		})
+	}
+}
+
 func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	log.SetFlags(log.LstdFlags | log.LUTC | log.Lshortfile)
+	go closeTimedoutTcpStreamChannels()
 
 	defer util.Run()()
 	if *debug {
@@ -275,10 +331,11 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 	source.Lazy = *lazy
 	source.NoCopy = true
 	rlog.Info("Starting to read packets")
-	statsTracker.setStartTime(time.Now())
+	appStats.SetStartTime(time.Now())
 	defragger := ip4defrag.NewIPv4Defragmenter()
 
 	var emitter api.Emitter = &api.Emitting{
+		AppStats:      &appStats,
 		OutputChannel: outputItems,
 	}
 
@@ -321,7 +378,7 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 			errorsSummery := fmt.Sprintf("%v", errorsMap)
 			errorsMapMutex.Unlock()
 			log.Printf("%v (errors: %v, errTypes:%v) - Errors Summary: %s",
-				time.Since(statsTracker.appStats.StartTime),
+				time.Since(appStats.StartTime),
 				nErrors,
 				errorMapLen,
 				errorsSummery,
@@ -344,7 +401,7 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 				cleanStats.closed,
 				cleanStats.deleted,
 			)
-			currentAppStats := statsTracker.dumpStats()
+			currentAppStats := appStats.DumpStats()
 			appStatsJSON, _ := json.Marshal(currentAppStats)
 			log.Printf("app stats - %v", string(appStatsJSON))
 		}
@@ -354,11 +411,18 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 		startMemoryProfiler()
 	}
 
-	for packet := range source.Packets() {
-		packetsCount := statsTracker.incPacketsCount()
+	for {
+		packet, err := source.NextPacket()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			rlog.Debugf("Error:", err)
+			continue
+		}
+		packetsCount := appStats.IncPacketsCount()
 		rlog.Debugf("PACKET #%d", packetsCount)
 		data := packet.Data()
-		statsTracker.updateProcessedBytes(int64(len(data)))
+		appStats.UpdateProcessedBytes(uint64(len(data)))
 		if *hexdumppkt {
 			rlog.Debugf("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data))
 		}
@@ -392,7 +456,7 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 
 		tcp := packet.Layer(layers.LayerTypeTCP)
 		if tcp != nil {
-			statsTracker.incTcpPacketsCount()
+			appStats.IncTcpPacketsCount()
 			tcp := tcp.(*layers.TCP)
 			if *checksum {
 				err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
@@ -410,15 +474,15 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) {
 			assemblerMutex.Unlock()
 		}
 
-		done := *maxcount > 0 && statsTracker.appStats.PacketsCount >= *maxcount
+		done := *maxcount > 0 && int64(appStats.PacketsCount) >= *maxcount
 		if done {
 			errorsMapMutex.Lock()
 			errorMapLen := len(errorsMap)
 			errorsMapMutex.Unlock()
 			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)",
-				statsTracker.appStats.PacketsCount,
-				statsTracker.appStats.ProcessedBytes,
-				time.Since(statsTracker.appStats.StartTime),
+				appStats.PacketsCount,
+				appStats.ProcessedBytes,
+				time.Since(appStats.StartTime),
 				nErrors,
 				errorMapLen)
 		}

tap/settings.go

@@ -3,14 +3,19 @@ package tap
 import (
 	"os"
 	"strconv"
+	"time"
 )
 
 const (
 	MemoryProfilingEnabledEnvVarName          = "MEMORY_PROFILING_ENABLED"
+	MemoryProfilingDumpPath                   = "MEMORY_PROFILING_DUMP_PATH"
+	MemoryProfilingTimeIntervalSeconds        = "MEMORY_PROFILING_TIME_INTERVAL"
 	MaxBufferedPagesTotalEnvVarName           = "MAX_BUFFERED_PAGES_TOTAL"
 	MaxBufferedPagesPerConnectionEnvVarName   = "MAX_BUFFERED_PAGES_PER_CONNECTION"
+	TcpStreamChannelTimeoutMsEnvVarName       = "TCP_STREAM_CHANNEL_TIMEOUT_MS"
 	MaxBufferedPagesTotalDefaultValue         = 5000
 	MaxBufferedPagesPerConnectionDefaultValue = 5000
+	TcpStreamChannelTimeoutMsDefaultValue     = 10000
 )
 
 type globalSettings struct {
@@ -47,6 +52,14 @@ func GetMaxBufferedPagesPerConnection() int {
 	return valueFromEnv
 }
 
+func GetTcpChannelTimeoutMs() time.Duration {
+	valueFromEnv, err := strconv.Atoi(os.Getenv(TcpStreamChannelTimeoutMsEnvVarName))
+	if err != nil {
+		return TcpStreamChannelTimeoutMsDefaultValue * time.Millisecond
+	}
+	return time.Duration(valueFromEnv) * time.Millisecond
+}
+
 func GetMemoryProfilingEnabled() bool {
 	return os.Getenv(MemoryProfilingEnabledEnvVarName) == "1"
 }

tap/stats_tracker.go

@@ -1,104 +0,0 @@
-package tap
-
-import (
-	"sync"
-	"time"
-)
-
-type AppStats struct {
-	StartTime                   time.Time `json:"-"`
-	ProcessedBytes              int64     `json:"processedBytes"`
-	PacketsCount                int64     `json:"packetsCount"`
-	TcpPacketsCount             int64     `json:"tcpPacketsCount"`
-	ReassembledTcpPayloadsCount int64     `json:"reassembledTcpPayloadsCount"`
-	TlsConnectionsCount         int64     `json:"tlsConnectionsCount"`
-	MatchedPairs                int64     `json:"matchedPairs"`
-}
-
-type StatsTracker struct {
-	appStats                         AppStats
-	processedBytesMutex              sync.Mutex
-	packetsCountMutex                sync.Mutex
-	tcpPacketsCountMutex             sync.Mutex
-	reassembledTcpPayloadsCountMutex sync.Mutex
-	tlsConnectionsCountMutex         sync.Mutex
-	matchedPairsMutex                sync.Mutex
-}
-
-func (st *StatsTracker) incMatchedPairs() {
-	st.matchedPairsMutex.Lock()
-	st.appStats.MatchedPairs++
-	st.matchedPairsMutex.Unlock()
-}
-
-func (st *StatsTracker) incPacketsCount() int64 {
-	st.packetsCountMutex.Lock()
-	st.appStats.PacketsCount++
-	currentPacketsCount := st.appStats.PacketsCount
-	st.packetsCountMutex.Unlock()
-	return currentPacketsCount
-}
-
-func (st *StatsTracker) incTcpPacketsCount() {
-	st.tcpPacketsCountMutex.Lock()
-	st.appStats.TcpPacketsCount++
-	st.tcpPacketsCountMutex.Unlock()
-}
-
-func (st *StatsTracker) incReassembledTcpPayloadsCount() {
-	st.reassembledTcpPayloadsCountMutex.Lock()
-	st.appStats.ReassembledTcpPayloadsCount++
-	st.reassembledTcpPayloadsCountMutex.Unlock()
-}
-
-func (st *StatsTracker) incTlsConnectionsCount() {
-	st.tlsConnectionsCountMutex.Lock()
-	st.appStats.TlsConnectionsCount++
-	st.tlsConnectionsCountMutex.Unlock()
-}
-
-func (st *StatsTracker) updateProcessedBytes(size int64) {
-	st.processedBytesMutex.Lock()
-	st.appStats.ProcessedBytes += size
-	st.processedBytesMutex.Unlock()
-}
-
-func (st *StatsTracker) setStartTime(startTime time.Time) {
-	st.appStats.StartTime = startTime
-}
-
-func (st *StatsTracker) dumpStats() *AppStats {
-	currentAppStats := &AppStats{StartTime: st.appStats.StartTime}
-
-	st.processedBytesMutex.Lock()
-	currentAppStats.ProcessedBytes = st.appStats.ProcessedBytes
-	st.appStats.ProcessedBytes = 0
-	st.processedBytesMutex.Unlock()
-
-	st.packetsCountMutex.Lock()
-	currentAppStats.PacketsCount = st.appStats.PacketsCount
-	st.appStats.PacketsCount = 0
-	st.packetsCountMutex.Unlock()
-
-	st.tcpPacketsCountMutex.Lock()
-	currentAppStats.TcpPacketsCount = st.appStats.TcpPacketsCount
-	st.appStats.TcpPacketsCount = 0
-	st.tcpPacketsCountMutex.Unlock()
-
-	st.reassembledTcpPayloadsCountMutex.Lock()
-	currentAppStats.ReassembledTcpPayloadsCount = st.appStats.ReassembledTcpPayloadsCount
-	st.appStats.ReassembledTcpPayloadsCount = 0
-	st.reassembledTcpPayloadsCountMutex.Unlock()
-
-	st.tlsConnectionsCountMutex.Lock()
-	currentAppStats.TlsConnectionsCount = st.appStats.TlsConnectionsCount
-	st.appStats.TlsConnectionsCount = 0
-	st.tlsConnectionsCountMutex.Unlock()
-
-	st.matchedPairsMutex.Lock()
-	currentAppStats.MatchedPairs = st.appStats.MatchedPairs
-	st.appStats.MatchedPairs = 0
-	st.matchedPairsMutex.Unlock()
-
-	return currentAppStats
-}

tap/tcp_reader.go

@@ -47,6 +47,7 @@ func (tid *tcpID) String() string {
 type tcpReader struct {
 	ident              string
 	tcpID              *api.TcpID
+	isClosed           bool
 	isClient           bool
 	isOutgoing         bool
 	msgQueue           chan tcpReaderDataMsg // Channel of captured reassembled tcp payload
@@ -59,6 +60,7 @@ type tcpReader struct {
 	extension          *api.Extension
 	emitter            api.Emitter
 	counterPair        *api.CounterPair
+	sync.Mutex
 }
 
 func (h *tcpReader) Read(p []byte) (int, error) {
@@ -93,10 +95,19 @@ func (h *tcpReader) Read(p []byte) (int, error) {
 	return l, nil
 }
 
+func (h *tcpReader) Close() {
+	h.Lock()
+	if !h.isClosed {
+		h.isClosed = true
+		close(h.msgQueue)
+	}
+	h.Unlock()
+}
+
 func (h *tcpReader) run(wg *sync.WaitGroup) {
 	defer wg.Done()
 	b := bufio.NewReader(h)
-	err := h.extension.Dissector.Dissect(b, h.isClient, h.tcpID, h.counterPair, h.superTimer, h.emitter)
+	err := h.extension.Dissector.Dissect(b, h.isClient, h.tcpID, h.counterPair, h.superTimer, h.parent.superIdentifier, h.emitter, filteringOptions)
 	if err != nil {
 		io.Copy(ioutil.Discard, b)
 	}

tap/tcp_stream.go

@@ -8,6 +8,7 @@ import (
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers" // pulls in all layers decoders
 	"github.com/google/gopacket/reassembly"
+	"github.com/up9inc/mizu/tap/api"
 )
 
 /* It's a connection (bidirectional)
@@ -16,16 +17,19 @@ import (
  * In our implementation, we pass information from ReassembledSG to the tcpReader through a shared channel.
  */
 type tcpStream struct {
-	tcpstate       *reassembly.TCPSimpleFSM
-	fsmerr         bool
-	optchecker     reassembly.TCPOptionCheck
-	net, transport gopacket.Flow
-	isDNS          bool
-	isTapTarget    bool
-	clients        []tcpReader
-	servers        []tcpReader
-	urls           []string
-	ident          string
+	id              int64
+	isClosed        bool
+	superIdentifier *api.SuperIdentifier
+	tcpstate        *reassembly.TCPSimpleFSM
+	fsmerr          bool
+	optchecker      reassembly.TCPOptionCheck
+	net, transport  gopacket.Flow
+	isDNS           bool
+	isTapTarget     bool
+	clients         []tcpReader
+	servers         []tcpReader
+	urls            []string
+	ident           string
 	sync.Mutex
 }
 
@@ -143,15 +147,25 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
 		if length > 0 {
 			// This is where we pass the reassembled information onwards
 			// This channel is read by an tcpReader object
-			statsTracker.incReassembledTcpPayloadsCount()
+			appStats.IncReassembledTcpPayloadsCount()
 			timestamp := ac.GetCaptureInfo().Timestamp
 			if dir == reassembly.TCPDirClientToServer {
-				for _, reader := range t.clients {
-					reader.msgQueue <- tcpReaderDataMsg{data, timestamp}
+				for i := range t.clients {
+					reader := &t.clients[i]
+					reader.Lock()
+					if !reader.isClosed {
+						reader.msgQueue <- tcpReaderDataMsg{data, timestamp}
+					}
+					reader.Unlock()
 				}
 			} else {
-				for _, reader := range t.servers {
-					reader.msgQueue <- tcpReaderDataMsg{data, timestamp}
+				for i := range t.servers {
+					reader := &t.servers[i]
+					reader.Lock()
+					if !reader.isClosed {
+						reader.msgQueue <- tcpReaderDataMsg{data, timestamp}
+					}
+					reader.Unlock()
 				}
 			}
 		}
@@ -160,14 +174,33 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
 
 func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
 	Trace("%s: Connection closed", t.ident)
-	if t.isTapTarget {
-		for _, reader := range t.clients {
-			close(reader.msgQueue)
-		}
-		for _, reader := range t.servers {
-			close(reader.msgQueue)
-		}
+	if t.isTapTarget && !t.isClosed {
+		t.Close()
 	}
 	// do not remove the connection to allow last ACK
 	return false
 }
+
+func (t *tcpStream) Close() {
+	shouldReturn := false
+	t.Lock()
+	if t.isClosed {
+		shouldReturn = true
+	} else {
+		t.isClosed = true
+	}
+	t.Unlock()
+	if shouldReturn {
+		return
+	}
+	streams.Delete(t.id)
+
+	for i := range t.clients {
+		reader := &t.clients[i]
+		reader.Close()
+	}
+	for i := range t.servers {
+		reader := &t.servers[i]
+		reader.Close()
+	}
+}

tap/tcp_stream_factory.go

@@ -3,6 +3,7 @@ package tap
 import (
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/tap/api"
@@ -23,6 +24,14 @@ type tcpStreamFactory struct {
 	Emitter            api.Emitter
 }
 
+type tcpStreamWrapper struct {
+	stream    *tcpStream
+	createdAt time.Time
+}
+
+var streams *sync.Map = &sync.Map{} // global
+var streamId int64 = 0
+
 func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream {
 	rlog.Debugf("* NEW: %s %s", net, transport)
 	fsmOptions := reassembly.TCPSimpleFSMOptions{
@@ -39,15 +48,18 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 	props := factory.getStreamProps(srcIp, srcPort, dstIp, dstPort)
 	isTapTarget := props.isTapTarget
 	stream := &tcpStream{
-		net:         net,
-		transport:   transport,
-		isDNS:       tcp.SrcPort == 53 || tcp.DstPort == 53,
-		isTapTarget: isTapTarget,
-		tcpstate:    reassembly.NewTCPSimpleFSM(fsmOptions),
-		ident:       fmt.Sprintf("%s:%s", net, transport),
-		optchecker:  reassembly.NewTCPOptionCheck(),
+		net:             net,
+		transport:       transport,
+		isDNS:           tcp.SrcPort == 53 || tcp.DstPort == 53,
+		isTapTarget:     isTapTarget,
+		tcpstate:        reassembly.NewTCPSimpleFSM(fsmOptions),
+		ident:           fmt.Sprintf("%s:%s", net, transport),
+		optchecker:      reassembly.NewTCPOptionCheck(),
+		superIdentifier: &api.SuperIdentifier{},
 	}
 	if stream.isTapTarget {
+		streamId++
+		stream.id = streamId
 		for i, extension := range extensions {
 			counterPair := &api.CounterPair{
 				Request:  0,
@@ -89,6 +101,12 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 				emitter:            factory.Emitter,
 				counterPair:        counterPair,
 			})
+
+			streams.Store(stream.id, &tcpStreamWrapper{
+				stream:    stream,
+				createdAt: time.Now(),
+			})
+
 			factory.wg.Add(2)
 			// Start reading from channel stream.reader.bytes
 			go stream.clients[i].run(&factory.wg)
@@ -119,7 +137,7 @@ func (factory *tcpStreamFactory) getStreamProps(srcIP string, srcPort string, ds
 		}
 		return &streamProps{isTapTarget: false, isOutgoing: false}
 	} else {
-		rlog.Debugf("getStreamProps %s", fmt.Sprintf("+ notHost3 %s -> %s:%s", srcIP, dstIP, dstPort))
+		rlog.Debugf("getStreamProps %s", fmt.Sprintf("+ notHost3 %s:%s -> %s:%s", srcIP, srcPort, dstIP, dstPort))
 		return &streamProps{isTapTarget: true}
 	}
 }

ui/package.json

@@ -12,8 +12,8 @@
     "@types/node": "^12.20.10",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.3",
-    "jsonpath": "^1.1.1",
     "axios": "^0.21.1",
+    "jsonpath": "^1.1.1",
     "node-sass": "^5.0.0",
     "numeral": "^2.0.6",
     "protobuf-decoder": "^0.1.0",
@@ -21,7 +21,7 @@
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
-    "react-scrollable-feed": "^1.3.0",
+    "react-scrollable-feed-virtualized": "^1.4.3",
     "react-syntax-highlighter": "^15.4.3",
     "typescript": "^4.2.4",
     "web-vitals": "^1.1.1"

ui/src/App.tsx

@@ -91,7 +91,7 @@ const App = () => {
                 </table>
 
             </span>
-      
+
     return (
         <div className="mizuApp">
             <div className="header">

ui/src/components/EntriesList.tsx

@@ -2,7 +2,7 @@ import {EntryItem} from "./EntryListItem/EntryListItem";
 import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
 import styles from './style/EntriesList.module.sass';
 import spinner from './assets/spinner.svg';
-import ScrollableFeed from "react-scrollable-feed";
+import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
 import {StatusType} from "./Filters";
 import Api from "../helpers/api";
 import down from "./assets/downImg.svg";
@@ -77,9 +77,6 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, fo
         }
         setIsLoadingTop(false);
         const newEntries = [...data, ...entries];
-        if(newEntries.length >= 1000) {
-            newEntries.splice(1000);
-        }
         setEntries(newEntries);
 
         if(scrollTo) {
@@ -100,10 +97,6 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, fo
         }
         scrollTo = document.getElementById(filteredEntries?.[filteredEntries.length -1]?.id);
         let newEntries = [...entries, ...data];
-        if(newEntries.length >= 1000) {
-            setNoMoreDataTop(false);
-            newEntries = newEntries.slice(-1000);
-        }
         setEntries(newEntries);
         if(scrollTo) {
             scrollTo.scrollIntoView({behavior: "smooth"});
@@ -116,19 +109,20 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, fo
                     {isLoadingTop && <div className={styles.spinnerContainer}>
                         <img alt="spinner" src={spinner} style={{height: 25}}/>
                     </div>}
-                    <ScrollableFeed ref={scrollableRef} onScroll={(isAtBottom) => onScrollEvent(isAtBottom)}>
+                    <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onScroll={(isAtBottom) => onScrollEvent(isAtBottom)}>
                         {noMoreDataTop && !connectionOpen && <div id="noMoreDataTop" className={styles.noMoreDataAvailable}>No more data available</div>}
                         {filteredEntries.map(entry => <EntryItem key={entry.id}
                                                         entry={entry}
                                                         setFocusedEntryId={setFocusedEntryId}
-                                                        isSelected={focusedEntryId === entry.id}/>)}
-                        {!connectionOpen && !noMoreDataBottom && <div className={styles.fetchButtonContainer}>
-                            <div className={styles.styledButton} onClick={() => getNewEntries()}>Fetch more entries</div>
-                        </div>}
-                    </ScrollableFeed>
-                    <button type="button" 
-                        className={`${styles.btnLive} ${scrollableList ? styles.showButton : styles.hideButton}`} 
-                        onClick={(_) => scrollableRef.current.scrollToBottom()}>
+                                                        isSelected={focusedEntryId === entry.id}
+                                                        style={{}}/>)}
+                    </ScrollableFeedVirtualized>
+                    {!connectionOpen && !noMoreDataBottom && <div className={styles.fetchButtonContainer}>
+                        <div className={styles.styledButton} onClick={() => getNewEntries()}>Fetch more entries</div>
+                    </div>}
+                    <button type="button"
+                        className={`${styles.btnLive} ${scrollableList ? styles.showButton : styles.hideButton}`}
+                        onClick={(_) => scrollableRef.current.jumpToBottom()}>
                         <img alt="down" src={down} />
                     </button>
                 </div>

ui/src/components/EntryDetailed.tsx

@@ -41,8 +41,7 @@ const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime}) => {
         <Protocol protocol={protocol} horizontal={true}/>
         <div style={{right: "30px", position: "absolute", display: "flex"}}>
             {response.payload && <div style={{margin: "0 18px", opacity: 0.5}}>{formatSize(bodySize)}</div>}
-            <div style={{marginRight: 18, opacity: 0.5}}>{Math.round(elapsedTime)}ms</div>
-            <div style={{opacity: 0.5}}>{'rulesMatched' in data ? data.rulesMatched?.length : '0'} Rules Applied</div>
+            {response.payload && <div style={{marginRight: 18, opacity: 0.5}}>{Math.round(elapsedTime)}ms</div>}
         </div>
     </div>;
 };
@@ -72,7 +71,7 @@ export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData}) => {
         />
         {entryData.data && <EntrySummary data={entryData.data}/>}
         <>
-            {entryData.data && <EntryViewer representation={entryData.representation} color={entryData.protocol.background_color}/>}
+            {entryData.data && <EntryViewer representation={entryData.representation} isRulesEnabled={entryData.isRulesEnabled} rulesMatched={entryData.rulesMatched} elapsedTime={entryData.data.elapsedTime} color={entryData.protocol.backgroundColor}/>}
         </>
     </>
 };

ui/src/components/EntryDetailed/EntrySections.tsx

@@ -153,10 +153,8 @@ export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, ar
 
 
 interface EntryPolicySectionProps {
-    service: string,
     title: string,
     color: string,
-    response: any,
     latency?: number,
     arrayToIterate: any[],
 }
@@ -200,7 +198,7 @@ export const EntryPolicySectionContainer: React.FC<EntryPolicySectionContainerPr
     </CollapsibleContainer>
 }
 
-export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({service, title, color, response, latency, arrayToIterate}) => {
+export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({title, color, latency, arrayToIterate}) => {
     return <React.Fragment>
         {
             arrayToIterate && arrayToIterate.length > 0 ?
@@ -215,10 +213,10 @@ export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({serv
                                                 <>
                                                     {
                                                         rule.Key &&
-                                                        <tr className={styles.dataValue}><td><b>Key</b>:</td><td>{rule.Key}</td></tr>
+                                                        <tr className={styles.dataValue}><td><b>Key:</b></td> <td>{rule.Key}</td></tr>
                                                     }
                                                     {
-                                                        rule.Latency &&
+                                                        rule.Latency !== 0 &&
                                                         <tr className={styles.dataValue}><td><b>Latency:</b></td> <td>{rule.Latency}</td></tr>
                                                     }
                                                     {
@@ -231,7 +229,7 @@ export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({serv
                                                     }
                                                     {
                                                         rule.Service &&
-                                                        <tr className={styles.dataValue}><td><b>Service:</b></td> <td>{service}</td></tr>
+                                                        <tr className={styles.dataValue}><td><b>Service:</b></td> <td>{rule.Service}</td></tr>
                                                     }
                                                     {
                                                         rule.Type &&
@@ -251,7 +249,7 @@ export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({serv
                         </tbody>
                     </table>
                 </EntrySectionContainer>
-                </> : <span/>
+                </> : <span className={styles.noRules}>No rules could be applied to this request.</span>
         }
     </React.Fragment>
 }

ui/src/components/EntryDetailed/EntryViewer.tsx

@@ -33,9 +33,8 @@ const SectionsRepresentation: React.FC<any> = ({data, color}) => {
     return <>{sections}</>;
 }
 
-const AutoRepresentation: React.FC<any> = ({representation, color}) => {
-    const rulesMatched = []
-    const TABS = [
+const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rulesMatched, elapsedTime, color}) => {
+    var TABS = [
         {
             tab: 'request'
         },
@@ -55,6 +54,14 @@ const AutoRepresentation: React.FC<any> = ({representation, color}) => {
 
     const {request, response} = JSON.parse(representation);
 
+    if (!response) {
+        TABS[1]['hidden'] = true;
+    }
+
+    if (!isRulesEnabled) {
+        TABS.pop()
+    }
+
     return <div className={styles.Entry}>
         {<div className={styles.body}>
             <div className={styles.bodyHeader}>
@@ -64,12 +71,11 @@ const AutoRepresentation: React.FC<any> = ({representation, color}) => {
             {currentTab === TABS[0].tab && <React.Fragment>
                 <SectionsRepresentation data={request} color={color}/>
             </React.Fragment>}
-            {currentTab === TABS[1].tab && <React.Fragment>
+            {response && currentTab === TABS[1].tab && <React.Fragment>
                 <SectionsRepresentation data={response} color={color}/>
             </React.Fragment>}
-            {currentTab === TABS[2].tab && <React.Fragment>
-                {// FIXME: Fix here
-                <EntryTablePolicySection service={representation.service} title={'Rule'} color={color} latency={0} response={response} arrayToIterate={rulesMatched ? rulesMatched : []}/>}
+            {TABS.length > 2 && currentTab === TABS[2].tab && <React.Fragment>
+                <EntryTablePolicySection title={'Rule'} color={color} latency={elapsedTime} arrayToIterate={rulesMatched ? rulesMatched : []}/>
             </React.Fragment>}
         </div>}
     </div>;
@@ -77,11 +83,14 @@ const AutoRepresentation: React.FC<any> = ({representation, color}) => {
 
 interface Props {
     representation: any;
-    color: string,
+    isRulesEnabled: boolean;
+    rulesMatched: any;
+    color: string;
+    elapsedTime: number;
 }
 
-const EntryViewer: React.FC<Props> = ({representation, color}) => {
-    return <AutoRepresentation representation={representation} color={color}/>
+const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatched, elapsedTime, color}) => {
+    return <AutoRepresentation representation={representation} isRulesEnabled={isRulesEnabled} rulesMatched={rulesMatched} elapsedTime={elapsedTime} color={color}/>
 };
 
 export default EntryViewer;

ui/src/components/EntryListItem/EntryListItem.module.sass

@@ -19,20 +19,31 @@
 
 .rowSelected
   border: 1px $blue-color solid
-//   border-left: 5px $blue-color solid
-  margin-left: 10px
   margin-right: 3px
 
 .ruleSuccessRow
-  border: 1px $success-color solid
-//   border-left: 5px $success-color solid
+  background: #E8FFF1
+
+.ruleSuccessRowSelected
+  border: 1px #6FCF97 solid
+  border-left: 5px #6FCF97 solid
 
 .ruleFailureRow
   background: #FFE9EF
 
 .ruleFailureRowSelected
   border: 1px $failure-color solid
-//   border-left: 5px $failure-color solid
+  border-left: 5px $failure-color solid
+
+.ruleNumberText
+  font-size: 12px
+  font-weight: 600
+
+.ruleNumberTextFailure
+  color: #DB2156
+
+.ruleNumberTextSuccess
+  color: #219653
 
 .service
   text-overflow: ellipsis

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -16,14 +16,14 @@ interface Entry {
     summary: string,
     service: string,
     id: string,
-    status_code?: number;
+    statusCode?: number;
     url?: string;
     timestamp: Date;
-    source_ip: string,
-    source_port: string,
-    destination_ip: string,
-    destination_port: string,
-	isOutgoing?: boolean;
+    sourceIp: string,
+    sourcePort: string,
+    destinationIp: string,
+    destinationPort: string,
+    isOutgoing?: boolean;
     latency: number;
     rules: Rules;
 }
@@ -38,10 +38,12 @@ interface EntryProps {
     entry: Entry;
     setFocusedEntryId: (id: string) => void;
     isSelected?: boolean;
+    style: object;
 }
 
-export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSelected}) => {
-    const classification = getClassification(entry.status_code)
+export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSelected, style}) => {
+    const classification = getClassification(entry.statusCode)
+    const numberOfRules = entry.rules.numberOfRules
     let ingoingIcon;
     let outgoingIcon;
     switch(classification) {
@@ -61,53 +63,51 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
             break;
         }
     }
-    // let additionalRulesProperties = "";
-    // let ruleSuccess: boolean;
+    let additionalRulesProperties = "";
+    let ruleSuccess: boolean;
     let rule = 'latency' in entry.rules
     if (rule) {
         if (entry.rules.latency !== -1) {
-            if (entry.rules.latency >= entry.latency) {
-                // additionalRulesProperties = styles.ruleSuccessRow
-                // ruleSuccess = true
+            if (entry.rules.latency >= entry.latency || !('latency' in entry)) {
+                additionalRulesProperties = styles.ruleSuccessRow
+                ruleSuccess = true
             } else {
-                // additionalRulesProperties = styles.ruleFailureRow
-                // ruleSuccess = false
+                additionalRulesProperties = styles.ruleFailureRow
+                ruleSuccess = false
             }
             if (isSelected) {
-                // additionalRulesProperties += ` ${entry.rules.latency >= entry.latency ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
+                additionalRulesProperties += ` ${entry.rules.latency >= entry.latency ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
             }
         } else {
             if (entry.rules.status) {
-                // additionalRulesProperties = styles.ruleSuccessRow
-                // ruleSuccess = true
+                additionalRulesProperties = styles.ruleSuccessRow
+                ruleSuccess = true
             } else {
-                // additionalRulesProperties = styles.ruleFailureRow
-                // ruleSuccess = false
+                additionalRulesProperties = styles.ruleFailureRow
+                ruleSuccess = false
             }
             if (isSelected) {
-                // additionalRulesProperties += ` ${entry.rules.status ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
+                additionalRulesProperties += ` ${entry.rules.status ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
             }
         }
     }
-    let backgroundColor = "";
-    if ('latency' in entry.rules) {
-        if (entry.rules.latency !== -1) {
-            backgroundColor = entry.rules.latency >= entry.latency ? styles.ruleSuccessRow : styles.ruleFailureRow
-        } else {
-            backgroundColor = entry.rules.status ? styles.ruleSuccessRow : styles.ruleFailureRow
-        }
-    }
     return <>
         <div
             id={entry.id}
             className={`${styles.row}
-            ${isSelected ? styles.rowSelected : backgroundColor}`}
+            ${isSelected && !rule ? styles.rowSelected : additionalRulesProperties}`}
             onClick={() => setFocusedEntryId(entry.id)}
-            style={{border: isSelected ? `1px ${entry.protocol.background_color} solid` : "1px transparent solid"}}
+            style={{
+                border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid",
+                position: "absolute",
+                top: style['top'],
+                marginTop: style['marginTop'],
+                width: "calc(100% - 25px)",
+            }}
         >
             <Protocol protocol={entry.protocol} horizontal={false}/>
-            {((entry.protocol.name === "http" && "status_code" in entry) || entry.status_code !== 0) && <div>
-                <StatusCode statusCode={entry.status_code}/>
+            {((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0) && <div>
+                <StatusCode statusCode={entry.statusCode}/>
             </div>}
             <div className={styles.endpointServiceContainer}>
                 <EndpointPath method={entry.method} path={entry.summary}/>
@@ -115,14 +115,21 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
                     <span title="Service Name">{entry.service}</span>
                 </div>
             </div>
+            {
+                rule ?
+                    <div className={`${styles.ruleNumberText} ${ruleSuccess ? styles.ruleNumberTextSuccess : styles.ruleNumberTextFailure}`}>
+                        {`Rules (${numberOfRules})`}
+                    </div>
+                : ""
+            }
             <div className={styles.directionContainer}>
-                <span className={styles.port} title="Source Port">{entry.source_port}</span>
+                <span className={styles.port} title="Source Port">{entry.sourcePort}</span>
                 {entry.isOutgoing ?
                     <img src={outgoingIcon} alt="Ingoing traffic" title="Ingoing"/>
                     :
                     <img src={ingoingIcon} alt="Outgoing traffic" title="Outgoing"/>
                 }
-                <span className={styles.port} title="Destination Port">{entry.destination_port}</span>
+                <span className={styles.port} title="Destination Port">{entry.destinationPort}</span>
             </div>
             <div className={styles.timestamp}>
                 <span title="Timestamp">
@@ -131,4 +138,5 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
             </div>
         </div>
     </>
-};
+
+}

ui/src/components/TrafficPage.tsx

@@ -86,10 +86,6 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     }
                     if (!focusedEntryId) setFocusedEntryId(entry.id)
                     let newEntries = [...entries];
-                    if (entries.length === 1000) {
-                        newEntries = newEntries.splice(1);
-                        setNoMoreDataTop(false);
-                    }
                     setEntries([...newEntries, entry])
                     if(listEntry.current) {
                         if(isScrollable(listEntry.current.firstChild)) {

ui/src/components/UI/Protocol.tsx

@@ -3,12 +3,12 @@ import styles from './style/Protocol.module.sass';
 
 export interface ProtocolInterface {
     name: string
-    long_name: string
+    longName: string
     abbreviation: string
-    background_color: string
-    foreground_color: string
-    font_size: number
-    reference_link: string
+    backgroundColor: string
+    foregroundColor: string
+    fontSize: number
+    referenceLink: string
     ports: string[]
     inbound_ports: string
 }
@@ -20,29 +20,29 @@ interface ProtocolProps {
 
 const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal}) => {
     if (horizontal) {
-        return <a target="_blank" rel="noopener noreferrer" href={protocol.reference_link}>
+        return <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
             <span
                 className={`${styles.base} ${styles.horizontal}`}
                 style={{
-                    backgroundColor: protocol.background_color,
-                    color: protocol.foreground_color,
+                    backgroundColor: protocol.backgroundColor,
+                    color: protocol.foregroundColor,
                     fontSize: 13,
                 }}
                 title={protocol.abbreviation}
             >
-                {protocol.long_name}
+                {protocol.longName}
             </span>
         </a>
     } else {
-        return <a target="_blank" rel="noopener noreferrer" href={protocol.reference_link}>
+        return <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
             <span
                 className={`${styles.base} ${styles.vertical}`}
                 style={{
-                    backgroundColor: protocol.background_color,
-                    color: protocol.foreground_color,
-                    fontSize: protocol.font_size,
+                    backgroundColor: protocol.backgroundColor,
+                    color: protocol.foregroundColor,
+                    fontSize: protocol.fontSize,
                 }}
-                title={protocol.long_name}
+                title={protocol.longName}
             >
                 {protocol.abbreviation}
             </span>

ui/src/components/style/EntriesList.module.sass

@@ -14,7 +14,6 @@
   flex-direction: column
   overflow: hidden
   flex-grow: 1
-  padding-top: 20px
 
 .footer
   display: flex