Improve Go TLS address availability (#1207)

Fetch source and destination addresses with bpf from tcp kprobes, similar to how it is done for openssl lib.
Chunk contains both source address and destination address.
FD is no longer used to obtain addresses.

.github/workflows/static_code_analysis.yml

@@ -32,6 +32,10 @@ jobs:
         id: agent_modified_files
         run: devops/check_modified_files.sh agent/
 
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
       - name: Go lint - agent
         uses: golangci/golangci-lint-action@v2
         if: steps.agent_modified_files.outputs.matched == 'true'

.github/workflows/test.yml

@@ -40,6 +40,10 @@ jobs:
         run: |
           ./devops/install-capstone.sh
 
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
       - name: Check CLI modified files
         id: cli_modified_files
         run: devops/check_modified_files.sh cli/

.gitignore

@@ -56,3 +56,6 @@ tap/extensions/*/expect
 
 # Ignore *.log files
 *.log
+
+# Object files
+*.o

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent agent-debug cli tap docker
+.PHONY: help ui agent agent-debug cli tap docker bpf clean-bpf
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -20,6 +20,13 @@ TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
 export VER?=0.0
+ARCH=$(shell uname -m)
+ifeq ($(ARCH),$(filter $(ARCH),aarch64 arm64))
+	BPF_O_ARCH_LABEL=arm64
+else
+	BPF_O_ARCH_LABEL=x86
+endif
+BPF_O_FILES = tap/tlstapper/tlstapper46_bpfel_$(BPF_O_ARCH_LABEL).o tap/tlstapper/tlstapper_bpfel_$(BPF_O_ARCH_LABEL).o
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -31,11 +38,17 @@ cli: ## Build CLI.
 cli-debug: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build-debug
 
-agent: ## Build agent.
+agent: bpf ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
 	@ls -l agent/build
 
+bpf: $(BPF_O_FILES)
+
+$(BPF_O_FILES): $(wildcard tap/tlstapper/bpf/**/*.[ch])
+	@(echo "building tlstapper bpf")
+	@(./tap/tlstapper/bpf-builder/build.sh)
+
 agent-debug: ## Build agent for debug.
 	@(echo "building mizu agent for debug.." )
 	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
@@ -76,6 +89,9 @@ clean-cli:  ## Clean CLI.
 clean-docker:  ## Run clean docker
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+clean-bpf:
+	@(rm $(BPF_O_FILES) ; echo "bpf cleanup done" )
+
 test-lint:  ## Run lint on all modules
 	cd agent && golangci-lint run
 	cd shared && golangci-lint run

acceptanceTests/cypress.config.js

@@ -11,7 +11,6 @@ module.exports = defineConfig({
     testUrl: 'http://localhost:8899/',
     redactHeaderContent: 'User-Header[REDACTED]',
     redactBodyContent: '{ "User": "[REDACTED]" }',
-    regexMaskingBodyContent: '[REDACTED]',
     greenFilterColor: 'rgb(210, 250, 210)',
     redFilterColor: 'rgb(250, 214, 220)',
     bodyJsonClass: '.hljs',

acceptanceTests/cypress/e2e/tests/RegexMasking.js

@@ -1,7 +0,0 @@
-import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/go.mod

@@ -18,7 +18,6 @@ require (
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-logr/logr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -29,7 +28,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/up9inc/mizu/logger v0.0.0 // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect

acceptanceTests/go.sum

@@ -206,7 +206,6 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

acceptanceTests/setup.sh

@@ -27,7 +27,7 @@ else
 fi
 
 echo "Starting minikube..."
-minikube start --cpus 2 --memory 6946
+minikube start --cpus 2 --memory 6000
 
 echo "Creating mizu tests namespaces"
 kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -

acceptanceTests/tap_test.go

@@ -2,10 +2,8 @@ package acceptanceTests
 
 import (
 	"archive/zip"
-	"bytes"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os/exec"
 	"path"
 	"strings"
@@ -343,7 +341,7 @@ func TestTapRedact(t *testing.T) {
 
 	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-	tapCmdArgs = append(tapCmdArgs, "--redact")
+	tapCmdArgs = append(tapCmdArgs, "--redact", "--set", "tap.redact-patterns.request-headers=User-Header", "--set", "tap.redact-patterns.request-body=User")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
@@ -429,60 +427,6 @@ func TestTapNoRedact(t *testing.T) {
 	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/NoRedact.js\"")
 }
 
-func TestTapRegexMasking(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := GetCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := GetDefaultTapCommandArgs()
-
-	tapNamespace := GetDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmdArgs = append(tapCmdArgs, "--redact")
-
-	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
-
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
-	for i := 0; i < DefaultEntriesCount; i++ {
-		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
-		if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
-			t.Errorf("failed to send proxy request, err: %v", requestErr)
-			return
-		}
-	}
-
-	RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"")
-
-}
-
 func TestTapIgnoredUserAgents(t *testing.T) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")

acceptanceTests/testsUtils.go

@@ -215,12 +215,11 @@ func DeleteKubeFile(kubeContext string, namespace string, filename string) error
 func getDefaultCommandArgs() []string {
 	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
 	setFlag := "--set"
-	telemetry := "telemetry=false"
 	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
-	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
+	return []string{setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
 func GetDefaultTapCommandArgs() []string {

agent/go.mod

@@ -48,7 +48,6 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/beevik/etree v1.1.0 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
 	github.com/cilium/ebpf v0.9.0 // indirect

agent/go.sum

@@ -101,7 +101,6 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

agent/pkg/api/entry_streamer_socket_connector.go

@@ -22,7 +22,7 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
 	if params.EnableFullEntries {
 		message, _ = models.CreateFullEntryWebSocketMessage(entry)
 	} else {
-		protocol, ok := protocolsMap[entry.ProtocolId]
+		protocol, ok := protocolsMap[entry.Protocol.ToString()]
 		if !ok {
 			return fmt.Errorf("protocol not found, protocol: %v", protocol)
 		}

agent/pkg/api/main.go

@@ -126,7 +126,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
 
 		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
-		oasGenerator.HandleEntry(mizuEntry, &item.Protocol)
+		oasGenerator.HandleEntry(mizuEntry)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -97,7 +97,9 @@ func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool
 	websocketIdsLock.Unlock()
 
 	defer func() {
-		socketCleanup(socketId, connectedWebsockets[socketId])
+		if socketConnection := connectedWebsockets[socketId]; socketConnection != nil {
+			socketCleanup(socketId, socketConnection)
+		}
 	}()
 
 	eventHandlers.WebSocketConnect(c, socketId, isTapper)

agent/pkg/controllers/service_map_controller_test.go

@@ -36,11 +36,13 @@ var (
 )
 
 var ProtocolHttp = &tapApi.Protocol{
-	Name:            "http",
+	ProtocolSummary: tapApi.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.1",
+		Abbreviation: "HTTP",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
-	Abbreviation:    "HTTP",
 	Macro:           "http",
-	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,

agent/pkg/controllers/status_controller.go

@@ -1,7 +1,10 @@
 package controllers
 
 import (
+	"fmt"
 	"net/http"
+	"strconv"
+	"time"
 
 	core "k8s.io/api/core/v1"
 
@@ -80,7 +83,24 @@ func GetGeneralStats(c *gin.Context) {
 }
 
 func GetTrafficStats(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetTrafficStats())
+	startTime, endTime, err := getStartEndTime(c)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, providers.GetTrafficStats(startTime, endTime))
+}
+
+func getStartEndTime(c *gin.Context) (time.Time, time.Time, error) {
+	startTimeValue, err := strconv.Atoi(c.Query("startTimeMs"))
+	if err != nil {
+		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid start time: %v", err)
+	}
+	endTimeValue, err := strconv.Atoi(c.Query("endTimeMs"))
+	if err != nil {
+		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid end time: %v", err)
+	}
+	return time.UnixMilli(int64(startTimeValue)), time.UnixMilli(int64(endTimeValue)), nil
 }
 
 func GetCurrentResolvingInformation(c *gin.Context) {

agent/pkg/entries/entries_provider.go

@@ -38,7 +38,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 			return nil, nil, err
 		}
 
-		protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+		protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
 		if !ok {
 			return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
 		}
@@ -77,7 +77,7 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 		return nil, errors.New(string(bytes))
 	}
 
-	protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+	protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
 	if !ok {
 		return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
 	}

agent/pkg/har/utils.go

@@ -11,75 +11,30 @@ import (
 	"github.com/up9inc/mizu/logger"
 )
 
-// Keep it because we might want cookies in the future
+func BuildHeaders(rawHeaders map[string]interface{}) ([]Header, string, string, string, string, string) {
-//func BuildCookies(rawCookies []interface{}) []har.Cookie {
-//	cookies := make([]har.Cookie, 0, len(rawCookies))
-//
-//	for _, cookie := range rawCookies {
-//		c := cookie.(map[string]interface{})
-//		expiresStr := ""
-//		if c["expires"] != nil {
-//			expiresStr = c["expires"].(string)
-//		}
-//		expires, _ := time.Parse(time.RFC3339, expiresStr)
-//		httpOnly := false
-//		if c["httponly"] != nil {
-//			httpOnly, _ = strconv.ParseBool(c["httponly"].(string))
-//		}
-//		secure := false
-//		if c["secure"] != nil {
-//			secure, _ = strconv.ParseBool(c["secure"].(string))
-//		}
-//		path := ""
-//		if c["path"] != nil {
-//			path = c["path"].(string)
-//		}
-//		domain := ""
-//		if c["domain"] != nil {
-//			domain = c["domain"].(string)
-//		}
-//
-//		cookies = append(cookies, har.Cookie{
-//			Name:        c["name"].(string),
-//			Value:       c["value"].(string),
-//			Path:        path,
-//			Domain:      domain,
-//			HTTPOnly:    httpOnly,
-//			Secure:      secure,
-//			Expires:     expires,
-//			Expires8601: expiresStr,
-//		})
-//	}
-//
-//	return cookies
-//}
-
-func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) {
 	var host, scheme, authority, path, status string
 	headers := make([]Header, 0, len(rawHeaders))
 
-	for _, header := range rawHeaders {
+	for key, value := range rawHeaders {
-		h := header.(map[string]interface{})
-
 		headers = append(headers, Header{
-			Name:  h["name"].(string),
+			Name:  key,
-			Value: h["value"].(string),
+			Value: value.(string),
 		})
 
-		if h["name"] == "Host" {
+		if key == "Host" {
-			host = h["value"].(string)
+			host = value.(string)
 		}
-		if h["name"] == ":authority" {
+		if key == ":authority" {
-			authority = h["value"].(string)
+			authority = value.(string)
 		}
-		if h["name"] == ":scheme" {
+		if key == ":scheme" {
-			scheme = h["value"].(string)
+			scheme = value.(string)
 		}
-		if h["name"] == ":path" {
+		if key == ":path" {
-			path = h["value"].(string)
+			path = value.(string)
 		}
-		if h["name"] == ":status" {
+		if key == ":status" {
-			status = h["value"].(string)
+			status = value.(string)
 		}
 	}
 
@@ -119,8 +74,8 @@ func BuildPostParams(rawParams []interface{}) []Param {
 }
 
 func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
-	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
+	headers, host, scheme, authority, path, _ := BuildHeaders(request["headers"].(map[string]interface{}))
-	cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0)
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
@@ -134,12 +89,20 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 	}
 
 	queryString := make([]QueryString, 0)
-	for _, _qs := range request["_queryString"].([]interface{}) {
+	for key, value := range request["queryString"].(map[string]interface{}) {
-		qs := _qs.(map[string]interface{})
+		if valuesInterface, ok := value.([]interface{}); ok {
-		queryString = append(queryString, QueryString{
+			for _, valueInterface := range valuesInterface {
-			Name:  qs["name"].(string),
+				queryString = append(queryString, QueryString{
-			Value: qs["value"].(string),
+					Name:  key,
-		})
+					Value: valueInterface.(string),
+				})
+			}
+		} else {
+			queryString = append(queryString, QueryString{
+				Name:  key,
+				Value: value.(string),
+			})
+		}
 	}
 
 	url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
@@ -172,8 +135,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 }
 
 func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
-	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
+	headers, _, _, _, _, _status := BuildHeaders(response["headers"].(map[string]interface{}))
-	cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0)
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]

agent/pkg/oas/oas_generator.go

@@ -16,7 +16,7 @@ var (
 )
 
 type OasGeneratorSink interface {
-	HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol)
+	HandleEntry(mizuEntry *api.Entry)
 }
 
 type OasGenerator interface {
@@ -58,12 +58,12 @@ func (g *defaultOasGenerator) IsStarted() bool {
 	return g.started
 }
 
-func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol) {
+func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
 	if !g.started {
 		return
 	}
 
-	if protocol.Name == "http" {
+	if mizuEntry.Protocol.Name == "http" {
 		dest := mizuEntry.Destination.Name
 		if dest == "" {
 			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
@@ -85,7 +85,7 @@ func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry, protocol *api.Pr
 
 		g.handleHARWithSource(entryWSource)
 	} else {
-		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, protocol.Name)
+		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, mizuEntry.Protocol.Name)
 	}
 }
 

agent/pkg/providers/stats_provider.go

@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"reflect"
-	"strings"
 	"sync"
 	"time"
 
@@ -82,13 +81,12 @@ func GetGeneralStats() *GeneralStats {
 
 func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
 	for item, value := range protocolMap {
-		splitted := strings.SplitN(item, "/", 3)
+		protocolToColor[api.GetProtocolSummary(item).Abbreviation] = value.BackgroundColor
-		protocolToColor[splitted[len(splitted)-1]] = value.BackgroundColor
 	}
 }
 
-func GetTrafficStats() *TrafficStatsResponse {
+func GetTrafficStats(startTime time.Time, endTime time.Time) *TrafficStatsResponse {
-	bucketsStatsCopy := getBucketStatsCopy()
+	bucketsStatsCopy := getFilteredBucketStatsCopy(startTime, endTime)
 
 	return &TrafficStatsResponse{
 		Protocols:     getAvailableProtocols(bucketsStatsCopy),
@@ -264,7 +262,7 @@ func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string
 	return protocolsData
 }
 
-func getBucketStatsCopy() BucketStats {
+func getFilteredBucketStatsCopy(startTime time.Time, endTime time.Time) BucketStats {
 	bucketStatsCopy := BucketStats{}
 	bucketStatsLocker.Lock()
 	if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
@@ -272,7 +270,18 @@ func getBucketStatsCopy() BucketStats {
 		return nil
 	}
 	bucketStatsLocker.Unlock()
-	return bucketStatsCopy
+
+	filteredBucketStatsCopy := BucketStats{}
+	interval := InternalBucketThreshold
+
+	for _, bucket := range bucketStatsCopy {
+		if (bucket.BucketTime.After(startTime.Add(-1*interval/2).Round(interval)) && bucket.BucketTime.Before(endTime.Add(-1*interval/2).Round(interval))) ||
+			bucket.BucketTime.Equal(startTime.Add(-1*interval/2).Round(interval)) ||
+			bucket.BucketTime.Equal(endTime.Add(-1*interval/2).Round(interval)) {
+			filteredBucketStatsCopy = append(filteredBucketStatsCopy, bucket)
+		}
+	}
+	return filteredBucketStatsCopy
 }
 
 func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {

agent/pkg/providers/stats_provider_test.go

@@ -26,7 +26,7 @@ func TestEntryAddedCount(t *testing.T) {
 
 	entryBucketKey := time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC)
 	valueLessThanBucketThreshold := time.Second * 130
-	mockSummery := &api.BaseEntry{Protocol: api.Protocol{Name: "mock"}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
 			for i := 0; i < entriesCount; i++ {
@@ -61,7 +61,7 @@ func TestEntryAddedVolume(t *testing.T) {
 	var expectedEntriesCount int
 	var expectedVolumeInGB float64
 
-	mockSummery := &api.BaseEntry{Protocol: api.Protocol{Name: "mock"}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
 
 	for _, data := range tests {
 		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {

agent/pkg/servicemap/servicemap_test.go

@@ -50,11 +50,13 @@ var (
 		IP:   fmt.Sprintf("%s.%s", Ip, UnresolvedNodeName),
 	}
 	ProtocolHttp = &tapApi.Protocol{
-		Name:            "http",
+		ProtocolSummary: tapApi.ProtocolSummary{
+			Name:         "http",
+			Version:      "1.1",
+			Abbreviation: "HTTP",
+		},
 		LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
-		Abbreviation:    "HTTP",
 		Macro:           "http",
-		Version:         "1.1",
 		BackgroundColor: "#205cf5",
 		ForegroundColor: "#ffffff",
 		FontSize:        12,
@@ -63,11 +65,13 @@ var (
 		Priority:        0,
 	}
 	ProtocolRedis = &tapApi.Protocol{
-		Name:            "redis",
+		ProtocolSummary: tapApi.ProtocolSummary{
+			Name:         "redis",
+			Version:      "3.x",
+			Abbreviation: "REDIS",
+		},
 		LongName:        "Redis Serialization Protocol",
-		Abbreviation:    "REDIS",
 		Macro:           "redis",
-		Version:         "3.x",
 		BackgroundColor: "#a41e11",
 		ForegroundColor: "#ffffff",
 		FontSize:        11,

cli/apiserver/provider.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"time"
 
 	"github.com/up9inc/mizu/cli/utils"
@@ -93,45 +91,3 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 		}
 	}
 }
-
-func (provider *Provider) GetGeneralStats() (map[string]interface{}, error) {
-	generalStatsUrl := fmt.Sprintf("%s/status/general", provider.url)
-
-	response, requestErr := utils.Get(generalStatsUrl, provider.client)
-	if requestErr != nil {
-		return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr)
-	}
-
-	defer response.Body.Close()
-
-	data, readErr := ioutil.ReadAll(response.Body)
-	if readErr != nil {
-		return nil, fmt.Errorf("failed to read general stats for telemetry, err: %v", readErr)
-	}
-
-	var generalStats map[string]interface{}
-	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
-		return nil, fmt.Errorf("failed to parse general stats for telemetry, err: %v", parseErr)
-	}
-	return generalStats, nil
-}
-
-func (provider *Provider) GetVersion() (string, error) {
-	versionUrl, _ := url.Parse(fmt.Sprintf("%s/metadata/version", provider.url))
-	req := &http.Request{
-		Method: http.MethodGet,
-		URL:    versionUrl,
-	}
-	statusResp, err := utils.Do(req, provider.client)
-	if err != nil {
-		return "", err
-	}
-	defer statusResp.Body.Close()
-
-	versionResponse := &shared.VersionResponse{}
-	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
-		return "", err
-	}
-
-	return versionResponse.Ver, nil
-}

cli/cmd/check.go

@@ -4,7 +4,6 @@ import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -12,7 +11,6 @@ var checkCmd = &cobra.Command{
 	Use:   "check",
 	Short: "Check the Mizu installation for potential problems",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("check", nil)
 		runMizuCheck()
 		return nil
 	},

cli/cmd/clean.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/telemetry"
 )
 
 var cleanCmd = &cobra.Command{
 	Use:   "clean",
 	Short: "Removes all mizu resources",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("clean", nil)
 		performCleanCommand()
 		return nil
 	},

cli/cmd/config.go

@@ -7,7 +7,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/logger"
 )
@@ -16,8 +15,6 @@ var configCmd = &cobra.Command{
 	Use:   "config",
 	Short: "Generate config with default values",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("config", config.Config.Config)
-
 		configWithDefaults, err := config.GetConfigWithDefaults()
 		if err != nil {
 			logger.Log.Errorf("Failed generating config with defaults, err: %v", err)

cli/cmd/install.go

@@ -4,7 +4,6 @@ import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -12,7 +11,6 @@ var installCmd = &cobra.Command{
 	Use:   "install",
 	Short: "Installs mizu components",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("install", nil)
 		runMizuInstall()
 		return nil
 	},

cli/cmd/logs.go

@@ -9,7 +9,6 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -17,8 +16,6 @@ var logsCmd = &cobra.Command{
 	Use:   "logs",
 	Short: "Create a zip file with logs for Github issue or troubleshoot",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("logs", config.Config.Logs)
-
 		kubernetesProvider, err := getKubernetesProviderForCli()
 		if err != nil {
 			return nil

cli/cmd/tap.go

@@ -48,7 +48,6 @@ func init() {
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
-	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.EnableRedactionTapName, defaultTapConfig.EnableRedaction, "Enables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.")

cli/cmd/tapRunner.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/up9inc/mizu/cli/resources"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/utils"
 
 	core "k8s.io/api/core/v1"
@@ -109,8 +108,6 @@ func RunMizuTap() {
 }
 
 func finishTapExecution(kubernetesProvider *kubernetes.Provider) {
-	telemetry.ReportTapTelemetry(apiProvider, config.Config.Tap, state.startTime)
-
 	finishMizuExecution(kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
 }
 
@@ -126,7 +123,6 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		AgentDatabasePath:      shared.DataDirPath,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
-		Telemetry:              config.Config.Telemetry,
 	}
 
 	return &mizuAgentConfig
@@ -151,17 +147,18 @@ func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.
 	}
 }
 
-func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error {
+func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, startTime time.Time) error {
 	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
-		TargetNamespaces:         targetNamespaces,
+		TargetNamespaces:       targetNamespaces,
-		PodFilterRegex:           *config.Config.Tap.PodRegex(),
+		PodFilterRegex:         *config.Config.Tap.PodRegex(),
-		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
+		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
-		AgentImage:               config.Config.AgentImage,
+		AgentImage:             config.Config.AgentImage,
-		TapperResources:          config.Config.Tap.TapperResources,
+		TapperResources:        config.Config.Tap.TapperResources,
-		ImagePullPolicy:          config.Config.ImagePullPolicy(),
+		ImagePullPolicy:        config.Config.ImagePullPolicy(),
-		LogLevel:                 config.Config.LogLevel(),
+		LogLevel:               config.Config.LogLevel(),
-		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
+		MizuApiFilteringOptions: api.TrafficFilteringOptions{
-		MizuApiFilteringOptions:  mizuApiFilteringOptions,
+			IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
+		},
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
 		ServiceMesh:              config.Config.Tap.ServiceMesh,
 		Tls:                      config.Config.Tap.Tls,
@@ -229,27 +226,6 @@ func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError)
 	}
 }
 
-func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
-	var compiledRegexSlice []*api.SerializableRegexp
-
-	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
-		compiledRegexSlice = make([]*api.SerializableRegexp, 0)
-		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
-			compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr)
-			if err != nil {
-				return nil, err
-			}
-			compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
-		}
-	}
-
-	return &api.TrafficFilteringOptions{
-		PlainTextMaskingRegexes: compiledRegexSlice,
-		IgnoredUserAgents:       config.Config.Tap.IgnoredUserAgents,
-		EnableRedaction:        config.Config.Tap.EnableRedaction,
-	}, nil
-}
-
 func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName))
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
@@ -367,8 +343,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel, config.Config.Tap.GuiPort)
 
-	options, _ := getMizuApiFilteringOptions()
+	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, state.startTime); err != nil {
-	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", errormessage.FormatError(err)))
 		cancel()
 	}

cli/cmd/version.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 
 	"github.com/creasty/defaults"
@@ -18,8 +17,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("version", config.Config.Version)
-
 		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
 			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.Ver, mizu.Branch, mizu.GitCommitHash)

cli/cmd/view.go

@@ -3,9 +3,7 @@ package cmd
 import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -13,7 +11,6 @@ var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("view", config.Config.View)
 		runMizuView()
 		return nil
 	},

cli/config/configStruct.go

@@ -31,7 +31,6 @@ type ConfigStruct struct {
 	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
-	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
 	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
 	KubeContext            string                      `yaml:"kube-context"`

cli/config/configStructs/tapConfig.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strings"
 
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
@@ -15,38 +16,43 @@ import (
 )
 
 const (
-	GuiPortTapName                = "gui-port"
+	GuiPortTapName               = "gui-port"
-	NamespacesTapName             = "namespaces"
+	NamespacesTapName            = "namespaces"
-	AllNamespacesTapName          = "all-namespaces"
+	AllNamespacesTapName         = "all-namespaces"
-	PlainTextFilterRegexesTapName = "regex-masking"
+	EnableRedactionTapName       = "redact"
-	EnableRedactionTapName        = "redact"
+	HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
-	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	InsertionFilterName          = "insertion-filter"
-	InsertionFilterName           = "insertion-filter"
+	DryRunTapName                = "dry-run"
-	DryRunTapName                 = "dry-run"
+	ServiceMeshName              = "service-mesh"
-	ServiceMeshName               = "service-mesh"
+	TlsName                      = "tls"
-	TlsName                       = "tls"
+	ProfilerName                 = "profiler"
-	ProfilerName                  = "profiler"
+	MaxLiveStreamsName           = "max-live-streams"
-	MaxLiveStreamsName            = "max-live-streams"
 )
 
 type TapConfig struct {
-	PodRegexStr            string           `yaml:"regex" default:".*"`
+	PodRegexStr       string   `yaml:"regex" default:".*"`
-	GuiPort                uint16           `yaml:"gui-port" default:"8899"`
+	GuiPort           uint16   `yaml:"gui-port" default:"8899"`
-	ProxyHost              string           `yaml:"proxy-host" default:"127.0.0.1"`
+	ProxyHost         string   `yaml:"proxy-host" default:"127.0.0.1"`
-	Namespaces             []string         `yaml:"namespaces"`
+	Namespaces        []string `yaml:"namespaces"`
-	AllNamespaces          bool             `yaml:"all-namespaces" default:"false"`
+	AllNamespaces     bool     `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string         `yaml:"regex-masking"`
+	IgnoredUserAgents []string `yaml:"ignored-user-agents"`
-	IgnoredUserAgents      []string         `yaml:"ignored-user-agents"`
+	EnableRedaction   bool     `yaml:"redact" default:"false"`
-	EnableRedaction        bool             `yaml:"redact" default:"false"`
+	RedactPatterns    struct {
-	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
+		RequestHeaders     []string `yaml:"request-headers"`
-	InsertionFilter        string           `yaml:"insertion-filter" default:""`
+		ResponseHeaders    []string `yaml:"response-headers"`
-	DryRun                 bool             `yaml:"dry-run" default:"false"`
+		RequestBody        []string `yaml:"request-body"`
-	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
+		ResponseBody       []string `yaml:"response-body"`
-	TapperResources        shared.Resources `yaml:"tapper-resources"`
+		RequestQueryParams []string `yaml:"request-query-params"`
-	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
+	} `yaml:"redact-patterns"`
-	Tls                    bool             `yaml:"tls" default:"false"`
+	HumanMaxEntriesDBSize string           `yaml:"max-entries-db-size" default:"200MB"`
-	Profiler               bool             `yaml:"profiler" default:"false"`
+	InsertionFilter       string           `yaml:"insertion-filter" default:""`
-	MaxLiveStreams         int              `yaml:"max-live-streams" default:"500"`
+	DryRun                bool             `yaml:"dry-run" default:"false"`
+	ApiServerResources    shared.Resources `yaml:"api-server-resources"`
+	TapperResources       shared.Resources `yaml:"tapper-resources"`
+	ServiceMesh           bool             `yaml:"service-mesh" default:"false"`
+	Tls                   bool             `yaml:"tls" default:"false"`
+	Profiler              bool             `yaml:"profiler" default:"false"`
+	MaxLiveStreams        int              `yaml:"max-live-streams" default:"500"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -71,9 +77,48 @@ func (config *TapConfig) GetInsertionFilter() string {
 			}
 		}
 	}
+
+	redactFilter := getRedactFilter(config)
+	if insertionFilter != "" && redactFilter != "" {
+		return fmt.Sprintf("(%s) and (%s)", insertionFilter, redactFilter)
+	} else if insertionFilter == "" && redactFilter != "" {
+		return redactFilter
+	}
+
 	return insertionFilter
 }
 
+func getRedactFilter(config *TapConfig) string {
+	if !config.EnableRedaction {
+		return ""
+	}
+
+	var redactValues []string
+	for _, requestHeader := range config.RedactPatterns.RequestHeaders {
+		redactValues = append(redactValues, fmt.Sprintf("request.headers['%s']", requestHeader))
+	}
+	for _, responseHeader := range config.RedactPatterns.ResponseHeaders {
+		redactValues = append(redactValues, fmt.Sprintf("response.headers['%s']", responseHeader))
+	}
+
+	for _, requestBody := range config.RedactPatterns.RequestBody {
+		redactValues = append(redactValues, fmt.Sprintf("request.postData.text.json()...%s", requestBody))
+	}
+	for _, responseBody := range config.RedactPatterns.ResponseBody {
+		redactValues = append(redactValues, fmt.Sprintf("response.content.text.json()...%s", responseBody))
+	}
+
+	for _, requestQueryParams := range config.RedactPatterns.RequestQueryParams {
+		redactValues = append(redactValues, fmt.Sprintf("request.queryString['%s']", requestQueryParams))
+	}
+
+	if len(redactValues) == 0 {
+		return ""
+	}
+
+	return fmt.Sprintf("redact(\"%s\")", strings.Join(redactValues, "\",\""))
+}
+
 func (config *TapConfig) Validate() error {
 	_, compileErr := regexp.Compile(config.PodRegexStr)
 	if compileErr != nil {

cli/go.mod

@@ -4,7 +4,6 @@ go 1.17
 
 require (
 	github.com/creasty/defaults v1.5.2
-	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/google/go-github/v37 v37.0.0
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0

cli/go.sum

@@ -145,8 +145,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
-github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
-github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=

cli/mizu/consts.go

@@ -14,8 +14,6 @@ var (
 	Platform                                   = ""
 )
 
-const DEVENVVAR = "MIZU_DISABLE_TELEMTRY"
-
 func GetMizuFolderPath() string {
 	home, homeDirErr := os.UserHomeDir()
 	if homeDirErr != nil {

cli/mizu/version/versionCheck.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"runtime"
 	"strings"
 	"time"
@@ -18,10 +17,6 @@ import (
 )
 
 func CheckNewerVersion(versionChan chan string) {
-	if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
-		versionChan <- ""
-		return
-	}
 	logger.Log.Debugf("Checking for newer version...")
 	start := time.Now()
 	client := github.NewClient(nil)

cli/telemetry/telemetry.go

@@ -1,97 +0,0 @@
-package telemetry
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/denisbrodbeck/machineid"
-	"github.com/up9inc/mizu/cli/apiserver"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/logger"
-)
-
-const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
-
-func ReportRun(cmd string, args interface{}) {
-	if !shouldRunTelemetry() {
-		logger.Log.Debug("not reporting telemetry")
-		return
-	}
-
-	argsBytes, _ := json.Marshal(args)
-	argsMap := map[string]interface{}{
-		"cmd":  cmd,
-		"args": string(argsBytes),
-	}
-
-	if err := sendTelemetry(argsMap); err != nil {
-		logger.Log.Debug(err)
-		return
-	}
-
-	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
-}
-
-func ReportTapTelemetry(apiProvider *apiserver.Provider, args interface{}, startTime time.Time) {
-	if !shouldRunTelemetry() {
-		logger.Log.Debug("not reporting telemetry")
-		return
-	}
-
-	generalStats, err := apiProvider.GetGeneralStats()
-	if err != nil {
-		logger.Log.Debugf("[ERROR] failed to get general stats from api server %v", err)
-		return
-	}
-	argsBytes, _ := json.Marshal(args)
-	argsMap := map[string]interface{}{
-		"cmd":                    "tap",
-		"args":                   string(argsBytes),
-		"executionTimeInSeconds": int(time.Since(startTime).Seconds()),
-		"apiCallsCount":          generalStats["EntriesCount"],
-		"trafficVolumeInGB":      generalStats["EntriesVolumeInGB"],
-	}
-
-	if err := sendTelemetry(argsMap); err != nil {
-		logger.Log.Debug(err)
-		return
-	}
-
-	logger.Log.Debug("successfully reported telemetry of tap command")
-}
-
-func shouldRunTelemetry() bool {
-	if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
-		return false
-	}
-	if !config.Config.Telemetry {
-		return false
-	}
-
-	return mizu.Branch == "main" || mizu.Branch == "develop"
-}
-
-func sendTelemetry(argsMap map[string]interface{}) error {
-	argsMap["component"] = "mizu_cli"
-	argsMap["buildTimestamp"] = mizu.BuildTimestamp
-	argsMap["branch"] = mizu.Branch
-	argsMap["version"] = mizu.Ver
-	argsMap["platform"] = mizu.Platform
-
-	if machineId, err := machineid.ProtectedID("mizu"); err == nil {
-		argsMap["machineId"] = machineId
-	}
-
-	jsonValue, _ := json.Marshal(argsMap)
-
-	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
-		return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp)
-	}
-
-	return nil
-}

performance_analysis/run_tapper_benchmark.sh

@@ -57,7 +57,7 @@ log "Writing output to $MIZU_BENCHMARK_OUTPUT_DIR"
 cd $MIZU_HOME || exit 1
 
 export HOST_MODE=0
-export SENSITIVE_DATA_FILTERING_OPTIONS='{"EnableRedaction": false}'
+export SENSITIVE_DATA_FILTERING_OPTIONS='{}'
 export MIZU_DEBUG_DISABLE_PCAP=false
 export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false
 export MIZU_DEBUG_DISABLE_TCP_STREAM=false

shared/kubernetes/mizuTapperSyncer.go

@@ -43,7 +43,6 @@ type TapperSyncerConfig struct {
 	TapperResources          shared.Resources
 	ImagePullPolicy          core.PullPolicy
 	LogLevel                 logging.Level
-	IgnoredUserAgents        []string
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
 	ServiceMesh              bool

shared/models.go

@@ -43,7 +43,6 @@ type MizuAgentConfig struct {
 	AgentDatabasePath      string        `json:"agentDatabasePath"`
 	ServiceMap             bool          `json:"serviceMap"`
 	OAS                    OASConfig     `json:"oas"`
-	Telemetry              bool          `json:"telemetry"`
 }
 
 type WebSocketMessageMetadata struct {

tap/api/api.go

@@ -2,7 +2,9 @@ package api
 
 import (
 	"bufio"
+	"fmt"
 	"net"
+	"strings"
 	"sync"
 	"time"
 
@@ -14,12 +16,29 @@ const UnknownNamespace = ""
 var UnknownIp = net.IP{0, 0, 0, 0}
 var UnknownPort uint16 = 0
 
+type ProtocolSummary struct {
+	Name         string `json:"name"`
+	Version      string `json:"version"`
+	Abbreviation string `json:"abbr"`
+}
+
+func (protocol *ProtocolSummary) ToString() string {
+	return fmt.Sprintf("%s?%s?%s", protocol.Name, protocol.Version, protocol.Abbreviation)
+}
+
+func GetProtocolSummary(inputString string) *ProtocolSummary {
+	splitted := strings.SplitN(inputString, "?", 3)
+	return &ProtocolSummary{
+		Name:         splitted[0],
+		Version:      splitted[1],
+		Abbreviation: splitted[2],
+	}
+}
+
 type Protocol struct {
-	Name            string   `json:"name"`
+	ProtocolSummary
 	LongName        string   `json:"longName"`
-	Abbreviation    string   `json:"abbr"`
 	Macro           string   `json:"macro"`
-	Version         string   `json:"version"`
 	BackgroundColor string   `json:"backgroundColor"`
 	ForegroundColor string   `json:"foregroundColor"`
 	FontSize        int8     `json:"fontSize"`
@@ -151,7 +170,7 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 
 type Entry struct {
 	Id           string                 `json:"id"`
-	ProtocolId   string                 `json:"protocol"`
+	Protocol     ProtocolSummary        `json:"protocol"`
 	Capture      Capture                `json:"capture"`
 	Source       *TCP                   `json:"src"`
 	Destination  *TCP                   `json:"dst"`

tap/api/options.go

@@ -1,7 +1,5 @@
 package api
 
 type TrafficFilteringOptions struct {
-	IgnoredUserAgents       []string
+	IgnoredUserAgents []string
-	PlainTextMaskingRegexes []*SerializableRegexp
-	EnableRedaction         bool
 }

tap/extensions/amqp/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/amqp/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect16/amqp/\* expect

tap/extensions/amqp/go.mod

@@ -4,16 +4,20 @@ go 1.17
 
 require (
 	github.com/stretchr/testify v1.7.0
+	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
 
+replace github.com/up9inc/mizu/logger v0.0.0 => ../../../logger
+
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api
 
 replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../../dbgctl

tap/extensions/amqp/go.sum

@@ -1,5 +1,7 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

tap/extensions/amqp/helpers.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
-	"time"
 
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
 
@@ -25,14 +25,14 @@ var connectionMethodMap = map[int]string{
 	61: "connection unblocked",
 }
 
-// var channelMethodMap = map[int]string{
+var channelMethodMap = map[int]string{
-// 	10: "channel open",
+	10: "channel open",
-// 	11: "channel open-ok",
+	11: "channel open-ok",
-// 	20: "channel flow",
+	20: "channel flow",
-// 	21: "channel flow-ok",
+	21: "channel flow-ok",
-// 	40: "channel close",
+	40: "channel close",
-// 	41: "channel close-ok",
+	41: "channel close-ok",
-// }
+}
 
 var exchangeMethodMap = map[int]string{
 	10: "exchange declare",
@@ -94,29 +94,41 @@ type AMQPWrapper struct {
 	Details interface{} `json:"details"`
 }
 
-func emitAMQP(event interface{}, _type string, method string, connectionInfo *api.ConnectionInfo, captureTime time.Time, captureSize int, emitter api.Emitter, capture api.Capture) {
+type emptyResponse struct {
-	request := &api.GenericMessage{
+}
-		IsRequest:   true,
+
-		CaptureTime: captureTime,
+const emptyMethod = "empty"
-		Payload: AMQPPayload{
+
-			Data: &AMQPWrapper{
+func getIdent(reader api.TcpReader, methodFrame *MethodFrame) (ident string) {
-				Method:  method,
+	tcpID := reader.GetTcpID()
-				Url:     "",
+	// To match methods to their Ok(s)
-				Details: event,
+	methodId := methodFrame.MethodId - methodFrame.MethodId%10
-			},
+
-		},
+	if reader.GetIsClient() {
+		ident = fmt.Sprintf(
+			"%s_%s_%s_%s_%d_%d_%d",
+			tcpID.SrcIP,
+			tcpID.DstIP,
+			tcpID.SrcPort,
+			tcpID.DstPort,
+			methodFrame.ChannelId,
+			methodFrame.ClassId,
+			methodId,
+		)
+	} else {
+		ident = fmt.Sprintf(
+			"%s_%s_%s_%s_%d_%d_%d",
+			tcpID.DstIP,
+			tcpID.SrcIP,
+			tcpID.DstPort,
+			tcpID.SrcPort,
+			methodFrame.ChannelId,
+			methodFrame.ClassId,
+			methodId,
+		)
 	}
-	item := &api.OutputChannelItem{
+
-		Protocol:       protocol,
+	return
-		Capture:        capture,
-		Timestamp:      captureTime.UnixNano() / int64(time.Millisecond),
-		ConnectionInfo: connectionInfo,
-		Pair: &api.RequestResponsePair{
-			Request:  *request,
-			Response: api.GenericMessage{},
-		},
-	}
-	emitter.Emit(item)
 }
 
 func representProperties(properties map[string]interface{}, rep []interface{}) ([]interface{}, string, string) {
@@ -460,6 +472,36 @@ func representQueueDeclare(event map[string]interface{}) []interface{} {
 	return rep
 }
 
+func representQueueDeclareOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `response.queue`,
+		},
+		{
+			Name:     "Message Count",
+			Value:    fmt.Sprintf("%g", event["messageCount"].(float64)),
+			Selector: `response.messageCount`,
+		},
+		{
+			Name:     "Consumer Count",
+			Value:    fmt.Sprintf("%g", event["consumerCount"].(float64)),
+			Selector: `response.consumerCount`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
 func representExchangeDeclare(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
@@ -571,7 +613,7 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 				x, _ := json.Marshal(value)
 				outcome = string(x)
 			default:
-				panic("Unknown data type for the server property!")
+				logger.Log.Info("Unknown data type for the server property!")
 			}
 			headers = append(headers, api.TableData{
 				Name:     name,
@@ -593,6 +635,65 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 	return rep
 }
 
+func representConnectionStartOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Mechanism",
+			Value:    event["mechanism"].(string),
+			Selector: `response.mechanism`,
+		},
+		{
+			Name:     "Mechanism",
+			Value:    event["mechanism"].(string),
+			Selector: `response.response`,
+		},
+		{
+			Name:     "Locale",
+			Value:    event["locale"].(string),
+			Selector: `response.locale`,
+		},
+	})
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	if event["clientProperties"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["clientProperties"].(map[string]interface{}) {
+			var outcome string
+			switch v := value.(type) {
+			case string:
+				outcome = v
+			case map[string]interface{}:
+				x, _ := json.Marshal(value)
+				outcome = string(x)
+			default:
+				logger.Log.Info("Unknown data type for the client property!")
+			}
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    outcome,
+				Selector: fmt.Sprintf(`response.clientProperties["%s"]`, name),
+			})
+		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Client Properties",
+			Data:  string(headersMarshaled),
+		})
+	}
+
+	return rep
+}
+
 func representConnectionClose(event map[string]interface{}) []interface{} {
 	replyCode := ""
 
@@ -750,3 +851,122 @@ func representBasicConsume(event map[string]interface{}) []interface{} {
 
 	return rep
 }
+
+func representBasicConsumeOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `response.consumerTag`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representConnectionOpen(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Virtual Host",
+			Value:    event["virtualHost"].(string),
+			Selector: `request.virtualHost`,
+		},
+	})
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representConnectionTune(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Channel Max",
+			Value:    fmt.Sprintf("%g", event["channelMax"].(float64)),
+			Selector: `request.channelMax`,
+		},
+		{
+			Name:     "Frame Max",
+			Value:    fmt.Sprintf("%g", event["frameMax"].(float64)),
+			Selector: `request.frameMax`,
+		},
+		{
+			Name:     "Heartbeat",
+			Value:    fmt.Sprintf("%g", event["heartbeat"].(float64)),
+			Selector: `request.heartbeat`,
+		},
+	})
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representBasicCancel(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `response.consumerTag`,
+		},
+		{
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representBasicCancelOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `response.consumerTag`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representEmpty(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	return rep
+}

tap/extensions/amqp/main.go

@@ -13,11 +13,13 @@ import (
 )
 
 var protocol = api.Protocol{
-	Name:            "amqp",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "amqp",
+		Version:      "0-9-1",
+		Abbreviation: "AMQP",
+	},
 	LongName:        "Advanced Message Queuing Protocol 0-9-1",
-	Abbreviation:    "AMQP",
 	Macro:           "amqp",
-	Version:         "0-9-1",
 	BackgroundColor: "#ff6600",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -27,7 +29,7 @@ var protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
+	protocol.ToString(): &protocol,
 }
 
 type dissecting string
@@ -44,22 +46,12 @@ func (d dissecting) Ping() {
 	log.Printf("pong %s", protocol.Name)
 }
 
-const amqpRequest string = "amqp_request"
-
 func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.TrafficFilteringOptions) error {
 	r := AmqpReader{b}
 
 	var remaining int
 	var header *HeaderFrame
 
-	connectionInfo := &api.ConnectionInfo{
-		ClientIP:   reader.GetTcpID().SrcIP,
-		ClientPort: reader.GetTcpID().SrcPort,
-		ServerIP:   reader.GetTcpID().DstIP,
-		ServerPort: reader.GetTcpID().DstPort,
-		IsOutgoing: true,
-	}
-
 	eventBasicPublish := &BasicPublish{
 		Exchange:   "",
 		RoutingKey: "",
@@ -81,6 +73,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 
 	var lastMethodFrameMessage Message
 
+	var ident string
+	isClient := reader.GetIsClient()
+	reqResMatcher := reader.GetReqResMatcher().(*requestResponseMatcher)
+
 	for {
 		frameVal, err := r.readFrame()
 		if err == io.EOF {
@@ -119,16 +115,22 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 			switch lastMethodFrameMessage.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Body = f.Body
-				emitAMQP(*eventBasicPublish, amqpRequest, basicMethodMap[40], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[40], *eventBasicPublish, reader)
+				reqResMatcher.emitEvent(!isClient, ident, emptyMethod, &emptyResponse{}, reader)
+
 			case *BasicDeliver:
 				eventBasicDeliver.Body = f.Body
-				emitAMQP(*eventBasicDeliver, amqpRequest, basicMethodMap[60], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(!isClient, ident, basicMethodMap[60], *eventBasicDeliver, reader)
+				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
 			}
 
 		case *MethodFrame:
 			reader.GetParent().SetProtocol(&protocol)
 
 			lastMethodFrameMessage = f.Method
+
+			ident = getIdent(reader, f)
+
 			switch m := f.Method.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Exchange = m.Exchange
@@ -144,7 +146,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				emitAMQP(*eventQueueBind, amqpRequest, queueMethodMap[20], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[20], *eventQueueBind, reader)
+
+			case *QueueBindOk:
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[21], m, reader)
 
 			case *BasicConsume:
 				eventBasicConsume := &BasicConsume{
@@ -156,7 +161,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:      m.NoWait,
 					Arguments:   m.Arguments,
 				}
-				emitAMQP(*eventBasicConsume, amqpRequest, basicMethodMap[20], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[20], *eventBasicConsume, reader)
+
+			case *BasicConsumeOk:
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[21], m, reader)
 
 			case *BasicDeliver:
 				eventBasicDeliver.ConsumerTag = m.ConsumerTag
@@ -175,7 +183,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				emitAMQP(*eventQueueDeclare, amqpRequest, queueMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[10], *eventQueueDeclare, reader)
+
+			case *QueueDeclareOk:
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[11], m, reader)
 
 			case *ExchangeDeclare:
 				eventExchangeDeclare := &ExchangeDeclare{
@@ -188,17 +199,19 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				emitAMQP(*eventExchangeDeclare, amqpRequest, exchangeMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, exchangeMethodMap[10], *eventExchangeDeclare, reader)
+
+			case *ExchangeDeclareOk:
+				reqResMatcher.emitEvent(isClient, ident, exchangeMethodMap[11], m, reader)
 
 			case *ConnectionStart:
-				eventConnectionStart := &ConnectionStart{
+				// In our tests, *ConnectionStart does not result in *ConnectionStartOk
-					VersionMajor:     m.VersionMajor,
+				reqResMatcher.emitEvent(!isClient, ident, connectionMethodMap[10], m, reader)
-					VersionMinor:     m.VersionMinor,
+				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
-					ServerProperties: m.ServerProperties,
+
-					Mechanisms:       m.Mechanisms,
+			case *ConnectionStartOk:
-					Locales:          m.Locales,
+				// In our tests, *ConnectionStart does not result in *ConnectionStartOk
-				}
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[11], m, reader)
-				emitAMQP(*eventConnectionStart, amqpRequest, connectionMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 
 			case *ConnectionClose:
 				eventConnectionClose := &ConnectionClose{
@@ -207,7 +220,40 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					ClassId:   m.ClassId,
 					MethodId:  m.MethodId,
 				}
-				emitAMQP(*eventConnectionClose, amqpRequest, connectionMethodMap[50], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[50], *eventConnectionClose, reader)
+
+			case *ConnectionCloseOk:
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[51], m, reader)
+
+			case *connectionOpen:
+				eventConnectionOpen := &connectionOpen{
+					VirtualHost: m.VirtualHost,
+				}
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[40], *eventConnectionOpen, reader)
+
+			case *connectionOpenOk:
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[41], m, reader)
+
+			case *channelOpen:
+				reqResMatcher.emitEvent(isClient, ident, channelMethodMap[10], m, reader)
+
+			case *channelOpenOk:
+				reqResMatcher.emitEvent(isClient, ident, channelMethodMap[11], m, reader)
+
+			case *connectionTune:
+				// In our tests, *connectionTune does not result in *connectionTuneOk
+				reqResMatcher.emitEvent(!isClient, ident, connectionMethodMap[30], m, reader)
+				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
+
+			case *connectionTuneOk:
+				// In our tests, *connectionTune does not result in *connectionTuneOk
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[31], m, reader)
+
+			case *basicCancel:
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[30], m, reader)
+
+			case *basicCancelOk:
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[31], m, reader)
 			}
 
 		default:
@@ -218,12 +264,20 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 
 func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
+	response := item.Pair.Response.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
+	resDetails := response["details"].(map[string]interface{})
+
+	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
 
 	reqDetails["method"] = request["method"]
+	resDetails["method"] = response["method"]
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Protocol: protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -234,13 +288,15 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 			IP:   item.ConnectionInfo.ServerIP,
 			Port: item.ConnectionInfo.ServerPort,
 		},
-		Namespace:   namespace,
+		Namespace:    namespace,
-		Outgoing:    item.ConnectionInfo.IsOutgoing,
+		Outgoing:     item.ConnectionInfo.IsOutgoing,
-		Request:     reqDetails,
+		Request:      reqDetails,
-		RequestSize: item.Pair.Request.CaptureSize,
+		Response:     resDetails,
-		Timestamp:   item.Timestamp,
+		RequestSize:  item.Pair.Request.CaptureSize,
-		StartTime:   item.Pair.Request.CaptureTime,
+		ResponseSize: item.Pair.Response.CaptureSize,
-		ElapsedTime: 0,
+		Timestamp:    item.Timestamp,
+		StartTime:    item.Pair.Request.CaptureTime,
+		ElapsedTime:  elapsedTime,
 	}
 
 }
@@ -281,11 +337,26 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	case basicMethodMap[20]:
 		summary = entry.Request["queue"].(string)
 		summaryQuery = fmt.Sprintf(`request.queue == "%s"`, summary)
+	case connectionMethodMap[40]:
+		summary = entry.Request["virtualHost"].(string)
+		summaryQuery = fmt.Sprintf(`request.virtualHost == "%s"`, summary)
+	case connectionMethodMap[30]:
+		summary = fmt.Sprintf("%g", entry.Request["channelMax"].(float64))
+		summaryQuery = fmt.Sprintf(`request.channelMax == "%s"`, summary)
+	case connectionMethodMap[31]:
+		summary = fmt.Sprintf("%g", entry.Request["channelMax"].(float64))
+		summaryQuery = fmt.Sprintf(`request.channelMax == "%s"`, summary)
+	case basicMethodMap[30]:
+		summary = entry.Request["consumerTag"].(string)
+		summaryQuery = fmt.Sprintf(`request.consumerTag == "%s"`, summary)
+	case basicMethodMap[31]:
+		summary = entry.Request["consumerTag"].(string)
+		summaryQuery = fmt.Sprintf(`request.consumerTag == "%s"`, summary)
 	}
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -304,6 +375,8 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, err error) {
 	representation := make(map[string]interface{})
 	var repRequest []interface{}
+	var repResponse []interface{}
+
 	switch request["method"].(string) {
 	case basicMethodMap[40]:
 		repRequest = representBasicPublish(request)
@@ -321,20 +394,56 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 		repRequest = representQueueBind(request)
 	case basicMethodMap[20]:
 		repRequest = representBasicConsume(request)
+	case connectionMethodMap[40]:
+		repRequest = representConnectionOpen(request)
+	case channelMethodMap[10]:
+		repRequest = representEmpty(request)
+	case connectionMethodMap[30]:
+		repRequest = representConnectionTune(request)
+	case basicMethodMap[30]:
+		repRequest = representBasicCancel(request)
 	}
+
+	switch response["method"].(string) {
+	case queueMethodMap[11]:
+		repResponse = representQueueDeclareOk(response)
+	case exchangeMethodMap[11]:
+		repResponse = representEmpty(response)
+	case connectionMethodMap[11]:
+		repResponse = representConnectionStartOk(response)
+	case connectionMethodMap[51]:
+		repResponse = representEmpty(response)
+	case basicMethodMap[21]:
+		repResponse = representBasicConsumeOk(response)
+	case queueMethodMap[21]:
+		repResponse = representEmpty(response)
+	case connectionMethodMap[41]:
+		repResponse = representEmpty(response)
+	case channelMethodMap[11]:
+		repResponse = representEmpty(request)
+	case connectionMethodMap[31]:
+		repResponse = representConnectionTune(request)
+	case basicMethodMap[31]:
+		repResponse = representBasicCancelOk(request)
+	case emptyMethod:
+		repResponse = representEmpty(response)
+	}
+
 	representation["request"] = repRequest
+	representation["response"] = repResponse
 	object, err = json.Marshal(representation)
+
 	return
 }
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`amqp`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
+		`amqp`: fmt.Sprintf(`protocol.name == "%s"`, protocol.Name),
 	}
 }
 
 func (d dissecting) NewResponseRequestMatcher() api.RequestResponseMatcher {
-	return nil
+	return createResponseRequestMatcher()
 }
 
 var Dissector dissecting

tap/extensions/amqp/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"amqp": `protocol == "amqp/0-9-1/AMQP"`,
+		"amqp": `protocol.name == "amqp"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/amqp/matcher.go

@@ -0,0 +1,113 @@
+package amqp
+
+import (
+	"sync"
+	"time"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+// Key is {client_addr}_{client_port}_{dest_addr}_{dest_port}_{channel_id}_{class_id}_{method_id}
+type requestResponseMatcher struct {
+	openMessagesMap *sync.Map
+}
+
+func createResponseRequestMatcher() api.RequestResponseMatcher {
+	return &requestResponseMatcher{openMessagesMap: &sync.Map{}}
+}
+
+func (matcher *requestResponseMatcher) GetMap() *sync.Map {
+	return matcher.openMessagesMap
+}
+
+func (matcher *requestResponseMatcher) SetMaxTry(value int) {
+}
+
+func (matcher *requestResponseMatcher) emitEvent(isRequest bool, ident string, method string, event interface{}, reader api.TcpReader) {
+	reader.GetParent().SetProtocol(&protocol)
+
+	var item *api.OutputChannelItem
+	if isRequest {
+		item = matcher.registerRequest(ident, method, event, reader.GetCaptureTime(), reader.GetReadProgress().Current())
+	} else {
+		item = matcher.registerResponse(ident, method, event, reader.GetCaptureTime(), reader.GetReadProgress().Current())
+	}
+
+	if item != nil {
+		item.ConnectionInfo = &api.ConnectionInfo{
+			ClientIP:   reader.GetTcpID().SrcIP,
+			ClientPort: reader.GetTcpID().SrcPort,
+			ServerIP:   reader.GetTcpID().DstIP,
+			ServerPort: reader.GetTcpID().DstPort,
+			IsOutgoing: true,
+		}
+		item.Capture = reader.GetParent().GetOrigin()
+		reader.GetEmitter().Emit(item)
+	}
+}
+
+func (matcher *requestResponseMatcher) registerRequest(ident string, method string, request interface{}, captureTime time.Time, captureSize int) *api.OutputChannelItem {
+	requestAMQPMessage := api.GenericMessage{
+		IsRequest:   true,
+		CaptureTime: captureTime,
+		CaptureSize: captureSize,
+		Payload: AMQPPayload{
+			Data: &AMQPWrapper{
+				Method:  method,
+				Url:     "",
+				Details: request,
+			},
+		},
+	}
+
+	if response, found := matcher.openMessagesMap.LoadAndDelete(ident); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		responseAMQPMessage := response.(*api.GenericMessage)
+		if responseAMQPMessage.IsRequest {
+			return nil
+		}
+		return matcher.preparePair(&requestAMQPMessage, responseAMQPMessage)
+	}
+
+	matcher.openMessagesMap.Store(ident, &requestAMQPMessage)
+	return nil
+}
+
+func (matcher *requestResponseMatcher) registerResponse(ident string, method string, response interface{}, captureTime time.Time, captureSize int) *api.OutputChannelItem {
+	responseAMQPMessage := api.GenericMessage{
+		IsRequest:   false,
+		CaptureTime: captureTime,
+		CaptureSize: captureSize,
+		Payload: AMQPPayload{
+			Data: &AMQPWrapper{
+				Method:  method,
+				Url:     "",
+				Details: response,
+			},
+		},
+	}
+
+	if request, found := matcher.openMessagesMap.LoadAndDelete(ident); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		requestAMQPMessage := request.(*api.GenericMessage)
+		if !requestAMQPMessage.IsRequest {
+			return nil
+		}
+		return matcher.preparePair(requestAMQPMessage, &responseAMQPMessage)
+	}
+
+	matcher.openMessagesMap.Store(ident, &responseAMQPMessage)
+	return nil
+}
+
+func (matcher *requestResponseMatcher) preparePair(requestAMQPMessage *api.GenericMessage, responseAMQPMessage *api.GenericMessage) *api.OutputChannelItem {
+	return &api.OutputChannelItem{
+		Protocol:       protocol,
+		Timestamp:      requestAMQPMessage.CaptureTime.UnixNano() / int64(time.Millisecond),
+		ConnectionInfo: nil,
+		Pair: &api.RequestResponsePair{
+			Request:  *requestAMQPMessage,
+			Response: *responseAMQPMessage,
+		},
+	}
+}

tap/extensions/amqp/spec091.go

@@ -81,10 +81,10 @@ func (msg *ConnectionStart) read(r io.Reader) (err error) {
 }
 
 type ConnectionStartOk struct {
-	ClientProperties Table
+	ClientProperties Table  `json:"clientProperties"`
-	Mechanism        string
+	Mechanism        string `json:"mechanism"`
-	Response         string
+	Response         string `json:"response"`
-	Locale           string
+	Locale           string `json:"locale"`
 }
 
 func (msg *ConnectionStartOk) read(r io.Reader) (err error) {
@@ -135,9 +135,9 @@ func (msg *connectionSecureOk) read(r io.Reader) (err error) {
 }
 
 type connectionTune struct {
-	ChannelMax uint16
+	ChannelMax uint16 `json:"channelMax"`
-	FrameMax   uint32
+	FrameMax   uint32 `json:"frameMax"`
-	Heartbeat  uint16
+	Heartbeat  uint16 `json:"heartbeat"`
 }
 
 func (msg *connectionTune) read(r io.Reader) (err error) {
@@ -181,7 +181,7 @@ func (msg *connectionTuneOk) read(r io.Reader) (err error) {
 }
 
 type connectionOpen struct {
-	VirtualHost string
+	VirtualHost string `json:"virtualHost"`
 	reserved1   string
 	reserved2   bool
 }
@@ -580,9 +580,9 @@ func (msg *QueueDeclare) read(r io.Reader) (err error) {
 }
 
 type QueueDeclareOk struct {
-	Queue         string
+	Queue         string `json:"queue"`
-	MessageCount  uint32
+	MessageCount  uint32 `json:"messageCount"`
-	ConsumerCount uint32
+	ConsumerCount uint32 `json:"consumerCount"`
 }
 
 func (msg *QueueDeclareOk) read(r io.Reader) (err error) {
@@ -840,7 +840,7 @@ func (msg *BasicConsume) read(r io.Reader) (err error) {
 }
 
 type BasicConsumeOk struct {
-	ConsumerTag string
+	ConsumerTag string `json:"consumerTag"`
 }
 
 func (msg *BasicConsumeOk) read(r io.Reader) (err error) {
@@ -853,8 +853,8 @@ func (msg *BasicConsumeOk) read(r io.Reader) (err error) {
 }
 
 type basicCancel struct {
-	ConsumerTag string
+	ConsumerTag string `json:"consumerTag"`
-	NoWait      bool
+	NoWait      bool   `json:"noWait"`
 }
 
 func (msg *basicCancel) read(r io.Reader) (err error) {
@@ -873,7 +873,7 @@ func (msg *basicCancel) read(r io.Reader) (err error) {
 }
 
 type basicCancelOk struct {
-	ConsumerTag string
+	ConsumerTag string `json:"consumerTag"`
 }
 
 func (msg *basicCancelOk) read(r io.Reader) (err error) {

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect16/http/\* expect

tap/extensions/http/handlers.go

@@ -18,10 +18,6 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
 		return
 	}
 
-	if options.EnableRedaction {
-		FilterSensitiveData(item, options)
-	}
-
 	replaceForwardedFor(item)
 
 	emitter.Emit(item)

tap/extensions/http/helpers.go

@@ -6,13 +6,16 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/up9inc/mizu/tap/api"
 )
 
 func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) {
 	newMap = make(map[string]interface{})
-	for _, item := range mapSlice {
+
+	mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice)
+	for _, item := range mergedMapSlice {
 		h := item.(map[string]interface{})
 		newMap[h["name"].(string)] = h["value"]
 	}
@@ -20,6 +23,28 @@ func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}
 	return
 }
 
+func mapSliceRebuildAsMergedMap(mapSlice []interface{}) (newMap map[string]interface{}) {
+	newMap = make(map[string]interface{})
+
+	mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice)
+	for _, item := range mergedMapSlice {
+		h := item.(map[string]interface{})
+
+		if valuesInterface, ok := h["value"].([]interface{}); ok {
+			var values []string
+			for _, valueInterface := range valuesInterface {
+				values = append(values, valueInterface.(string))
+			}
+
+			newMap[h["name"].(string)] = strings.Join(values, ",")
+		} else {
+			newMap[h["name"].(string)] = h["value"]
+		}
+	}
+
+	return
+}
+
 func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{}) {
 	newMapSlice = make([]interface{}, 0)
 	valuesMap := make(map[string][]interface{})
@@ -47,6 +72,24 @@ func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{
 	return
 }
 
+func representMapAsTable(mapToTable map[string]interface{}, selectorPrefix string) (representation string) {
+	var table []api.TableData
+
+	keys := make([]string, 0, len(mapToTable))
+	for k := range mapToTable {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		table = append(table, createTableForKey(key, mapToTable[key], selectorPrefix)...)
+	}
+
+	obj, _ := json.Marshal(table)
+	representation = string(obj)
+	return
+}
+
 func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
 	for _, item := range mapSlice {
@@ -54,34 +97,7 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re
 		key := h["name"].(string)
 		value := h["value"]
 
-		var reflectKind reflect.Kind
+		table = append(table, createTableForKey(key, value, selectorPrefix)...)
-		reflectType := reflect.TypeOf(value)
-		if reflectType == nil {
-			reflectKind = reflect.Interface
-		} else {
-			reflectKind = reflect.TypeOf(value).Kind()
-		}
-
-		switch reflectKind {
-		case reflect.Slice:
-			fallthrough
-		case reflect.Array:
-			for i, el := range value.([]interface{}) {
-				selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i)
-				table = append(table, api.TableData{
-					Name:     fmt.Sprintf("%s [%d]", key, i),
-					Value:    el,
-					Selector: selector,
-				})
-			}
-		default:
-			selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
-			table = append(table, api.TableData{
-				Name:     key,
-				Value:    value,
-				Selector: selector,
-			})
-		}
 	}
 
 	obj, _ := json.Marshal(table)
@@ -89,6 +105,41 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re
 	return
 }
 
+func createTableForKey(key string, value interface{}, selectorPrefix string) []api.TableData {
+	var table []api.TableData
+
+	var reflectKind reflect.Kind
+	reflectType := reflect.TypeOf(value)
+	if reflectType == nil {
+		reflectKind = reflect.Interface
+	} else {
+		reflectKind = reflect.TypeOf(value).Kind()
+	}
+
+	switch reflectKind {
+	case reflect.Slice:
+		fallthrough
+	case reflect.Array:
+		for i, el := range value.([]interface{}) {
+			selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i)
+			table = append(table, api.TableData{
+				Name:     fmt.Sprintf("%s [%d]", key, i),
+				Value:    el,
+				Selector: selector,
+			})
+		}
+	default:
+		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
+		table = append(table, api.TableData{
+			Name:     key,
+			Value:    value,
+			Selector: selector,
+		})
+	}
+
+	return table
+}
+
 func representSliceAsTable(slice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
 	for i, item := range slice {

tap/extensions/http/main.go

@@ -15,11 +15,13 @@ import (
 )
 
 var http10protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.0",
+		Abbreviation: "HTTP",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.0",
-	Abbreviation:    "HTTP",
 	Macro:           "http",
-	Version:         "1.0",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -29,11 +31,13 @@ var http10protocol = api.Protocol{
 }
 
 var http11protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.1",
+		Abbreviation: "HTTP",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
-	Abbreviation:    "HTTP",
 	Macro:           "http",
-	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -43,11 +47,13 @@ var http11protocol = api.Protocol{
 }
 
 var http2Protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "2.0",
+		Abbreviation: "HTTP/2",
+	},
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2)",
-	Abbreviation:    "HTTP/2",
 	Macro:           "http2",
-	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -57,11 +63,13 @@ var http2Protocol = api.Protocol{
 }
 
 var grpcProtocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "2.0",
+		Abbreviation: "gRPC",
+	},
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ gRPC over HTTP/2 ]",
-	Abbreviation:    "gRPC",
 	Macro:           "grpc",
-	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -71,11 +79,13 @@ var grpcProtocol = api.Protocol{
 }
 
 var graphQL1Protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.1",
+		Abbreviation: "GQL",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1 [ GraphQL over HTTP/1.1 ]",
-	Abbreviation:    "GQL",
 	Macro:           "gql",
-	Version:         "1.1",
 	BackgroundColor: "#e10098",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -85,11 +95,13 @@ var graphQL1Protocol = api.Protocol{
 }
 
 var graphQL2Protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "2.0",
+		Abbreviation: "GQL",
+	},
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ GraphQL over HTTP/2 ]",
-	Abbreviation:    "GQL",
 	Macro:           "gql",
-	Version:         "2.0",
 	BackgroundColor: "#e10098",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -99,12 +111,12 @@ var graphQL2Protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation):       &http10protocol,
+	http10protocol.ToString():   &http10protocol,
-	fmt.Sprintf("%s/%s/%s", http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation):       &http11protocol,
+	http11protocol.ToString():   &http11protocol,
-	fmt.Sprintf("%s/%s/%s", http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation):          &http2Protocol,
+	http2Protocol.ToString():    &http2Protocol,
-	fmt.Sprintf("%s/%s/%s", grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation):             &grpcProtocol,
+	grpcProtocol.ToString():     &grpcProtocol,
-	fmt.Sprintf("%s/%s/%s", graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation): &graphQL1Protocol,
+	graphQL1Protocol.ToString(): &graphQL1Protocol,
-	fmt.Sprintf("%s/%s/%s", graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation): &graphQL2Protocol,
+	graphQL2Protocol.ToString(): &graphQL2Protocol,
 }
 
 const (
@@ -274,19 +286,13 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	reqDetails["pathSegments"] = strings.Split(path, "/")[1:]
 
 	// Rearrange the maps for the querying
-	reqDetails["_headers"] = reqDetails["headers"]
+	reqDetails["headers"] = mapSliceRebuildAsMergedMap(reqDetails["headers"].([]interface{}))
-	reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{}))
+	resDetails["headers"] = mapSliceRebuildAsMergedMap(resDetails["headers"].([]interface{}))
-	resDetails["_headers"] = resDetails["headers"]
-	resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{}))
 
-	reqDetails["_cookies"] = reqDetails["cookies"]
+	reqDetails["cookies"] = mapSliceRebuildAsMergedMap(reqDetails["cookies"].([]interface{}))
-	reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{}))
+	resDetails["cookies"] = mapSliceRebuildAsMergedMap(resDetails["cookies"].([]interface{}))
-	resDetails["_cookies"] = resDetails["cookies"]
-	resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{}))
 
-	reqDetails["_queryString"] = reqDetails["queryString"]
+	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["queryString"].([]interface{}))
-	reqDetails["_queryStringMerged"] = mapSliceMergeRepeatedKeys(reqDetails["_queryString"].([]interface{}))
-	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryStringMerged"].([]interface{}))
 
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
 	if elapsedTime < 0 {
@@ -294,8 +300,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", item.Protocol.Name, item.Protocol.Version, item.Protocol.Abbreviation),
+		Protocol: item.Protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -328,7 +334,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -385,19 +391,19 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Headers",
-		Data:  representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`),
+		Data:  representMapAsTable(request["headers"].(map[string]interface{}), `request.headers`),
 	})
 
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Cookies",
-		Data:  representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`),
+		Data:  representMapAsTable(request["cookies"].(map[string]interface{}), `request.cookies`),
 	})
 
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Query String",
-		Data:  representMapSliceAsTable(request["_queryStringMerged"].([]interface{}), `request.queryString`),
+		Data:  representMapAsTable(request["queryString"].(map[string]interface{}), `request.queryString`),
 	})
 
 	postData, _ := request["postData"].(map[string]interface{})
@@ -473,13 +479,13 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 	repResponse = append(repResponse, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Headers",
-		Data:  representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`),
+		Data:  representMapAsTable(response["headers"].(map[string]interface{}), `response.headers`),
 	})
 
 	repResponse = append(repResponse, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Cookies",
-		Data:  representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`),
+		Data:  representMapAsTable(response["cookies"].(map[string]interface{}), `response.cookies`),
 	})
 
 	content, _ := response["content"].(map[string]interface{})
@@ -515,10 +521,10 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`http`:  fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation, http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation),
+		`http`:  fmt.Sprintf(`protocol.abbr == "%s"`, http11protocol.Abbreviation),
-		`http2`: fmt.Sprintf(`protocol == "%s/%s/%s"`, http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation),
+		`http2`: fmt.Sprintf(`protocol.abbr == "%s"`, http2Protocol.Abbreviation),
-		`grpc`:  fmt.Sprintf(`protocol == "%s/%s/%s"`, grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation),
+		`grpc`:  fmt.Sprintf(`protocol.abbr == "%s"`, grpcProtocol.Abbreviation),
-		`gql`:   fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation, graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation),
+		`gql`:   fmt.Sprintf(`protocol.abbr == "%s"`, graphQL1Protocol.Abbreviation),
 	}
 }
 

tap/extensions/http/main_test.go

@@ -44,10 +44,10 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"http":  `protocol == "http/1.0/HTTP" or protocol == "http/1.1/HTTP"`,
+		"http":  `protocol.abbr == "HTTP"`,
-		"http2": `protocol == "http/2.0/HTTP/2"`,
+		"http2": `protocol.abbr == "HTTP/2"`,
-		"grpc":  `protocol == "http/2.0/gRPC"`,
+		"grpc":  `protocol.abbr == "gRPC"`,
-		"gql":   `protocol == "http/1.1/GQL" or protocol == "http/2.0/GQL"`,
+		"gql":   `protocol.abbr == "GQL"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/http/sensitive_data_cleaner.go

@@ -1,30 +1,14 @@
 package http
 
 import (
-	"bytes"
-	"encoding/json"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 
-	"github.com/beevik/etree"
 	"github.com/up9inc/mizu/tap/api"
 )
 
-const maskedFieldPlaceholderValue = "[REDACTED]"
 const userAgent = "user-agent"
 
-//these values MUST be all lower case and contain no `-` or `_` characters
-var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
-	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
-	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
-	"zip", "zipcode", "address", "country", "firstname", "lastname",
-	"middlename", "fname", "lname", "birthdate"}
-
 func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool {
 	if item.Protocol.Name != "http" {
 		return false
@@ -48,192 +32,3 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 
 	return false
 }
-
-func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
-	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
-	response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)
-
-	filterHeaders(&request.Header)
-	filterHeaders(&response.Header)
-	filterUrl(request.URL)
-	filterRequestBody(request, options)
-	filterResponseBody(response, options)
-}
-
-func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) {
-	contenType := getContentTypeHeaderValue(request.Header)
-	body, err := ioutil.ReadAll(request.Body)
-	if err != nil {
-		return
-	}
-	filteredBody, err := filterHttpBody(body, contenType, options)
-	if err == nil {
-		request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
-	} else {
-		request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	}
-}
-
-func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) {
-	contentType := getContentTypeHeaderValue(response.Header)
-	body, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		return
-	}
-	filteredBody, err := filterHttpBody(body, contentType, options)
-	if err == nil {
-		response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
-	} else {
-		response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	}
-}
-
-func filterHeaders(headers *http.Header) {
-	for key := range *headers {
-		if strings.ToLower(key) == userAgent {
-			continue
-		}
-
-		if strings.ToLower(key) == "cookie" {
-			headers.Del(key)
-		} else if isFieldNameSensitive(key) {
-			headers.Set(key, maskedFieldPlaceholderValue)
-		}
-	}
-}
-
-func getContentTypeHeaderValue(headers http.Header) string {
-	for key := range headers {
-		if strings.ToLower(key) == "content-type" {
-			return headers.Get(key)
-		}
-	}
-	return ""
-}
-
-func isFieldNameSensitive(fieldName string) bool {
-	if fieldName == ":authority" {
-		return false
-	}
-
-	name := strings.ToLower(fieldName)
-	name = strings.ReplaceAll(name, "_", "")
-	name = strings.ReplaceAll(name, "-", "")
-	name = strings.ReplaceAll(name, " ", "")
-
-	for _, sensitiveField := range personallyIdentifiableDataFields {
-		if strings.Contains(name, sensitiveField) {
-			return true
-		}
-	}
-
-	return false
-}
-
-func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) {
-	mimeType := strings.Split(contentType, ";")[0]
-	switch strings.ToLower(mimeType) {
-	case "application/json":
-		return filterJsonBody(bytes)
-	case "text/html":
-		fallthrough
-	case "application/xhtml+xml":
-		fallthrough
-	case "text/xml":
-		fallthrough
-	case "application/xml":
-		return filterXmlEtree(bytes)
-	case "text/plain":
-		if options != nil && options.PlainTextMaskingRegexes != nil {
-			return filterPlainText(bytes, options), nil
-		}
-	}
-	return bytes, nil
-}
-
-func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte {
-	for _, regex := range options.PlainTextMaskingRegexes {
-		bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
-	}
-	return bytes
-}
-
-func filterXmlEtree(bytes []byte) ([]byte, error) {
-	if !IsValidXML(bytes) {
-		return nil, errors.New("Invalid XML")
-	}
-	xmlDoc := etree.NewDocument()
-	err := xmlDoc.ReadFromBytes(bytes)
-	if err != nil {
-		return nil, err
-	} else {
-		filterXmlElement(xmlDoc.Root())
-	}
-	return xmlDoc.WriteToBytes()
-}
-
-func IsValidXML(data []byte) bool {
-	return xml.Unmarshal(data, new(interface{})) == nil
-}
-
-func filterXmlElement(element *etree.Element) {
-	for i, attribute := range element.Attr {
-		if isFieldNameSensitive(attribute.Key) {
-			element.Attr[i].Value = maskedFieldPlaceholderValue
-		}
-	}
-	if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
-		if isFieldNameSensitive(element.Tag) {
-			element.SetText(maskedFieldPlaceholderValue)
-		}
-	} else {
-		for _, element := range element.ChildElements() {
-			filterXmlElement(element)
-		}
-	}
-}
-
-func filterJsonBody(bytes []byte) ([]byte, error) {
-	var bodyJsonMap map[string]interface{}
-	err := json.Unmarshal(bytes, &bodyJsonMap)
-	if err != nil {
-		return nil, err
-	}
-	filterJsonMap(bodyJsonMap)
-	return json.Marshal(bodyJsonMap)
-}
-
-func filterJsonMap(jsonMap map[string]interface{}) {
-	for key, value := range jsonMap {
-		// Do not replace nil values with maskedFieldPlaceholderValue
-		if value == nil {
-			continue
-		}
-
-		nestedMap, isNested := value.(map[string]interface{})
-		if isNested {
-			filterJsonMap(nestedMap)
-		} else {
-			if isFieldNameSensitive(key) {
-				jsonMap[key] = maskedFieldPlaceholderValue
-			}
-		}
-	}
-}
-
-func filterUrl(url *url.URL) {
-	if len(url.RawQuery) > 0 {
-		newQueryArgs := make([]string, 0)
-		for urlQueryParamName, urlQueryParamValues := range url.Query() {
-			newValues := urlQueryParamValues
-			if isFieldNameSensitive(urlQueryParamName) {
-				newValues = []string{maskedFieldPlaceholderValue}
-			}
-			for _, paramValue := range newValues {
-				newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
-			}
-		}
-
-		url.RawQuery = strings.Join(newQueryArgs, "&")
-	}
-}

tap/extensions/kafka/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/kafka/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/kafka/\* expect

tap/extensions/kafka/main.go

@@ -11,11 +11,13 @@ import (
 )
 
 var _protocol = api.Protocol{
-	Name:            "kafka",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "kafka",
+		Version:      "12",
+		Abbreviation: "KAFKA",
+	},
 	LongName:        "Apache Kafka Protocol",
-	Abbreviation:    "KAFKA",
 	Macro:           "kafka",
-	Version:         "12",
 	BackgroundColor: "#000000",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -25,7 +27,7 @@ var _protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation): &_protocol,
+	_protocol.ToString(): &_protocol,
 }
 
 type dissecting string
@@ -70,8 +72,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation),
+		Protocol: _protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -195,7 +197,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -250,7 +252,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`kafka`: fmt.Sprintf(`protocol == "%s/%s/%s"`, _protocol.Name, _protocol.Version, _protocol.Abbreviation),
+		`kafka`: fmt.Sprintf(`protocol.name == "%s"`, _protocol.Name),
 	}
 }
 

tap/extensions/kafka/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"kafka": `protocol == "kafka/12/KAFKA"`,
+		"kafka": `protocol.name == "kafka"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/redis/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/redis/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/redis/\* expect

tap/extensions/redis/main.go

@@ -11,11 +11,13 @@ import (
 )
 
 var protocol = api.Protocol{
-	Name:            "redis",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "redis",
+		Version:      "3.x",
+		Abbreviation: "REDIS",
+	},
 	LongName:        "Redis Serialization Protocol",
-	Abbreviation:    "REDIS",
 	Macro:           "redis",
-	Version:         "3.x",
 	BackgroundColor: "#a41e11",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -25,7 +27,7 @@ var protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
+	protocol.ToString(): &protocol,
 }
 
 type dissecting string
@@ -78,8 +80,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Protocol: protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -123,7 +125,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -151,7 +153,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`redis`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
+		`redis`: fmt.Sprintf(`protocol.name == "%s"`, protocol.Name),
 	}
 }
 

tap/extensions/redis/main_test.go

@@ -45,7 +45,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"redis": `protocol == "redis/3.x/REDIS"`,
+		"redis": `protocol.name == "redis"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/tlstapper/bpf-builder/build.sh

@@ -9,7 +9,7 @@ docker build -t mizu-ebpf-builder . || exit 1
 BPF_TARGET=amd64
 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
 ARCH=$(uname -m)
-if [[ $ARCH == "aarch64" ]]; then
+if [[ $ARCH == "aarch64" || $ARCH == "arm64" ]]; then
     BPF_TARGET=arm64
     BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
 fi
@@ -18,7 +18,7 @@ docker run --rm \
 	--name mizu-ebpf-builder \
 	-v $MIZU_HOME:/mizu \
 	-v $(go env GOPATH):/root/go \
-	-it mizu-ebpf-builder \
+	mizu-ebpf-builder \
 	sh -c "
 		BPF_TARGET=\"$BPF_TARGET\" BPF_CFLAGS=\"$BPF_CFLAGS\" go generate tap/tlstapper/tls_tapper.go
         chown $(id -u):$(id -g) tap/tlstapper/tlstapper*_bpf*

tap/tlstapper/bpf/common.c

@@ -12,23 +12,20 @@ Copyright (C) UP9 Inc.
 #include "include/common.h"
 
 
-static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd) {
+static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd, struct ssl_info* info) {
     __u32 pid = id >> 32;
     __u64 key = (__u64) pid << 32 | fd;
 
-    struct fd_info *fdinfo = bpf_map_lookup_elem(&file_descriptor_to_ipv4, &key);
+    conn_flags *flags = bpf_map_lookup_elem(&connection_context, &key);
 
-    if (fdinfo == NULL) {
+    // Happens when we don't catch the connect / accept (if the connection is created before tapping is started)
+    if (flags == NULL) {
         return 0;
     }
 
-    int err = bpf_probe_read(chunk->address, sizeof(chunk->address), fdinfo->ipv4_addr);
+    chunk->flags |= (*flags & FLAGS_IS_CLIENT_BIT);
-    chunk->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT);
 
-    if (err != 0) {
+    bpf_probe_read(&chunk->address_info, sizeof(chunk->address_info), &info->address_info);
-        log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
-        return 0;
-    }
 
     return 1;
 }
@@ -104,7 +101,7 @@ static __always_inline void output_ssl_chunk(struct pt_regs *ctx, struct ssl_inf
     chunk->len = count_bytes;
     chunk->fd = info->fd;
 
-    if (!add_address_to_chunk(ctx, chunk, id, chunk->fd)) {
+    if (!add_address_to_chunk(ctx, chunk, id, chunk->fd, info)) {
         // Without an address, we drop the chunk because there is not much to do with it in Go
         //
         return;

tap/tlstapper/bpf/fd_to_address_tracepoints.c

@@ -14,7 +14,6 @@ Copyright (C) UP9 Inc.
 #define IPV4_ADDR_LEN (16)
 
 struct accept_info {
-	__u64* sockaddr;
 	__u32* addrlen;
 };
 
@@ -39,7 +38,6 @@ void sys_enter_accept4(struct sys_enter_accept4_ctx *ctx) {
 	
 	struct accept_info info = {};
 	
-	info.sockaddr = ctx->sockaddr;
 	info.addrlen = ctx->addrlen;
 	
 	long err = bpf_map_update_elem(&accept_syscall_context, &id, &info, BPF_ANY);
@@ -94,26 +92,21 @@ void sys_exit_accept4(struct sys_exit_accept4_ctx *ctx) {
 		return;
 	}
 	
-	struct fd_info fdinfo = {
+	conn_flags flags = 0;
-		.flags = 0
-	};
-	
-	bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr);
 	
 	__u32 pid = id >> 32;
 	__u32 fd = (__u32) ctx->ret;
 	
 	__u64 key = (__u64) pid << 32 | fd;
-	err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY);
+	err = bpf_map_update_elem(&connection_context, &key, &flags, BPF_ANY);
 	
 	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FD_MAPPING, id, err, ORIGIN_SYS_EXIT_ACCEPT4_CODE);
+		log_error(ctx, LOG_ERROR_PUTTING_CONNECTION_CONTEXT, id, err, ORIGIN_SYS_EXIT_ACCEPT4_CODE);
 	}
 }
 
 struct connect_info {
 	__u64 fd;
-	__u64* sockaddr;
 	__u32 addrlen;
 };
 
@@ -138,7 +131,6 @@ void sys_enter_connect(struct sys_enter_connect_ctx *ctx) {
 	
 	struct connect_info info = {};
 	
-	info.sockaddr = ctx->sockaddr;
 	info.addrlen = ctx->addrlen;
 	info.fd = ctx->fd;
 	
@@ -193,19 +185,15 @@ void sys_exit_connect(struct sys_exit_connect_ctx *ctx) {
 		return;
 	}
 	
-	struct fd_info fdinfo = {
+	conn_flags flags = FLAGS_IS_CLIENT_BIT;
-		.flags = FLAGS_IS_CLIENT_BIT
-	};
-	
-	bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr);
 	
 	__u32 pid = id >> 32;
 	__u32 fd = (__u32) info.fd;
 	
 	__u64 key = (__u64) pid << 32 | fd;
-	err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY);
+	err = bpf_map_update_elem(&connection_context, &key, &flags, BPF_ANY);
 	
 	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FD_MAPPING, id, err, ORIGIN_SYS_EXIT_CONNECT_CODE);
+		log_error(ctx, LOG_ERROR_PUTTING_CONNECTION_CONTEXT, id, err, ORIGIN_SYS_EXIT_CONNECT_CODE);
 	}
 }

tap/tlstapper/bpf/fd_tracepoints.c

@@ -10,8 +10,9 @@ Copyright (C) UP9 Inc.
 #include "include/log.h"
 #include "include/logger_messages.h"
 #include "include/pids.h"
+#include "include/common.h"
 
-struct sys_enter_read_ctx {
+struct sys_enter_read_write_ctx {
 	__u64 __unused_syscall_header;
 	__u32 __unused_syscall_nr;
 	
@@ -20,8 +21,46 @@ struct sys_enter_read_ctx {
 	__u64 count;
 };
 
+struct sys_exit_read_write_ctx {
+	__u64 __unused_syscall_header;
+	__u32 __unused_syscall_nr;
+
+	__u64 ret;
+};
+
+
+static __always_inline void fd_tracepoints_handle_openssl(struct sys_enter_read_write_ctx *ctx, __u64 id, struct ssl_info *infoPtr, struct bpf_map_def *map_fd, __u64 origin_code) {
+	struct ssl_info info;
+	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, origin_code);
+		return;
+	}
+	
+	info.fd = ctx->fd;
+	
+	err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, origin_code);
+		return;
+	}
+}
+
+static __always_inline void fd_tracepoints_handle_go(struct sys_enter_read_write_ctx *ctx, __u64 id, struct bpf_map_def *map_fd, __u64 origin_code) {
+	__u32 fd = ctx->fd;
+
+	long err = bpf_map_update_elem(map_fd, &id, &fd, BPF_ANY);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, origin_code);
+		return;
+	}
+}
+
 SEC("tracepoint/syscalls/sys_enter_read")
-void sys_enter_read(struct sys_enter_read_ctx *ctx) {
+void sys_enter_read(struct sys_enter_read_write_ctx *ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -30,38 +69,15 @@ void sys_enter_read(struct sys_enter_read_ctx *ctx) {
 	
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_read_context, &id);
 	
-	if (infoPtr == NULL) {
+	if (infoPtr != NULL) {
-		return;
+		fd_tracepoints_handle_openssl(ctx, id, infoPtr, &openssl_read_context, ORIGIN_SYS_ENTER_READ_CODE);
 	}
 
-	struct ssl_info info;
+	fd_tracepoints_handle_go(ctx, id, &go_kernel_read_context, ORIGIN_SYS_ENTER_READ_CODE);
-	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, ORIGIN_SYS_ENTER_READ_CODE);
-		return;
-	}
-	
-	info.fd = ctx->fd;
-	
-	err = bpf_map_update_elem(&openssl_read_context, &id, &info, BPF_ANY);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_READ_CODE);
-	}
 }
 	
-struct sys_enter_write_ctx {
-	__u64 __unused_syscall_header;
-	__u32 __unused_syscall_nr;
-	
-	__u64 fd;
-	__u64* buf;
-	__u64 count;
-};
-
 SEC("tracepoint/syscalls/sys_enter_write")
-void sys_enter_write(struct sys_enter_write_ctx *ctx) {
+void sys_enter_write(struct sys_enter_read_write_ctx *ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -70,23 +86,25 @@ void sys_enter_write(struct sys_enter_write_ctx *ctx) {
 	
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_write_context, &id);
 	
-	if (infoPtr == NULL) {
+	if (infoPtr != NULL) {
-		return;
+		fd_tracepoints_handle_openssl(ctx, id, infoPtr, &openssl_write_context, ORIGIN_SYS_ENTER_WRITE_CODE);
 	}
 
-	struct ssl_info info;
+	fd_tracepoints_handle_go(ctx, id, &go_kernel_write_context, ORIGIN_SYS_ENTER_WRITE_CODE);
-	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
+}
-	
+
-	if (err != 0) {
+SEC("tracepoint/syscalls/sys_exit_read")
-		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, ORIGIN_SYS_ENTER_WRITE_CODE);
+void sys_exit_read(struct sys_exit_read_write_ctx *ctx) {
-		return;
+	__u64 id = bpf_get_current_pid_tgid();
-	}
+	// Delete from go map. The value is not used after exiting this syscall.
-	
+	// Keep value in openssl map.
-	info.fd = ctx->fd;
+	bpf_map_delete_elem(&go_kernel_read_context, &id);
-	
+}
-	err = bpf_map_update_elem(&openssl_write_context, &id, &info, BPF_ANY);
+
-	
+SEC("tracepoint/syscalls/sys_exit_write")
-	if (err != 0) {
+void sys_exit_write(struct sys_exit_read_write_ctx *ctx) {
-		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_WRITE_CODE);
+	__u64 id = bpf_get_current_pid_tgid();
-	}
+	// Delete from go map. The value is not used after exiting this syscall.
+	// Keep value in openssl map.
+	bpf_map_delete_elem(&go_kernel_write_context, &id);
 }

tap/tlstapper/bpf/go_uprobes.c

@@ -220,7 +220,7 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf
     return;
 }
 
-static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, __u32 flags, enum ABI abi) {
+static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, struct bpf_map_def* go_user_kernel_context, __u32 flags, enum ABI abi) {
     __u64 pid_tgid = bpf_get_current_pid_tgid();
     __u64 pid = pid_tgid >> 32;
     if (!should_tap(pid)) {
@@ -285,6 +285,21 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct
         }
     }
 
+    __u64 key = (__u64) pid << 32 | info_ptr->fd;
+    struct address_info *address_info = bpf_map_lookup_elem(go_user_kernel_context, &key);
+    // Ideally we would delete the entry from the map after reading it,
+    // but sometimes the uprobe is called twice in a row without the tcp kprobes in between to fill in
+    // the entry again. Keeping it in the map and rely on LRU logic.
+    if (address_info == NULL) {
+        log_error(ctx, LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT, pid_tgid, info_ptr->fd, err);
+        return;
+    }
+
+    info.address_info.daddr = address_info->daddr;
+    info.address_info.dport = address_info->dport;
+    info.address_info.saddr = address_info->saddr;
+    info.address_info.sport = address_info->sport;
+
     output_ssl_chunk(ctx, &info, info.buffer_len, pid_tgid, flags);
 
     return;
@@ -298,7 +313,7 @@ int BPF_KPROBE(go_crypto_tls_abi0_write) {
 
 SEC("uprobe/go_crypto_tls_abi0_write_ex")
 int BPF_KPROBE(go_crypto_tls_abi0_write_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0, ABI0);
+    go_crypto_tls_ex_uprobe(ctx, &go_write_context, &go_user_kernel_write_context, 0, ABI0);
     return 1;
 }
 
@@ -310,7 +325,7 @@ int BPF_KPROBE(go_crypto_tls_abi0_read) {
 
 SEC("uprobe/go_crypto_tls_abi0_read_ex")
 int BPF_KPROBE(go_crypto_tls_abi0_read_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT, ABI0);
+    go_crypto_tls_ex_uprobe(ctx, &go_read_context, &go_user_kernel_read_context, FLAGS_IS_READ_BIT, ABI0);
     return 1;
 }
 
@@ -322,7 +337,7 @@ int BPF_KPROBE(go_crypto_tls_abi_internal_write) {
 
 SEC("uprobe/go_crypto_tls_abi_internal_write_ex")
 int BPF_KPROBE(go_crypto_tls_abi_internal_write_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0, ABIInternal);
+    go_crypto_tls_ex_uprobe(ctx, &go_write_context, &go_user_kernel_write_context, 0, ABIInternal);
     return 1;
 }
 
@@ -334,6 +349,6 @@ int BPF_KPROBE(go_crypto_tls_abi_internal_read) {
 
 SEC("uprobe/go_crypto_tls_abi_internal_read_ex")
 int BPF_KPROBE(go_crypto_tls_abi_internal_read_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT, ABIInternal);
+    go_crypto_tls_ex_uprobe(ctx, &go_read_context, &go_user_kernel_read_context, FLAGS_IS_READ_BIT, ABIInternal);
     return 1;
 }

tap/tlstapper/bpf/include/common.h

@@ -7,9 +7,11 @@ Copyright (C) UP9 Inc.
 #ifndef __COMMON__
 #define __COMMON__
 
+#define AF_INET	2	/* Internet IP Protocol */
+
 const __s32 invalid_fd = -1;
 
-static int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd);
+static int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd, struct ssl_info* info);
 static void send_chunk_part(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk, int start, int end);
 static void send_chunk(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk);
 static void output_ssl_chunk(struct pt_regs *ctx, struct ssl_info* info, int count_bytes, __u64 id, __u32 flags);

tap/tlstapper/bpf/include/headers.h

@@ -15,6 +15,7 @@ Copyright (C) UP9 Inc.
 
 #include "legacy_kernel.h"
 
+#include <bpf/bpf_endian.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 #include <bpf/bpf_core_read.h>

tap/tlstapper/bpf/include/logger_messages.h

@@ -10,22 +10,28 @@ Copyright (C) UP9 Inc.
 // Must be synced with bpf_logger_messages.go
 //
 #define LOG_ERROR_READING_BYTES_COUNT (0)
-#define LOG_ERROR_READING_FD_ADDRESS (1)
+#define LOG_ERROR_READING_FROM_SSL_BUFFER (1)
-#define LOG_ERROR_READING_FROM_SSL_BUFFER (2)
+#define LOG_ERROR_BUFFER_TOO_BIG (2)
-#define LOG_ERROR_BUFFER_TOO_BIG (3)
+#define LOG_ERROR_ALLOCATING_CHUNK (3)
-#define LOG_ERROR_ALLOCATING_CHUNK (4)
+#define LOG_ERROR_READING_SSL_CONTEXT (4)
-#define LOG_ERROR_READING_SSL_CONTEXT (5)
+#define LOG_ERROR_PUTTING_SSL_CONTEXT (5)
-#define LOG_ERROR_PUTTING_SSL_CONTEXT (6)
+#define LOG_ERROR_GETTING_SSL_CONTEXT (6)
-#define LOG_ERROR_GETTING_SSL_CONTEXT (7)
+#define LOG_ERROR_MISSING_FILE_DESCRIPTOR (7)
-#define LOG_ERROR_MISSING_FILE_DESCRIPTOR (8)
+#define LOG_ERROR_PUTTING_FILE_DESCRIPTOR (8)
-#define LOG_ERROR_PUTTING_FILE_DESCRIPTOR (9)
+#define LOG_ERROR_PUTTING_ACCEPT_INFO (9)
-#define LOG_ERROR_PUTTING_ACCEPT_INFO (10)
+#define LOG_ERROR_GETTING_ACCEPT_INFO (10)
-#define LOG_ERROR_GETTING_ACCEPT_INFO (11)
+#define LOG_ERROR_READING_ACCEPT_INFO (11)
-#define LOG_ERROR_READING_ACCEPT_INFO (12)
+#define LOG_ERROR_PUTTING_CONNECTION_CONTEXT (12)
-#define LOG_ERROR_PUTTING_FD_MAPPING (13)
+#define LOG_ERROR_PUTTING_CONNECT_INFO (13)
-#define LOG_ERROR_PUTTING_CONNECT_INFO (14)
+#define LOG_ERROR_GETTING_CONNECT_INFO (14)
-#define LOG_ERROR_GETTING_CONNECT_INFO (15)
+#define LOG_ERROR_READING_CONNECT_INFO (15)
-#define LOG_ERROR_READING_CONNECT_INFO (16)
+#define LOG_ERROR_READING_SOCKET_FAMILY (16)
+#define LOG_ERROR_READING_SOCKET_DADDR (17)
+#define LOG_ERROR_READING_SOCKET_SADDR (18)
+#define LOG_ERROR_READING_SOCKET_DPORT (19)
+#define LOG_ERROR_READING_SOCKET_SPORT (20)
+#define LOG_ERROR_PUTTING_GO_USER_KERNEL_CONTEXT (21)
+#define LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT (22)
 
 // Sometimes we have the same error, happening from different locations.
 // 	in order to be able to distinct between them in the log, we add an 

tap/tlstapper/bpf/include/maps.h

@@ -24,6 +24,14 @@ Copyright (C) UP9 Inc.
 //  
 //  Be careful when editing, alignment and padding should be exactly the same in go/c.
 //
+
+struct address_info {
+    __be32 saddr;
+    __be32 daddr;
+    __be16 sport;
+    __be16 dport;
+};
+
 struct tls_chunk {
     __u32 pid;
     __u32 tgid;
@@ -32,7 +40,7 @@ struct tls_chunk {
     __u32 recorded;
     __u32 fd;
     __u32 flags;
-    __u8 address[16];
+    struct address_info address_info;
     __u8 data[CHUNK_SIZE]; // Must be N^2
 };
 
@@ -41,6 +49,7 @@ struct ssl_info {
     __u32 buffer_len;
     __u32 fd;
     __u64 created_at_nano;
+    struct address_info address_info;
     
     // for ssl_write and ssl_read must be zero
     // for ssl_write_ex and ssl_read_ex save the *written/*readbytes pointer. 
@@ -48,10 +57,7 @@ struct ssl_info {
     size_t *count_ptr;
 };
 
-struct fd_info {
+typedef __u8 conn_flags;
-    __u8 ipv4_addr[16]; // struct sockaddr (linux-src/include/linux/socket.h)
-    __u8 flags;
-};
 
 struct goid_offsets {
     __u64 g_addr_offset;
@@ -89,7 +95,7 @@ struct {
 
 // Generic
 BPF_HASH(pids_map, __u32, __u32);
-BPF_LRU_HASH(file_descriptor_to_ipv4, __u64, struct fd_info);
+BPF_LRU_HASH(connection_context, __u64, conn_flags);
 BPF_PERF_OUTPUT(chunks_buffer);
 BPF_PERF_OUTPUT(log_buffer);
 
@@ -101,5 +107,9 @@ BPF_LRU_HASH(openssl_read_context, __u64, struct ssl_info);
 BPF_HASH(goid_offsets_map, __u32, struct goid_offsets);
 BPF_LRU_HASH(go_write_context, __u64, struct ssl_info);
 BPF_LRU_HASH(go_read_context, __u64, struct ssl_info);
+BPF_LRU_HASH(go_kernel_write_context, __u64, __u32);
+BPF_LRU_HASH(go_kernel_read_context, __u64, __u32);
+BPF_LRU_HASH(go_user_kernel_write_context, __u64, struct address_info);
+BPF_LRU_HASH(go_user_kernel_read_context, __u64, struct address_info);
 
 #endif /* __MAPS__ */

tap/tlstapper/bpf/openssl_uprobes.c

@@ -42,6 +42,8 @@ static __always_inline int get_count_bytes(struct pt_regs *ctx, struct ssl_info*
 }
 
 static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buffer, int num, struct bpf_map_def* map_fd, size_t *count_ptr) {
+	long err;
+
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -53,7 +55,7 @@ static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buf
 	info.count_ptr = count_ptr;
 	info.buffer = buffer;
 	
-	long err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
+	err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
 	
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, id, err, 0l);
@@ -99,10 +101,10 @@ static __always_inline void ssl_uretprobe(struct pt_regs *ctx, struct bpf_map_de
 		return;
 	}
 
-    int count_bytes = get_count_bytes(ctx, &info, id);
+	int count_bytes = get_count_bytes(ctx, &info, id);
-    if (count_bytes <= 0) {
+	if (count_bytes <= 0) {
-        return;
+		return;
-    }
+	}
 
 	output_ssl_chunk(ctx, &info, count_bytes, id, flags);
 }

tap/tlstapper/bpf/tcp_kprobes.c

@@ -0,0 +1,118 @@
+#include "include/headers.h"
+#include "include/maps.h"
+#include "include/log.h"
+#include "include/logger_messages.h"
+#include "include/pids.h"
+#include "include/common.h"
+
+
+static __always_inline int tcp_kprobes_get_address_pair_from_ctx(struct pt_regs *ctx, __u64 id, struct address_info *address_info_ptr) {
+	long err;
+	struct sock *sk = (struct sock *) PT_REGS_PARM1(ctx);
+
+	short unsigned int family;
+	err = bpf_probe_read(&family, sizeof(family), (void *)&sk->__sk_common.skc_family);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_FAMILY, id, err, 0l);
+		return -1;
+	}
+	if (family != AF_INET) {
+		return -1;
+	}
+
+	// daddr, saddr and dport are in network byte order (big endian)
+	// sport is in host byte order
+	__be32 saddr;
+	__be32 daddr;
+	__be16 dport;
+	__u16 sport;
+
+	err = bpf_probe_read(&saddr, sizeof(saddr), (void *)&sk->__sk_common.skc_rcv_saddr);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_SADDR, id, err, 0l);
+		return -1;
+	}
+	err = bpf_probe_read(&daddr, sizeof(daddr), (void *)&sk->__sk_common.skc_daddr);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_DADDR, id, err, 0l);
+		return -1;
+	}
+	err = bpf_probe_read(&dport, sizeof(dport), (void *)&sk->__sk_common.skc_dport);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_DPORT, id, err, 0l);
+		return -1;
+	}
+	err = bpf_probe_read(&sport, sizeof(sport), (void *)&sk->__sk_common.skc_num);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_SPORT, id, err, 0l);
+		return -1;
+	}
+
+	address_info_ptr->daddr = daddr;
+	address_info_ptr->saddr = saddr;
+	address_info_ptr->dport = dport;
+	address_info_ptr->sport = bpf_htons(sport);
+
+	return 0;
+}
+
+static __always_inline void tcp_kprobes_forward_go(struct pt_regs *ctx, __u64 id, __u32 fd, struct address_info address_info, struct bpf_map_def *map_fd_go_user_kernel) {
+		__u32 pid = id >> 32;
+		__u64 key = (__u64) pid << 32 | fd;
+
+		long err = bpf_map_update_elem(map_fd_go_user_kernel, &key, &address_info, BPF_ANY);
+    if (err != 0) {
+        log_error(ctx, LOG_ERROR_PUTTING_GO_USER_KERNEL_CONTEXT, id, fd, err);
+				return;
+    }
+}
+
+static void __always_inline tcp_kprobes_forward_openssl(struct ssl_info *info_ptr, struct address_info address_info) {
+		info_ptr->address_info.daddr = address_info.daddr;
+		info_ptr->address_info.saddr = address_info.saddr;
+		info_ptr->address_info.dport = address_info.dport;
+		info_ptr->address_info.sport = address_info.sport;
+}
+
+static __always_inline void tcp_kprobe(struct pt_regs *ctx, struct bpf_map_def *map_fd_openssl, struct bpf_map_def *map_fd_go_kernel, struct bpf_map_def *map_fd_go_user_kernel) {
+	long err;
+
+	__u64 id = bpf_get_current_pid_tgid();
+
+	if (!should_tap(id >> 32)) {
+		return;
+	}
+
+	struct address_info address_info;
+	if (0 != tcp_kprobes_get_address_pair_from_ctx(ctx, id, &address_info)) {
+		return;
+	}
+
+	struct ssl_info *info_ptr = bpf_map_lookup_elem(map_fd_openssl, &id);
+	__u32 *fd_ptr;
+	if (info_ptr == NULL) {
+		fd_ptr = bpf_map_lookup_elem(map_fd_go_kernel, &id);
+		// Connection is used by a Go program
+		if (fd_ptr == NULL) {
+			// Connection was not created by a Go program or by openssl lib
+			return;
+		}
+		tcp_kprobes_forward_go(ctx, id, *fd_ptr, address_info, map_fd_go_user_kernel);
+	} else {
+		// Connection is used by openssl lib
+		tcp_kprobes_forward_openssl(info_ptr, address_info);
+	}
+
+}
+
+SEC("kprobe/tcp_sendmsg")
+void BPF_KPROBE(tcp_sendmsg) {
+	__u64 id = bpf_get_current_pid_tgid();
+	tcp_kprobe(ctx, &openssl_write_context, &go_kernel_write_context, &go_user_kernel_write_context);
+}
+
+SEC("kprobe/tcp_recvmsg")
+void BPF_KPROBE(tcp_recvmsg) {
+	__u64 id = bpf_get_current_pid_tgid();
+	tcp_kprobe(ctx, &openssl_read_context, &go_kernel_read_context, &go_user_kernel_read_context);
+}

tap/tlstapper/bpf/tls_tapper.c

@@ -15,6 +15,7 @@ Copyright (C) UP9 Inc.
 //
 #include "common.c"
 #include "openssl_uprobes.c"
+#include "tcp_kprobes.c"
 #include "go_uprobes.c"
 #include "fd_tracepoints.c"
 #include "fd_to_address_tracepoints.c"

tap/tlstapper/bpf_logger_messages.go

@@ -4,20 +4,26 @@ package tlstapper
 //
 var bpfLogMessages = []string{
 	/*0000*/ "[%d] Unable to read bytes count from _ex methods [err: %d]",
-	/*0001*/ "[%d] Unable to read ipv4 address [err: %d]",
+	/*0001*/ "[%d] Unable to read ssl buffer [err: %d] [origin: %d]",
-	/*0002*/ "[%d] Unable to read ssl buffer [err: %d]",
+	/*0002*/ "[%d] Buffer is too big [size: %d]",
-	/*0003*/ "[%d] Buffer is too big [size: %d]",
+	/*0003*/ "[%d] Unable to allocate chunk in bpf heap",
-	/*0004*/ "[%d] Unable to allocate chunk in bpf heap",
+	/*0004*/ "[%d] Unable to read ssl context [err: %d] [origin: %d]",
-	/*0005*/ "[%d] Unable to read ssl context [err: %d] [origin: %d]",
+	/*0005*/ "[%d] Unable to put ssl context [err: %d]",
-	/*0006*/ "[%d] Unable to put ssl context [err: %d]",
+	/*0006*/ "[%d] Unable to get ssl context",
-	/*0007*/ "[%d] Unable to get ssl context",
+	/*0007*/ "[%d] File descriptor is missing for tls chunk",
-	/*0008*/ "[%d] File descriptor is missing for tls chunk",
+	/*0008*/ "[%d] Unable to put file descriptor [err: %d] [origin: %d]",
-	/*0009*/ "[%d] Unable to put file descriptor [err: %d] [origin: %d]",
+	/*0009*/ "[%d] Unable to put accept info [err: %d]",
-	/*0010*/ "[%d] Unable to put accept info [err: %d]",
+	/*0010*/ "[%d] Unable to get accept info",
-	/*0011*/ "[%d] Unable to get accept info",
+	/*0011*/ "[%d] Unable to read accept info [err: %d]",
-	/*0012*/ "[%d] Unable to read accept info [err: %d]",
+	/*0012*/ "[%d] Unable to put connection info to connection context [err: %d] [origin: %d]",
-	/*0013*/ "[%d] Unable to put file descriptor to address mapping [err: %d] [origin: %d]",
+	/*0013*/ "[%d] Unable to put connect info [err: %d]",
-	/*0014*/ "[%d] Unable to put connect info [err: %d]",
+	/*0014*/ "[%d] Unable to get connect info",
-	/*0015*/ "[%d] Unable to get connect info",
+	/*0015*/ "[%d] Unable to read connect info [err: %d]",
-	/*0016*/ "[%d] Unable to read connect info [err: %d]",
+	/*0016*/ "[%d] Unable to read socket family [err: %d]",
+	/*0017*/ "[%d] Unable to read socket daddr [err: %d]",
+	/*0018*/ "[%d] Unable to read socket saddr [err: %d]",
+	/*0019*/ "[%d] Unable to read socket dport [err: %d]",
+	/*0020*/ "[%d] Unable to read socket sport [err: %d]",
+	/*0021*/ "[%d] Unable to put go user-kernel context [fd: %d] [err: %d]",
+	/*0022*/ "[%d] Unable to get go user-kernel context [fd: %d]]",
 }

tap/tlstapper/chunk.go

@@ -1,38 +1,33 @@
 package tlstapper
 
 import (
-	"bytes"
 	"encoding/binary"
 	"net"
-
+	"unsafe"
-	"github.com/go-errors/errors"
-	"github.com/up9inc/mizu/tap/api"
 )
 
 const FlagsIsClientBit uint32 = 1 << 0
 const FlagsIsReadBit uint32 = 1 << 1
 
-func (c *tlsTapperTlsChunk) getAddress() (net.IP, uint16, error) {
+type addressPair struct {
-	address := bytes.NewReader(c.Address[:])
+	srcIp   net.IP
-	var family uint16
+	srcPort uint16
-	var port uint16
+	dstIp   net.IP
-	var ip32 uint32
+	dstPort uint16
+}
 
-	if err := binary.Read(address, binary.BigEndian, &family); err != nil {
+func (c *tlsTapperTlsChunk) getSrcAddress() (net.IP, uint16) {
-		return nil, 0, errors.Wrap(err, 0)
+	ip := intToIP(c.AddressInfo.Saddr)
-	}
+	port := ntohs(c.AddressInfo.Sport)
 
-	if err := binary.Read(address, binary.BigEndian, &port); err != nil {
+	return ip, port
-		return nil, 0, errors.Wrap(err, 0)
+}
-	}
 
-	if err := binary.Read(address, binary.BigEndian, &ip32); err != nil {
+func (c *tlsTapperTlsChunk) getDstAddress() (net.IP, uint16) {
-		return nil, 0, errors.Wrap(err, 0)
+	ip := intToIP(c.AddressInfo.Daddr)
-	}
+	port := ntohs(c.AddressInfo.Dport)
 
-	ip := net.IP{uint8(ip32 >> 24), uint8(ip32 >> 16), uint8(ip32 >> 8), uint8(ip32)}
+	return ip, port
-
-	return ip, port, nil
 }
 
 func (c *tlsTapperTlsChunk) isClient() bool {
@@ -59,26 +54,36 @@ func (c *tlsTapperTlsChunk) isRequest() bool {
 	return (c.isClient() && c.isWrite()) || (c.isServer() && c.isRead())
 }
 
-func (c *tlsTapperTlsChunk) getAddressPair() (addressPair, error) {
+func (c *tlsTapperTlsChunk) getAddressPair() addressPair {
-	ip, port, err := c.getAddress()
+	var (
-
+		srcIp, dstIp     net.IP
-	if err != nil {
+		srcPort, dstPort uint16
-		return addressPair{}, err
+	)
-	}
 
 	if c.isRequest() {
-		return addressPair{
+		srcIp, srcPort = c.getSrcAddress()
-			srcIp:   api.UnknownIp,
+		dstIp, dstPort = c.getDstAddress()
-			srcPort: api.UnknownPort,
-			dstIp:   ip,
-			dstPort: port,
-		}, nil
 	} else {
-		return addressPair{
+		srcIp, srcPort = c.getDstAddress()
-			srcIp:   ip,
+		dstIp, dstPort = c.getSrcAddress()
-			srcPort: port,
+	}
-			dstIp:   api.UnknownIp,
+
-			dstPort: api.UnknownPort,
+	return addressPair{
-		}, nil
+		srcIp:   srcIp,
+		srcPort: srcPort,
+		dstIp:   dstIp,
+		dstPort: dstPort,
 	}
 }
+
+// intToIP converts IPv4 number to net.IP
+func intToIP(ip32be uint32) net.IP {
+	return net.IPv4(uint8(ip32be), uint8(ip32be>>8), uint8(ip32be>>16), uint8(ip32be>>24))
+}
+
+// ntohs converts big endian (network byte order) to little endian (assuming that's the host byte order)
+func ntohs(i16be uint16) uint16 {
+	b := make([]byte, 2)
+	binary.BigEndian.PutUint16(b, i16be)
+	return *(*uint16)(unsafe.Pointer(&b[0]))
+}

tap/tlstapper/sockfd_info.go

@@ -1,122 +0,0 @@
-package tlstapper
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"github.com/go-errors/errors"
-)
-
-var socketInodeRegex = regexp.MustCompile(`socket:\[(\d+)\]`)
-
-const (
-	SRC_ADDRESS_FILED_INDEX = 1
-	DST_ADDRESS_FILED_INDEX = 2
-	INODE_FILED_INDEX       = 9
-)
-
-type addressPair struct {
-	srcIp   net.IP
-	srcPort uint16
-	dstIp   net.IP
-	dstPort uint16
-}
-
-// This file helps to extract Ip and Port out of a Socket file descriptor.
-//
-// The equivalent bash commands are:
-//
-//  > ls -l /proc/<pid>/fd/<fd>
-// 	    Output something like "socket:[1234]" for sockets - 1234 is the inode of the socket
-//  > cat /proc/<pid>/net/tcp | grep <inode>
-//      Output a line per ipv4 socket, the 9th field is the inode of the socket
-//      The 1st and 2nd fields are the source and dest ip and ports in a Hex format
-//      0100007F:50 is 127.0.0.1:80
-
-func getAddressBySockfd(procfs string, pid uint32, fd uint32) (addressPair, error) {
-	inode, err := getSocketInode(procfs, pid, fd)
-
-	if err != nil {
-		return addressPair{}, err
-	}
-
-	tcppath := fmt.Sprintf("%s/%d/net/tcp", procfs, pid)
-	tcp, err := ioutil.ReadFile(tcppath)
-
-	if err != nil {
-		return addressPair{}, errors.Wrap(err, 0)
-	}
-
-	for _, line := range strings.Split(string(tcp), "\n") {
-		parts := strings.Fields(line)
-
-		if len(parts) < 10 {
-			continue
-		}
-
-		if inode == parts[INODE_FILED_INDEX] {
-			srcIp, srcPort, srcErr := parseHexAddress(parts[SRC_ADDRESS_FILED_INDEX])
-
-			if srcErr != nil {
-				return addressPair{}, srcErr
-			}
-
-			dstIp, dstPort, dstErr := parseHexAddress(parts[DST_ADDRESS_FILED_INDEX])
-
-			if dstErr != nil {
-				return addressPair{}, dstErr
-			}
-
-			return addressPair{
-				srcIp:   srcIp,
-				srcPort: srcPort,
-				dstIp:   dstIp,
-				dstPort: dstPort,
-			}, nil
-		}
-	}
-
-	return addressPair{}, errors.Errorf("address not found [pid: %d] [sockfd: %d] [inode: %s]", pid, fd, inode)
-}
-
-func getSocketInode(procfs string, pid uint32, fd uint32) (string, error) {
-	fdlinkPath := fmt.Sprintf("%s/%d/fd/%d", procfs, pid, fd)
-	fdlink, err := os.Readlink(fdlinkPath)
-
-	if err != nil {
-		return "", errors.Wrap(err, 0)
-	}
-
-	tokens := socketInodeRegex.FindStringSubmatch(fdlink)
-
-	if tokens == nil || len(tokens) < 1 {
-		return "", errors.Errorf("socket inode not found [pid: %d] [sockfd: %d] [link: %s]", pid, fd, fdlink)
-	}
-
-	return tokens[1], nil
-}
-
-// Format looks like 0100007F:50 for 127.0.0.1:80
-//
-func parseHexAddress(addr string) (net.IP, uint16, error) {
-	addrParts := strings.Split(addr, ":")
-
-	port, err := strconv.ParseUint(addrParts[1], 16, 16)
-
-	if err != nil {
-		return nil, 0, errors.Wrap(err, 0)
-	}
-
-	ip, err := strconv.ParseUint(addrParts[0], 16, 32)
-
-	if err != nil {
-		return nil, 0, errors.Wrap(err, 0)
-	}
-
-	return net.IP{uint8(ip), uint8(ip >> 8), uint8(ip >> 16), uint8(ip >> 24)}, uint16(port), nil
-}

tap/tlstapper/syscall_hooks.go

@@ -8,6 +8,8 @@ import (
 type syscallHooks struct {
 	sysEnterRead    link.Link
 	sysEnterWrite   link.Link
+	sysExitRead     link.Link
+	sysExitWrite    link.Link
 	sysEnterAccept4 link.Link
 	sysExitAccept4  link.Link
 	sysEnterConnect link.Link
@@ -29,6 +31,18 @@ func (s *syscallHooks) installSyscallHooks(bpfObjects *tlsTapperObjects) error {
 		return errors.Wrap(err, 0)
 	}
 
+	s.sysExitRead, err = link.Tracepoint("syscalls", "sys_exit_read", bpfObjects.SysExitRead, nil)
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	s.sysExitWrite, err = link.Tracepoint("syscalls", "sys_exit_write", bpfObjects.SysExitWrite, nil)
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
 	s.sysEnterAccept4, err = link.Tracepoint("syscalls", "sys_enter_accept4", bpfObjects.SysEnterAccept4, nil)
 
 	if err != nil {
@@ -67,6 +81,14 @@ func (s *syscallHooks) close() []error {
 		returnValue = append(returnValue, err)
 	}
 
+	if err := s.sysExitRead.Close(); err != nil {
+		returnValue = append(returnValue, err)
+	}
+
+	if err := s.sysExitWrite.Close(); err != nil {
+		returnValue = append(returnValue, err)
+	}
+
 	if err := s.sysEnterAccept4.Close(); err != nil {
 		returnValue = append(returnValue, err)
 	}

tap/tlstapper/tcp_kprobe_hooks.go

@@ -0,0 +1,45 @@
+package tlstapper
+
+import (
+	"github.com/cilium/ebpf/link"
+	"github.com/go-errors/errors"
+)
+
+type tcpKprobeHooks struct {
+	tcpSendmsg         link.Link
+	tcpRecvmsg         link.Link
+}
+
+func (s *tcpKprobeHooks) installTcpKprobeHooks(bpfObjects *tlsTapperObjects) error {
+	var err error
+
+	s.tcpSendmsg, err = link.Kprobe("tcp_sendmsg", bpfObjects.TcpSendmsg, nil)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	s.tcpRecvmsg, err = link.Kprobe("tcp_recvmsg", bpfObjects.TcpRecvmsg, nil)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	return nil
+}
+
+func (s *tcpKprobeHooks) close() []error {
+	returnValue := make([]error, 0)
+
+	if s.tcpSendmsg != nil {
+		if err := s.tcpSendmsg.Close(); err != nil {
+			returnValue = append(returnValue, err)
+		}
+	}
+
+	if s.tcpRecvmsg != nil {
+		if err := s.tcpRecvmsg.Close(); err != nil {
+			returnValue = append(returnValue, err)
+		}
+	}
+
+	return returnValue
+}

tap/tlstapper/tls_poller.go

@@ -134,15 +134,7 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsTapperTlsChunk) {
 
 func (p *tlsPoller) handleTlsChunk(chunk *tlsTapperTlsChunk, extension *api.Extension, emitter api.Emitter,
 	options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) error {
-	address, err := p.getSockfdAddressPair(chunk)
+	address := chunk.getAddressPair()
-
-	if err != nil {
-		address, err = chunk.getAddressPair()
-
-		if err != nil {
-			return err
-		}
-	}
 
 	key := buildTlsKey(address)
 	reader, exists := p.readers[key]
@@ -216,41 +208,6 @@ func (p *tlsPoller) closeReader(key string, r *tlsReader) {
 	p.closedReaders <- key
 }
 
-func (p *tlsPoller) getSockfdAddressPair(chunk *tlsTapperTlsChunk) (addressPair, error) {
-	address, err := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd)
-	fdCacheKey := fmt.Sprintf("%d:%d", chunk.Pid, chunk.Fd)
-
-	if err == nil {
-		if !chunk.isRequest() {
-			switchedAddress := addressPair{
-				srcIp:   address.dstIp,
-				srcPort: address.dstPort,
-				dstIp:   address.srcIp,
-				dstPort: address.srcPort,
-			}
-			p.fdCache.Add(fdCacheKey, switchedAddress)
-			return switchedAddress, nil
-		} else {
-			p.fdCache.Add(fdCacheKey, address)
-			return address, nil
-		}
-	}
-
-	fromCacheIfc, ok := p.fdCache.Get(fdCacheKey)
-
-	if !ok {
-		return addressPair{}, err
-	}
-
-	fromCache, ok := fromCacheIfc.(addressPair)
-
-	if !ok {
-		return address, errors.Errorf("Unable to cast %T to addressPair", fromCacheIfc)
-	}
-
-	return fromCache, nil
-}
-
 func buildTlsKey(address addressPair) string {
 	return fmt.Sprintf("%s:%d>%s:%d", address.srcIp, address.srcPort, address.dstIp, address.dstPort)
 }

tap/tlstapper/tls_tapper.go

@@ -22,6 +22,7 @@ const GlobalTapPid = 0
 type TlsTapper struct {
 	bpfObjects      tlsTapperObjects
 	syscallHooks    syscallHooks
+	tcpKprobeHooks  tcpKprobeHooks
 	sslHooksStructs []sslHooks
 	goHooksStructs  []goHooks
 	poller          *tlsPoller
@@ -63,6 +64,11 @@ func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string,
 		return err
 	}
 
+	t.tcpKprobeHooks = tcpKprobeHooks{}
+	if err := t.tcpKprobeHooks.installTcpKprobeHooks(&t.bpfObjects); err != nil {
+		return err
+	}
+
 	t.sslHooksStructs = make([]sslHooks, 0)
 
 	t.bpfLogger = newBpfLogger()
@@ -152,6 +158,8 @@ func (t *TlsTapper) Close() []error {
 
 	returnValue = append(returnValue, t.syscallHooks.close()...)
 
+	returnValue = append(returnValue, t.tcpKprobeHooks.close()...)
+
 	for _, sslHooks := range t.sslHooksStructs {
 		returnValue = append(returnValue, sslHooks.close()...)
 	}

tap/tlstapper/tlstapper46_bpfel_arm64.go

@@ -19,15 +19,21 @@ type tlsTapper46GoidOffsets struct {
 }
 
 type tlsTapper46TlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Mode  int32
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper46 returns the embedded CollectionSpec for tlsTapper46.
@@ -93,6 +99,8 @@ type tlsTapper46ProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapper46MapSpecs contains maps before they are loaded into the kernel.
@@ -189,6 +197,8 @@ type tlsTapper46Programs struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapper46Programs) Close() error {
@@ -215,6 +225,8 @@ func (p *tlsTapper46Programs) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

tap/tlstapper/tlstapper46_bpfel_x86.go

@@ -19,15 +19,20 @@ type tlsTapper46GoidOffsets struct {
 }
 
 type tlsTapper46TlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper46 returns the embedded CollectionSpec for tlsTapper46.
@@ -93,24 +98,32 @@ type tlsTapper46ProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.ProgramSpec `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.ProgramSpec `ebpf:"sys_exit_write"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapper46MapSpecs contains maps before they are loaded into the kernel.
 //
 // It can be passed ebpf.CollectionSpec.Assign.
 type tlsTapper46MapSpecs struct {
-	AcceptSyscallContext *ebpf.MapSpec `ebpf:"accept_syscall_context"`
+	AcceptSyscallContext     *ebpf.MapSpec `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.MapSpec `ebpf:"chunks_buffer"`
+	ChunksBuffer             *ebpf.MapSpec `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.MapSpec `ebpf:"connect_syscall_info"`
+	ConnectSyscallInfo       *ebpf.MapSpec `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"`
+	ConnectionContext        *ebpf.MapSpec `ebpf:"connection_context"`
-	GoReadContext        *ebpf.MapSpec `ebpf:"go_read_context"`
+	GoKernelReadContext      *ebpf.MapSpec `ebpf:"go_kernel_read_context"`
-	GoWriteContext       *ebpf.MapSpec `ebpf:"go_write_context"`
+	GoKernelWriteContext     *ebpf.MapSpec `ebpf:"go_kernel_write_context"`
-	GoidOffsetsMap       *ebpf.MapSpec `ebpf:"goid_offsets_map"`
+	GoReadContext            *ebpf.MapSpec `ebpf:"go_read_context"`
-	Heap                 *ebpf.MapSpec `ebpf:"heap"`
+	GoUserKernelReadContext  *ebpf.MapSpec `ebpf:"go_user_kernel_read_context"`
-	LogBuffer            *ebpf.MapSpec `ebpf:"log_buffer"`
+	GoUserKernelWriteContext *ebpf.MapSpec `ebpf:"go_user_kernel_write_context"`
-	OpensslReadContext   *ebpf.MapSpec `ebpf:"openssl_read_context"`
+	GoWriteContext           *ebpf.MapSpec `ebpf:"go_write_context"`
-	OpensslWriteContext  *ebpf.MapSpec `ebpf:"openssl_write_context"`
+	GoidOffsetsMap           *ebpf.MapSpec `ebpf:"goid_offsets_map"`
-	PidsMap              *ebpf.MapSpec `ebpf:"pids_map"`
+	Heap                     *ebpf.MapSpec `ebpf:"heap"`
+	LogBuffer                *ebpf.MapSpec `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.MapSpec `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.MapSpec `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.MapSpec `ebpf:"pids_map"`
 }
 
 // tlsTapper46Objects contains all objects after they have been loaded into the kernel.
@@ -132,18 +145,22 @@ func (o *tlsTapper46Objects) Close() error {
 //
 // It can be passed to loadTlsTapper46Objects or ebpf.CollectionSpec.LoadAndAssign.
 type tlsTapper46Maps struct {
-	AcceptSyscallContext *ebpf.Map `ebpf:"accept_syscall_context"`
+	AcceptSyscallContext     *ebpf.Map `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.Map `ebpf:"chunks_buffer"`
+	ChunksBuffer             *ebpf.Map `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.Map `ebpf:"connect_syscall_info"`
+	ConnectSyscallInfo       *ebpf.Map `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"`
+	ConnectionContext        *ebpf.Map `ebpf:"connection_context"`
-	GoReadContext        *ebpf.Map `ebpf:"go_read_context"`
+	GoKernelReadContext      *ebpf.Map `ebpf:"go_kernel_read_context"`
-	GoWriteContext       *ebpf.Map `ebpf:"go_write_context"`
+	GoKernelWriteContext     *ebpf.Map `ebpf:"go_kernel_write_context"`
-	GoidOffsetsMap       *ebpf.Map `ebpf:"goid_offsets_map"`
+	GoReadContext            *ebpf.Map `ebpf:"go_read_context"`
-	Heap                 *ebpf.Map `ebpf:"heap"`
+	GoUserKernelReadContext  *ebpf.Map `ebpf:"go_user_kernel_read_context"`
-	LogBuffer            *ebpf.Map `ebpf:"log_buffer"`
+	GoUserKernelWriteContext *ebpf.Map `ebpf:"go_user_kernel_write_context"`
-	OpensslReadContext   *ebpf.Map `ebpf:"openssl_read_context"`
+	GoWriteContext           *ebpf.Map `ebpf:"go_write_context"`
-	OpensslWriteContext  *ebpf.Map `ebpf:"openssl_write_context"`
+	GoidOffsetsMap           *ebpf.Map `ebpf:"goid_offsets_map"`
-	PidsMap              *ebpf.Map `ebpf:"pids_map"`
+	Heap                     *ebpf.Map `ebpf:"heap"`
+	LogBuffer                *ebpf.Map `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.Map `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.Map `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.Map `ebpf:"pids_map"`
 }
 
 func (m *tlsTapper46Maps) Close() error {
@@ -151,8 +168,12 @@ func (m *tlsTapper46Maps) Close() error {
 		m.AcceptSyscallContext,
 		m.ChunksBuffer,
 		m.ConnectSyscallInfo,
-		m.FileDescriptorToIpv4,
+		m.ConnectionContext,
+		m.GoKernelReadContext,
+		m.GoKernelWriteContext,
 		m.GoReadContext,
+		m.GoUserKernelReadContext,
+		m.GoUserKernelWriteContext,
 		m.GoWriteContext,
 		m.GoidOffsetsMap,
 		m.Heap,
@@ -189,6 +210,10 @@ type tlsTapper46Programs struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.Program `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.Program `ebpf:"sys_exit_write"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapper46Programs) Close() error {
@@ -215,6 +240,10 @@ func (p *tlsTapper46Programs) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.SysExitRead,
+		p.SysExitWrite,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

tap/tlstapper/tlstapper_bpfel_arm64.go

@@ -19,15 +19,21 @@ type tlsTapperGoidOffsets struct {
 }
 
 type tlsTapperTlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Mode  int32
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper returns the embedded CollectionSpec for tlsTapper.
@@ -93,6 +99,8 @@ type tlsTapperProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapperMapSpecs contains maps before they are loaded into the kernel.
@@ -189,6 +197,8 @@ type tlsTapperPrograms struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapperPrograms) Close() error {
@@ -215,6 +225,8 @@ func (p *tlsTapperPrograms) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

tap/tlstapper/tlstapper_bpfel_x86.go

@@ -19,15 +19,20 @@ type tlsTapperGoidOffsets struct {
 }
 
 type tlsTapperTlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper returns the embedded CollectionSpec for tlsTapper.
@@ -93,24 +98,32 @@ type tlsTapperProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.ProgramSpec `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.ProgramSpec `ebpf:"sys_exit_write"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapperMapSpecs contains maps before they are loaded into the kernel.
 //
 // It can be passed ebpf.CollectionSpec.Assign.
 type tlsTapperMapSpecs struct {
-	AcceptSyscallContext *ebpf.MapSpec `ebpf:"accept_syscall_context"`
+	AcceptSyscallContext     *ebpf.MapSpec `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.MapSpec `ebpf:"chunks_buffer"`
+	ChunksBuffer             *ebpf.MapSpec `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.MapSpec `ebpf:"connect_syscall_info"`
+	ConnectSyscallInfo       *ebpf.MapSpec `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"`
+	ConnectionContext        *ebpf.MapSpec `ebpf:"connection_context"`
-	GoReadContext        *ebpf.MapSpec `ebpf:"go_read_context"`
+	GoKernelReadContext      *ebpf.MapSpec `ebpf:"go_kernel_read_context"`
-	GoWriteContext       *ebpf.MapSpec `ebpf:"go_write_context"`
+	GoKernelWriteContext     *ebpf.MapSpec `ebpf:"go_kernel_write_context"`
-	GoidOffsetsMap       *ebpf.MapSpec `ebpf:"goid_offsets_map"`
+	GoReadContext            *ebpf.MapSpec `ebpf:"go_read_context"`
-	Heap                 *ebpf.MapSpec `ebpf:"heap"`
+	GoUserKernelReadContext  *ebpf.MapSpec `ebpf:"go_user_kernel_read_context"`
-	LogBuffer            *ebpf.MapSpec `ebpf:"log_buffer"`
+	GoUserKernelWriteContext *ebpf.MapSpec `ebpf:"go_user_kernel_write_context"`
-	OpensslReadContext   *ebpf.MapSpec `ebpf:"openssl_read_context"`
+	GoWriteContext           *ebpf.MapSpec `ebpf:"go_write_context"`
-	OpensslWriteContext  *ebpf.MapSpec `ebpf:"openssl_write_context"`
+	GoidOffsetsMap           *ebpf.MapSpec `ebpf:"goid_offsets_map"`
-	PidsMap              *ebpf.MapSpec `ebpf:"pids_map"`
+	Heap                     *ebpf.MapSpec `ebpf:"heap"`
+	LogBuffer                *ebpf.MapSpec `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.MapSpec `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.MapSpec `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.MapSpec `ebpf:"pids_map"`
 }
 
 // tlsTapperObjects contains all objects after they have been loaded into the kernel.
@@ -132,18 +145,22 @@ func (o *tlsTapperObjects) Close() error {
 //
 // It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign.
 type tlsTapperMaps struct {
-	AcceptSyscallContext *ebpf.Map `ebpf:"accept_syscall_context"`
+	AcceptSyscallContext     *ebpf.Map `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.Map `ebpf:"chunks_buffer"`
+	ChunksBuffer             *ebpf.Map `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.Map `ebpf:"connect_syscall_info"`
+	ConnectSyscallInfo       *ebpf.Map `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"`
+	ConnectionContext        *ebpf.Map `ebpf:"connection_context"`
-	GoReadContext        *ebpf.Map `ebpf:"go_read_context"`
+	GoKernelReadContext      *ebpf.Map `ebpf:"go_kernel_read_context"`
-	GoWriteContext       *ebpf.Map `ebpf:"go_write_context"`
+	GoKernelWriteContext     *ebpf.Map `ebpf:"go_kernel_write_context"`
-	GoidOffsetsMap       *ebpf.Map `ebpf:"goid_offsets_map"`
+	GoReadContext            *ebpf.Map `ebpf:"go_read_context"`
-	Heap                 *ebpf.Map `ebpf:"heap"`
+	GoUserKernelReadContext  *ebpf.Map `ebpf:"go_user_kernel_read_context"`
-	LogBuffer            *ebpf.Map `ebpf:"log_buffer"`
+	GoUserKernelWriteContext *ebpf.Map `ebpf:"go_user_kernel_write_context"`
-	OpensslReadContext   *ebpf.Map `ebpf:"openssl_read_context"`
+	GoWriteContext           *ebpf.Map `ebpf:"go_write_context"`
-	OpensslWriteContext  *ebpf.Map `ebpf:"openssl_write_context"`
+	GoidOffsetsMap           *ebpf.Map `ebpf:"goid_offsets_map"`
-	PidsMap              *ebpf.Map `ebpf:"pids_map"`
+	Heap                     *ebpf.Map `ebpf:"heap"`
+	LogBuffer                *ebpf.Map `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.Map `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.Map `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.Map `ebpf:"pids_map"`
 }
 
 func (m *tlsTapperMaps) Close() error {
@@ -151,8 +168,12 @@ func (m *tlsTapperMaps) Close() error {
 		m.AcceptSyscallContext,
 		m.ChunksBuffer,
 		m.ConnectSyscallInfo,
-		m.FileDescriptorToIpv4,
+		m.ConnectionContext,
+		m.GoKernelReadContext,
+		m.GoKernelWriteContext,
 		m.GoReadContext,
+		m.GoUserKernelReadContext,
+		m.GoUserKernelWriteContext,
 		m.GoWriteContext,
 		m.GoidOffsetsMap,
 		m.Heap,
@@ -189,6 +210,10 @@ type tlsTapperPrograms struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.Program `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.Program `ebpf:"sys_exit_write"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapperPrograms) Close() error {
@@ -215,6 +240,10 @@ func (p *tlsTapperPrograms) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.SysExitRead,
+		p.SysExitWrite,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

ui-common/package.json

@@ -26,14 +26,15 @@
     "@craco/craco": "^6.4.3",
     "@types/jest": "^26.0.24",
     "@types/node": "^12.20.54",
-    "sass": "^1.52.3",
     "react": "^17.0.2",
     "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "^17.0.2",
-    "recoil": "^0.7.2"
+    "recoil": "^0.7.2",
+    "sass": "^1.52.3"
   },
   "dependencies": {
     "@craco/craco": "^6.4.3",
+    "@elastic/eui": "^60.2.0",
     "@emotion/react": "^11.9.0",
     "@emotion/styled": "^11.8.1",
     "@mui/icons-material": "^5.8.2",
@@ -65,12 +66,14 @@
     "recharts": "^2.1.10",
     "redoc": "^2.0.0-rc.71",
     "styled-components": "^5.3.5",
+    "use-file-picker": "^1.4.2",
     "web-vitals": "^2.1.4",
     "xml-formatter": "^2.6.1"
   },
   "devDependencies": {
     "@rollup/plugin-node-resolve": "^13.3.0",
     "@svgr/rollup": "^6.2.1",
+    "@types/ace": "^0.0.48",
     "cross-env": "^7.0.3",
     "env-cmd": "^10.1.0",
     "gh-pages": "^4.0.0",
@@ -83,7 +86,7 @@
     "rollup-plugin-postcss": "^4.0.2",
     "rollup-plugin-sass": "^1.2.12",
     "rollup-plugin-scss": "^3.0.0",
-    "typescript": "^4.7.2"
+    "typescript": "^4.5.3"
   },
   "eslintConfig": {
     "extends": [
@@ -92,6 +95,7 @@
     ]
   },
   "files": [
+    "src/*.scss",
     "dist"
   ]
 }

ui-common/src/components.scss

@@ -17,6 +17,6 @@
       width: 100%;
       width: -moz-available;
       width: -webkit-fill-available;
-      width: strech;
+      width: stretch;
     }
 }

ui-common/src/components/EntriesList/EntriesList.tsx

@@ -1,20 +1,20 @@
-import React, {useCallback, useEffect, useMemo, useState} from "react";
+import React, { useCallback, useEffect, useMemo, useState } from "react";
 import styles from './EntriesList.module.sass';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
-import Moment from 'moment';
+import { EntryItem } from "../EntryListItem/EntryListItem";
-import {EntryItem} from "../EntryListItem/EntryListItem";
 import down from "assets/downImg.svg";
 import spinner from 'assets/spinner.svg';
-import {RecoilState, useRecoilState, useRecoilValue, useSetRecoilState} from "recoil";
+import { RecoilState, useRecoilState, useRecoilValue, useSetRecoilState } from "recoil";
 import entriesAtom from "../../recoil/entries";
 import queryAtom from "../../recoil/query";
 import TrafficViewerApiAtom from "../../recoil/TrafficViewerApi";
 import TrafficViewerApi from "../TrafficViewer/TrafficViewerApi";
 import focusedEntryIdAtom from "../../recoil/focusedEntryId";
-import {toast} from "react-toastify";
+import { toast } from "react-toastify";
-import {MAX_ENTRIES, TOAST_CONTAINER_ID} from "../../configs/Consts";
+import { MAX_ENTRIES, TOAST_CONTAINER_ID } from "../../configs/Consts";
 import tappingStatusAtom from "../../recoil/tappingStatus";
 import leftOffTopAtom from "../../recoil/leftOffTop";
+import Moment from "moment";
 
 interface EntriesListProps {
   listEntryREF: any;

ui-common/src/components/EntryDetailed/EntryViewer/AutoRepresentation.tsx

@@ -32,7 +32,7 @@ export const AutoRepresentation: React.FC<any> = ({ representation, color, opene
                 badge: null
             }]
 
-        if (response) {
+        if (response && response.length > 0) {
             arr.push({
                 tab: 'Response',
                 badge: null
@@ -71,7 +71,7 @@ export const AutoRepresentation: React.FC<any> = ({ representation, color, opene
             {getOpenedTabIndex() === TabsEnum.Request && <React.Fragment>
                 <SectionsRepresentation data={request} color={color} requestRepresentation={request} />
             </React.Fragment>}
-            {response && getOpenedTabIndex() === TabsEnum.Response && <React.Fragment>
+            {response && response.length > 0 && getOpenedTabIndex() === TabsEnum.Response && <React.Fragment>
                 <SectionsRepresentation data={response} color={color} />
             </React.Fragment>}
         </div>}

ui-common/src/components/UI/CodeEditor/CodeEditor.tsx

@@ -37,11 +37,6 @@ const CodeEditor: React.FC<CodeEditorProps> = ({
             theme="github"
             onChange={onChange}
             editorProps={{ $blockScrolling: true }}
-            setOptions={{
-                enableBasicAutocompletion: true,
-                enableLiveAutocompletion: true,
-                enableSnippets: true
-            }}
             showPrintMargin={false}
             value={code}
             width="100%"

ui-common/src/components/UI/FilePicker/FilePicker.tsx

@@ -0,0 +1,33 @@
+import React from 'react';
+import { useEffect } from 'react';
+import { useFilePicker } from 'use-file-picker';
+import { FileContent } from 'use-file-picker/dist/interfaces';
+
+interface IFilePickerProps {
+    onLoadingComplete: (file: FileContent) => void;
+    elem: any
+}
+
+const FilePicker = ({ elem, onLoadingComplete }: IFilePickerProps) => {
+    const [openFileSelector, { filesContent }] = useFilePicker({
+        accept: ['.json'],
+        limitFilesConfig: { max: 1 },
+        maxFileSize: 1
+    });
+
+    const onFileSelectorClick = (e) => {
+        e.preventDefault();
+        e.stopPropagation();
+        openFileSelector();
+    }
+
+    useEffect(() => {
+        filesContent.length && onLoadingComplete(filesContent[0])
+    }, [filesContent, onLoadingComplete]);
+
+    return (<React.Fragment>
+        {React.cloneElement(elem, { onClick: onFileSelectorClick })}
+    </React.Fragment>)
+}
+
+export default FilePicker;