Develop -> main (version 36.0) #major

Fetch source and destination addresses with bpf from tcp kprobes, similar to how it is done for openssl lib.
Chunk contains both source address and destination address.
FD is no longer used to obtain addresses.

.github/workflows/static_code_analysis.yml

@@ -32,6 +32,10 @@ jobs:
         id: agent_modified_files
         run: devops/check_modified_files.sh agent/
 
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
       - name: Go lint - agent
         uses: golangci/golangci-lint-action@v2
         if: steps.agent_modified_files.outputs.matched == 'true'

.github/workflows/test.yml

@@ -40,6 +40,10 @@ jobs:
         run: |
           ./devops/install-capstone.sh
 
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
       - name: Check CLI modified files
         id: cli_modified_files
         run: devops/check_modified_files.sh cli/

.gitignore

@@ -56,3 +56,6 @@ tap/extensions/*/expect
 
 # Ignore *.log files
 *.log
+
+# Object files
+*.o

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent agent-debug cli tap docker
+.PHONY: help ui agent agent-debug cli tap docker bpf clean-bpf
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -20,6 +20,13 @@ TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
 export VER?=0.0
+ARCH=$(shell uname -m)
+ifeq ($(ARCH),$(filter $(ARCH),aarch64 arm64))
+	BPF_O_ARCH_LABEL=arm64
+else
+	BPF_O_ARCH_LABEL=x86
+endif
+BPF_O_FILES = tap/tlstapper/tlstapper46_bpfel_$(BPF_O_ARCH_LABEL).o tap/tlstapper/tlstapper_bpfel_$(BPF_O_ARCH_LABEL).o
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -31,11 +38,17 @@ cli: ## Build CLI.
 cli-debug: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build-debug
 
-agent: ## Build agent.
+agent: bpf ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
 	@ls -l agent/build
 
+bpf: $(BPF_O_FILES)
+
+$(BPF_O_FILES): $(wildcard tap/tlstapper/bpf/**/*.[ch])
+	@(echo "building tlstapper bpf")
+	@(./tap/tlstapper/bpf-builder/build.sh)
+
 agent-debug: ## Build agent for debug.
 	@(echo "building mizu agent for debug.." )
 	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
@@ -76,6 +89,9 @@ clean-cli:  ## Clean CLI.
 clean-docker:  ## Run clean docker
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+clean-bpf:
+	@(rm $(BPF_O_FILES) ; echo "bpf cleanup done" )
+
 test-lint:  ## Run lint on all modules
 	cd agent && golangci-lint run
 	cd shared && golangci-lint run

acceptanceTests/cypress.config.js

@@ -11,7 +11,6 @@ module.exports = defineConfig({
     testUrl: 'http://localhost:8899/',
     redactHeaderContent: 'User-Header[REDACTED]',
     redactBodyContent: '{ "User": "[REDACTED]" }',
-    regexMaskingBodyContent: '[REDACTED]',
     greenFilterColor: 'rgb(210, 250, 210)',
     redFilterColor: 'rgb(250, 214, 220)',
     bodyJsonClass: '.hljs',

acceptanceTests/cypress/e2e/tests/RegexMasking.js

@@ -1,7 +0,0 @@
-import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/go.mod

@@ -18,7 +18,6 @@ require (
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-logr/logr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -29,7 +28,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/up9inc/mizu/logger v0.0.0 // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect

acceptanceTests/go.sum

@@ -206,7 +206,6 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

acceptanceTests/setup.sh

@@ -27,7 +27,7 @@ else
 fi
 
 echo "Starting minikube..."
-minikube start --cpus 2 --memory 6946
+minikube start --cpus 2 --memory 6000
 
 echo "Creating mizu tests namespaces"
 kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -

acceptanceTests/tap_test.go

@@ -2,10 +2,8 @@ package acceptanceTests
 
 import (
 	"archive/zip"
-	"bytes"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os/exec"
 	"path"
 	"strings"
@@ -343,7 +341,7 @@ func TestTapRedact(t *testing.T) {
 
 	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-	tapCmdArgs = append(tapCmdArgs, "--redact")
+	tapCmdArgs = append(tapCmdArgs, "--redact", "--set", "tap.redact-patterns.request-headers=User-Header", "--set", "tap.redact-patterns.request-body=User")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
@@ -429,60 +427,6 @@ func TestTapNoRedact(t *testing.T) {
 	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/NoRedact.js\"")
 }
 
-func TestTapRegexMasking(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := GetCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := GetDefaultTapCommandArgs()
-
-	tapNamespace := GetDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmdArgs = append(tapCmdArgs, "--redact")
-
-	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
-
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
-	for i := 0; i < DefaultEntriesCount; i++ {
-		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
-		if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
-			t.Errorf("failed to send proxy request, err: %v", requestErr)
-			return
-		}
-	}
-
-	RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"")
-
-}
-
 func TestTapIgnoredUserAgents(t *testing.T) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")

acceptanceTests/testsUtils.go

@@ -215,12 +215,11 @@ func DeleteKubeFile(kubeContext string, namespace string, filename string) error
 func getDefaultCommandArgs() []string {
 	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
 	setFlag := "--set"
-	telemetry := "telemetry=false"
 	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
-	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
+	return []string{setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
 func GetDefaultTapCommandArgs() []string {

agent/go.mod

@@ -48,7 +48,6 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/beevik/etree v1.1.0 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
 	github.com/cilium/ebpf v0.9.0 // indirect

agent/go.sum

@@ -101,7 +101,6 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

agent/pkg/api/socket_routes.go

@@ -97,7 +97,9 @@ func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool
 	websocketIdsLock.Unlock()
 
 	defer func() {
-		socketCleanup(socketId, connectedWebsockets[socketId])
+		if socketConnection := connectedWebsockets[socketId]; socketConnection != nil {
+			socketCleanup(socketId, socketConnection)
+		}
 	}()
 
 	eventHandlers.WebSocketConnect(c, socketId, isTapper)

agent/pkg/controllers/status_controller.go

@@ -1,7 +1,10 @@
 package controllers
 
 import (
+	"fmt"
 	"net/http"
+	"strconv"
+	"time"
 
 	core "k8s.io/api/core/v1"
 
@@ -80,7 +83,24 @@ func GetGeneralStats(c *gin.Context) {
 }
 
 func GetTrafficStats(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetTrafficStats())
+	startTime, endTime, err := getStartEndTime(c)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, providers.GetTrafficStats(startTime, endTime))
+}
+
+func getStartEndTime(c *gin.Context) (time.Time, time.Time, error) {
+	startTimeValue, err := strconv.Atoi(c.Query("startTimeMs"))
+	if err != nil {
+		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid start time: %v", err)
+	}
+	endTimeValue, err := strconv.Atoi(c.Query("endTimeMs"))
+	if err != nil {
+		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid end time: %v", err)
+	}
+	return time.UnixMilli(int64(startTimeValue)), time.UnixMilli(int64(endTimeValue)), nil
 }
 
 func GetCurrentResolvingInformation(c *gin.Context) {

agent/pkg/har/utils.go

@@ -11,75 +11,30 @@ import (
 	"github.com/up9inc/mizu/logger"
 )
 
-// Keep it because we might want cookies in the future
-//func BuildCookies(rawCookies []interface{}) []har.Cookie {
-//	cookies := make([]har.Cookie, 0, len(rawCookies))
-//
-//	for _, cookie := range rawCookies {
-//		c := cookie.(map[string]interface{})
-//		expiresStr := ""
-//		if c["expires"] != nil {
-//			expiresStr = c["expires"].(string)
-//		}
-//		expires, _ := time.Parse(time.RFC3339, expiresStr)
-//		httpOnly := false
-//		if c["httponly"] != nil {
-//			httpOnly, _ = strconv.ParseBool(c["httponly"].(string))
-//		}
-//		secure := false
-//		if c["secure"] != nil {
-//			secure, _ = strconv.ParseBool(c["secure"].(string))
-//		}
-//		path := ""
-//		if c["path"] != nil {
-//			path = c["path"].(string)
-//		}
-//		domain := ""
-//		if c["domain"] != nil {
-//			domain = c["domain"].(string)
-//		}
-//
-//		cookies = append(cookies, har.Cookie{
-//			Name:        c["name"].(string),
-//			Value:       c["value"].(string),
-//			Path:        path,
-//			Domain:      domain,
-//			HTTPOnly:    httpOnly,
-//			Secure:      secure,
-//			Expires:     expires,
-//			Expires8601: expiresStr,
-//		})
-//	}
-//
-//	return cookies
-//}
-
-func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) {
+func BuildHeaders(rawHeaders map[string]interface{}) ([]Header, string, string, string, string, string) {
 	var host, scheme, authority, path, status string
 	headers := make([]Header, 0, len(rawHeaders))
 
-	for _, header := range rawHeaders {
-		h := header.(map[string]interface{})
-
+	for key, value := range rawHeaders {
 		headers = append(headers, Header{
-			Name:  h["name"].(string),
-			Value: h["value"].(string),
+			Name:  key,
+			Value: value.(string),
 		})
 
-		if h["name"] == "Host" {
-			host = h["value"].(string)
+		if key == "Host" {
+			host = value.(string)
 		}
-		if h["name"] == ":authority" {
-			authority = h["value"].(string)
+		if key == ":authority" {
+			authority = value.(string)
 		}
-		if h["name"] == ":scheme" {
-			scheme = h["value"].(string)
+		if key == ":scheme" {
+			scheme = value.(string)
 		}
-		if h["name"] == ":path" {
-			path = h["value"].(string)
+		if key == ":path" {
+			path = value.(string)
 		}
-		if h["name"] == ":status" {
-			status = h["value"].(string)
+		if key == ":status" {
+			status = value.(string)
 		}
 	}
 
@@ -119,8 +74,8 @@ func BuildPostParams(rawParams []interface{}) []Param {
 }
 
 func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
-	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
-	cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
+	headers, host, scheme, authority, path, _ := BuildHeaders(request["headers"].(map[string]interface{}))
+	cookies := make([]Cookie, 0)
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
@@ -134,12 +89,20 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 	}
 
 	queryString := make([]QueryString, 0)
-	for _, _qs := range request["_queryString"].([]interface{}) {
-		qs := _qs.(map[string]interface{})
-		queryString = append(queryString, QueryString{
-			Name:  qs["name"].(string),
-			Value: qs["value"].(string),
-		})
+	for key, value := range request["queryString"].(map[string]interface{}) {
+		if valuesInterface, ok := value.([]interface{}); ok {
+			for _, valueInterface := range valuesInterface {
+				queryString = append(queryString, QueryString{
+					Name:  key,
+					Value: valueInterface.(string),
+				})
+			}
+		} else {
+			queryString = append(queryString, QueryString{
+				Name:  key,
+				Value: value.(string),
+			})
+		}
 	}
 
 	url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
@@ -172,8 +135,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 }
 
 func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
-	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
-	cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
+	headers, _, _, _, _, _status := BuildHeaders(response["headers"].(map[string]interface{}))
+	cookies := make([]Cookie, 0)
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]

agent/pkg/providers/stats_provider.go

@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"reflect"
-	"strings"
 	"sync"
 	"time"
 
@@ -82,13 +81,12 @@ func GetGeneralStats() *GeneralStats {
 
 func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
 	for item, value := range protocolMap {
-		splitted := strings.SplitN(item, "/", 3)
-		protocolToColor[splitted[len(splitted)-1]] = value.BackgroundColor
+		protocolToColor[api.GetProtocolSummary(item).Abbreviation] = value.BackgroundColor
 	}
 }
 
-func GetTrafficStats() *TrafficStatsResponse {
-	bucketsStatsCopy := getBucketStatsCopy()
+func GetTrafficStats(startTime time.Time, endTime time.Time) *TrafficStatsResponse {
+	bucketsStatsCopy := getFilteredBucketStatsCopy(startTime, endTime)
 
 	return &TrafficStatsResponse{
 		Protocols:     getAvailableProtocols(bucketsStatsCopy),
@@ -264,7 +262,7 @@ func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string
 	return protocolsData
 }
 
-func getBucketStatsCopy() BucketStats {
+func getFilteredBucketStatsCopy(startTime time.Time, endTime time.Time) BucketStats {
 	bucketStatsCopy := BucketStats{}
 	bucketStatsLocker.Lock()
 	if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
@@ -272,7 +270,18 @@ func getBucketStatsCopy() BucketStats {
 		return nil
 	}
 	bucketStatsLocker.Unlock()
-	return bucketStatsCopy
+
+	filteredBucketStatsCopy := BucketStats{}
+	interval := InternalBucketThreshold
+
+	for _, bucket := range bucketStatsCopy {
+		if (bucket.BucketTime.After(startTime.Add(-1*interval/2).Round(interval)) && bucket.BucketTime.Before(endTime.Add(-1*interval/2).Round(interval))) ||
+			bucket.BucketTime.Equal(startTime.Add(-1*interval/2).Round(interval)) ||
+			bucket.BucketTime.Equal(endTime.Add(-1*interval/2).Round(interval)) {
+			filteredBucketStatsCopy = append(filteredBucketStatsCopy, bucket)
+		}
+	}
+	return filteredBucketStatsCopy
 }
 
 func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {

cli/apiserver/provider.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"time"
 
 	"github.com/up9inc/mizu/cli/utils"
@@ -93,45 +91,3 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 		}
 	}
 }
-
-func (provider *Provider) GetGeneralStats() (map[string]interface{}, error) {
-	generalStatsUrl := fmt.Sprintf("%s/status/general", provider.url)
-
-	response, requestErr := utils.Get(generalStatsUrl, provider.client)
-	if requestErr != nil {
-		return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr)
-	}
-
-	defer response.Body.Close()
-
-	data, readErr := ioutil.ReadAll(response.Body)
-	if readErr != nil {
-		return nil, fmt.Errorf("failed to read general stats for telemetry, err: %v", readErr)
-	}
-
-	var generalStats map[string]interface{}
-	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
-		return nil, fmt.Errorf("failed to parse general stats for telemetry, err: %v", parseErr)
-	}
-	return generalStats, nil
-}
-
-func (provider *Provider) GetVersion() (string, error) {
-	versionUrl, _ := url.Parse(fmt.Sprintf("%s/metadata/version", provider.url))
-	req := &http.Request{
-		Method: http.MethodGet,
-		URL:    versionUrl,
-	}
-	statusResp, err := utils.Do(req, provider.client)
-	if err != nil {
-		return "", err
-	}
-	defer statusResp.Body.Close()
-
-	versionResponse := &shared.VersionResponse{}
-	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
-		return "", err
-	}
-
-	return versionResponse.Ver, nil
-}

cli/cmd/check.go

@@ -4,7 +4,6 @@ import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -12,7 +11,6 @@ var checkCmd = &cobra.Command{
 	Use:   "check",
 	Short: "Check the Mizu installation for potential problems",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("check", nil)
 		runMizuCheck()
 		return nil
 	},

cli/cmd/clean.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/telemetry"
 )
 
 var cleanCmd = &cobra.Command{
 	Use:   "clean",
 	Short: "Removes all mizu resources",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("clean", nil)
 		performCleanCommand()
 		return nil
 	},

cli/cmd/config.go

@@ -7,7 +7,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/logger"
 )
@@ -16,8 +15,6 @@ var configCmd = &cobra.Command{
 	Use:   "config",
 	Short: "Generate config with default values",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("config", config.Config.Config)
-
 		configWithDefaults, err := config.GetConfigWithDefaults()
 		if err != nil {
 			logger.Log.Errorf("Failed generating config with defaults, err: %v", err)

cli/cmd/install.go

@@ -4,7 +4,6 @@ import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -12,7 +11,6 @@ var installCmd = &cobra.Command{
 	Use:   "install",
 	Short: "Installs mizu components",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("install", nil)
 		runMizuInstall()
 		return nil
 	},

cli/cmd/logs.go

@@ -9,7 +9,6 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -17,8 +16,6 @@ var logsCmd = &cobra.Command{
 	Use:   "logs",
 	Short: "Create a zip file with logs for Github issue or troubleshoot",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("logs", config.Config.Logs)
-
 		kubernetesProvider, err := getKubernetesProviderForCli()
 		if err != nil {
 			return nil

cli/cmd/tap.go

@@ -48,7 +48,6 @@ func init() {
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
-	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.EnableRedactionTapName, defaultTapConfig.EnableRedaction, "Enables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.")

cli/cmd/tapRunner.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/up9inc/mizu/cli/resources"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/utils"
 
 	core "k8s.io/api/core/v1"
@@ -109,8 +108,6 @@ func RunMizuTap() {
 }
 
 func finishTapExecution(kubernetesProvider *kubernetes.Provider) {
-	telemetry.ReportTapTelemetry(apiProvider, config.Config.Tap, state.startTime)
-
 	finishMizuExecution(kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
 }
 
@@ -126,7 +123,6 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		AgentDatabasePath:      shared.DataDirPath,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
-		Telemetry:              config.Config.Telemetry,
 	}
 
 	return &mizuAgentConfig
@@ -151,17 +147,18 @@ func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.
 	}
 }
 
-func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error {
+func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, startTime time.Time) error {
 	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
-		TargetNamespaces:         targetNamespaces,
-		PodFilterRegex:           *config.Config.Tap.PodRegex(),
-		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
-		AgentImage:               config.Config.AgentImage,
-		TapperResources:          config.Config.Tap.TapperResources,
-		ImagePullPolicy:          config.Config.ImagePullPolicy(),
-		LogLevel:                 config.Config.LogLevel(),
-		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
-		MizuApiFilteringOptions:  mizuApiFilteringOptions,
+		TargetNamespaces:       targetNamespaces,
+		PodFilterRegex:         *config.Config.Tap.PodRegex(),
+		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
+		AgentImage:             config.Config.AgentImage,
+		TapperResources:        config.Config.Tap.TapperResources,
+		ImagePullPolicy:        config.Config.ImagePullPolicy(),
+		LogLevel:               config.Config.LogLevel(),
+		MizuApiFilteringOptions: api.TrafficFilteringOptions{
+			IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
+		},
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
 		ServiceMesh:              config.Config.Tap.ServiceMesh,
 		Tls:                      config.Config.Tap.Tls,
@@ -229,27 +226,6 @@ func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError)
 	}
 }
 
-func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
-	var compiledRegexSlice []*api.SerializableRegexp
-
-	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
-		compiledRegexSlice = make([]*api.SerializableRegexp, 0)
-		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
-			compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr)
-			if err != nil {
-				return nil, err
-			}
-			compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
-		}
-	}
-
-	return &api.TrafficFilteringOptions{
-		PlainTextMaskingRegexes: compiledRegexSlice,
-		IgnoredUserAgents:       config.Config.Tap.IgnoredUserAgents,
-		EnableRedaction:        config.Config.Tap.EnableRedaction,
-	}, nil
-}
-
 func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName))
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
@@ -367,8 +343,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel, config.Config.Tap.GuiPort)
 
-	options, _ := getMizuApiFilteringOptions()
-	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
+	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", errormessage.FormatError(err)))
 		cancel()
 	}

cli/cmd/version.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 
 	"github.com/creasty/defaults"
@@ -18,8 +17,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("version", config.Config.Version)
-
 		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
 			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.Ver, mizu.Branch, mizu.GitCommitHash)

cli/cmd/view.go

@@ -3,9 +3,7 @@ package cmd
 import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -13,7 +11,6 @@ var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("view", config.Config.View)
 		runMizuView()
 		return nil
 	},

cli/config/configStruct.go

@@ -31,7 +31,6 @@ type ConfigStruct struct {
 	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
-	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
 	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
 	KubeContext            string                      `yaml:"kube-context"`

cli/config/configStructs/tapConfig.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strings"
 
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
@@ -15,38 +16,43 @@ import (
 )
 
 const (
-	GuiPortTapName                = "gui-port"
-	NamespacesTapName             = "namespaces"
-	AllNamespacesTapName          = "all-namespaces"
-	PlainTextFilterRegexesTapName = "regex-masking"
-	EnableRedactionTapName        = "redact"
-	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
-	InsertionFilterName           = "insertion-filter"
-	DryRunTapName                 = "dry-run"
-	ServiceMeshName               = "service-mesh"
-	TlsName                       = "tls"
-	ProfilerName                  = "profiler"
-	MaxLiveStreamsName            = "max-live-streams"
+	GuiPortTapName               = "gui-port"
+	NamespacesTapName            = "namespaces"
+	AllNamespacesTapName         = "all-namespaces"
+	EnableRedactionTapName       = "redact"
+	HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
+	InsertionFilterName          = "insertion-filter"
+	DryRunTapName                = "dry-run"
+	ServiceMeshName              = "service-mesh"
+	TlsName                      = "tls"
+	ProfilerName                 = "profiler"
+	MaxLiveStreamsName           = "max-live-streams"
 )
 
 type TapConfig struct {
-	PodRegexStr            string           `yaml:"regex" default:".*"`
-	GuiPort                uint16           `yaml:"gui-port" default:"8899"`
-	ProxyHost              string           `yaml:"proxy-host" default:"127.0.0.1"`
-	Namespaces             []string         `yaml:"namespaces"`
-	AllNamespaces          bool             `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string         `yaml:"regex-masking"`
-	IgnoredUserAgents      []string         `yaml:"ignored-user-agents"`
-	EnableRedaction        bool             `yaml:"redact" default:"false"`
-	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
-	InsertionFilter        string           `yaml:"insertion-filter" default:""`
-	DryRun                 bool             `yaml:"dry-run" default:"false"`
-	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
-	TapperResources        shared.Resources `yaml:"tapper-resources"`
-	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
-	Tls                    bool             `yaml:"tls" default:"false"`
-	Profiler               bool             `yaml:"profiler" default:"false"`
-	MaxLiveStreams         int              `yaml:"max-live-streams" default:"500"`
+	PodRegexStr       string   `yaml:"regex" default:".*"`
+	GuiPort           uint16   `yaml:"gui-port" default:"8899"`
+	ProxyHost         string   `yaml:"proxy-host" default:"127.0.0.1"`
+	Namespaces        []string `yaml:"namespaces"`
+	AllNamespaces     bool     `yaml:"all-namespaces" default:"false"`
+	IgnoredUserAgents []string `yaml:"ignored-user-agents"`
+	EnableRedaction   bool     `yaml:"redact" default:"false"`
+	RedactPatterns    struct {
+		RequestHeaders     []string `yaml:"request-headers"`
+		ResponseHeaders    []string `yaml:"response-headers"`
+		RequestBody        []string `yaml:"request-body"`
+		ResponseBody       []string `yaml:"response-body"`
+		RequestQueryParams []string `yaml:"request-query-params"`
+	} `yaml:"redact-patterns"`
+	HumanMaxEntriesDBSize string           `yaml:"max-entries-db-size" default:"200MB"`
+	InsertionFilter       string           `yaml:"insertion-filter" default:""`
+	DryRun                bool             `yaml:"dry-run" default:"false"`
+	ApiServerResources    shared.Resources `yaml:"api-server-resources"`
+	TapperResources       shared.Resources `yaml:"tapper-resources"`
+	ServiceMesh           bool             `yaml:"service-mesh" default:"false"`
+	Tls                   bool             `yaml:"tls" default:"false"`
+	Profiler              bool             `yaml:"profiler" default:"false"`
+	MaxLiveStreams        int              `yaml:"max-live-streams" default:"500"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -71,9 +77,48 @@ func (config *TapConfig) GetInsertionFilter() string {
 			}
 		}
 	}
+
+	redactFilter := getRedactFilter(config)
+	if insertionFilter != "" && redactFilter != "" {
+		return fmt.Sprintf("(%s) and (%s)", insertionFilter, redactFilter)
+	} else if insertionFilter == "" && redactFilter != "" {
+		return redactFilter
+	}
+
 	return insertionFilter
 }
 
+func getRedactFilter(config *TapConfig) string {
+	if !config.EnableRedaction {
+		return ""
+	}
+
+	var redactValues []string
+	for _, requestHeader := range config.RedactPatterns.RequestHeaders {
+		redactValues = append(redactValues, fmt.Sprintf("request.headers['%s']", requestHeader))
+	}
+	for _, responseHeader := range config.RedactPatterns.ResponseHeaders {
+		redactValues = append(redactValues, fmt.Sprintf("response.headers['%s']", responseHeader))
+	}
+
+	for _, requestBody := range config.RedactPatterns.RequestBody {
+		redactValues = append(redactValues, fmt.Sprintf("request.postData.text.json()...%s", requestBody))
+	}
+	for _, responseBody := range config.RedactPatterns.ResponseBody {
+		redactValues = append(redactValues, fmt.Sprintf("response.content.text.json()...%s", responseBody))
+	}
+
+	for _, requestQueryParams := range config.RedactPatterns.RequestQueryParams {
+		redactValues = append(redactValues, fmt.Sprintf("request.queryString['%s']", requestQueryParams))
+	}
+
+	if len(redactValues) == 0 {
+		return ""
+	}
+
+	return fmt.Sprintf("redact(\"%s\")", strings.Join(redactValues, "\",\""))
+}
+
 func (config *TapConfig) Validate() error {
 	_, compileErr := regexp.Compile(config.PodRegexStr)
 	if compileErr != nil {

cli/go.mod

@@ -4,7 +4,6 @@ go 1.17
 
 require (
 	github.com/creasty/defaults v1.5.2
-	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/google/go-github/v37 v37.0.0
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0

cli/go.sum

@@ -145,8 +145,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
-github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
-github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=

cli/mizu/consts.go

@@ -14,8 +14,6 @@ var (
 	Platform                                   = ""
 )
 
-const DEVENVVAR = "MIZU_DISABLE_TELEMTRY"
-
 func GetMizuFolderPath() string {
 	home, homeDirErr := os.UserHomeDir()
 	if homeDirErr != nil {

cli/mizu/version/versionCheck.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"runtime"
 	"strings"
 	"time"
@@ -18,10 +17,6 @@ import (
 )
 
 func CheckNewerVersion(versionChan chan string) {
-	if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
-		versionChan <- ""
-		return
-	}
 	logger.Log.Debugf("Checking for newer version...")
 	start := time.Now()
 	client := github.NewClient(nil)

cli/telemetry/telemetry.go

@@ -1,97 +0,0 @@
-package telemetry
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/denisbrodbeck/machineid"
-	"github.com/up9inc/mizu/cli/apiserver"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/logger"
-)
-
-const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
-
-func ReportRun(cmd string, args interface{}) {
-	if !shouldRunTelemetry() {
-		logger.Log.Debug("not reporting telemetry")
-		return
-	}
-
-	argsBytes, _ := json.Marshal(args)
-	argsMap := map[string]interface{}{
-		"cmd":  cmd,
-		"args": string(argsBytes),
-	}
-
-	if err := sendTelemetry(argsMap); err != nil {
-		logger.Log.Debug(err)
-		return
-	}
-
-	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
-}
-
-func ReportTapTelemetry(apiProvider *apiserver.Provider, args interface{}, startTime time.Time) {
-	if !shouldRunTelemetry() {
-		logger.Log.Debug("not reporting telemetry")
-		return
-	}
-
-	generalStats, err := apiProvider.GetGeneralStats()
-	if err != nil {
-		logger.Log.Debugf("[ERROR] failed to get general stats from api server %v", err)
-		return
-	}
-	argsBytes, _ := json.Marshal(args)
-	argsMap := map[string]interface{}{
-		"cmd":                    "tap",
-		"args":                   string(argsBytes),
-		"executionTimeInSeconds": int(time.Since(startTime).Seconds()),
-		"apiCallsCount":          generalStats["EntriesCount"],
-		"trafficVolumeInGB":      generalStats["EntriesVolumeInGB"],
-	}
-
-	if err := sendTelemetry(argsMap); err != nil {
-		logger.Log.Debug(err)
-		return
-	}
-
-	logger.Log.Debug("successfully reported telemetry of tap command")
-}
-
-func shouldRunTelemetry() bool {
-	if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
-		return false
-	}
-	if !config.Config.Telemetry {
-		return false
-	}
-
-	return mizu.Branch == "main" || mizu.Branch == "develop"
-}
-
-func sendTelemetry(argsMap map[string]interface{}) error {
-	argsMap["component"] = "mizu_cli"
-	argsMap["buildTimestamp"] = mizu.BuildTimestamp
-	argsMap["branch"] = mizu.Branch
-	argsMap["version"] = mizu.Ver
-	argsMap["platform"] = mizu.Platform
-
-	if machineId, err := machineid.ProtectedID("mizu"); err == nil {
-		argsMap["machineId"] = machineId
-	}
-
-	jsonValue, _ := json.Marshal(argsMap)
-
-	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
-		return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp)
-	}
-
-	return nil
-}

performance_analysis/run_tapper_benchmark.sh

@@ -57,7 +57,7 @@ log "Writing output to $MIZU_BENCHMARK_OUTPUT_DIR"
 cd $MIZU_HOME || exit 1
 
 export HOST_MODE=0
-export SENSITIVE_DATA_FILTERING_OPTIONS='{"EnableRedaction": false}'
+export SENSITIVE_DATA_FILTERING_OPTIONS='{}'
 export MIZU_DEBUG_DISABLE_PCAP=false
 export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false
 export MIZU_DEBUG_DISABLE_TCP_STREAM=false

shared/kubernetes/mizuTapperSyncer.go

@@ -43,7 +43,6 @@ type TapperSyncerConfig struct {
 	TapperResources          shared.Resources
 	ImagePullPolicy          core.PullPolicy
 	LogLevel                 logging.Level
-	IgnoredUserAgents        []string
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
 	ServiceMesh              bool

shared/models.go

@@ -43,7 +43,6 @@ type MizuAgentConfig struct {
 	AgentDatabasePath      string        `json:"agentDatabasePath"`
 	ServiceMap             bool          `json:"serviceMap"`
 	OAS                    OASConfig     `json:"oas"`
-	Telemetry              bool          `json:"telemetry"`
 }
 
 type WebSocketMessageMetadata struct {

tap/api/api.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"net"
+	"strings"
 	"sync"
 	"time"
 
@@ -25,6 +26,15 @@ func (protocol *ProtocolSummary) ToString() string {
 	return fmt.Sprintf("%s?%s?%s", protocol.Name, protocol.Version, protocol.Abbreviation)
 }
 
+func GetProtocolSummary(inputString string) *ProtocolSummary {
+	splitted := strings.SplitN(inputString, "?", 3)
+	return &ProtocolSummary{
+		Name:         splitted[0],
+		Version:      splitted[1],
+		Abbreviation: splitted[2],
+	}
+}
+
 type Protocol struct {
 	ProtocolSummary
 	LongName        string   `json:"longName"`

tap/api/options.go

@@ -1,7 +1,5 @@
 package api
 
 type TrafficFilteringOptions struct {
-	IgnoredUserAgents       []string
-	PlainTextMaskingRegexes []*SerializableRegexp
-	EnableRedaction         bool
+	IgnoredUserAgents []string
 }

tap/extensions/amqp/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/amqp/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect16/amqp/\* expect

tap/extensions/amqp/go.mod

@@ -4,16 +4,20 @@ go 1.17
 
 require (
 	github.com/stretchr/testify v1.7.0
+	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
 
+replace github.com/up9inc/mizu/logger v0.0.0 => ../../../logger
+
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api
 
 replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../../dbgctl

tap/extensions/amqp/go.sum

@@ -1,5 +1,7 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

tap/extensions/amqp/helpers.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
-	"time"
 
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
 
@@ -25,14 +25,14 @@ var connectionMethodMap = map[int]string{
 	61: "connection unblocked",
 }
 
-// var channelMethodMap = map[int]string{
-// 	10: "channel open",
-// 	11: "channel open-ok",
-// 	20: "channel flow",
-// 	21: "channel flow-ok",
-// 	40: "channel close",
-// 	41: "channel close-ok",
-// }
+var channelMethodMap = map[int]string{
+	10: "channel open",
+	11: "channel open-ok",
+	20: "channel flow",
+	21: "channel flow-ok",
+	40: "channel close",
+	41: "channel close-ok",
+}
 
 var exchangeMethodMap = map[int]string{
 	10: "exchange declare",
@@ -94,29 +94,41 @@ type AMQPWrapper struct {
 	Details interface{} `json:"details"`
 }
 
-func emitAMQP(event interface{}, _type string, method string, connectionInfo *api.ConnectionInfo, captureTime time.Time, captureSize int, emitter api.Emitter, capture api.Capture) {
-	request := &api.GenericMessage{
-		IsRequest:   true,
-		CaptureTime: captureTime,
-		Payload: AMQPPayload{
-			Data: &AMQPWrapper{
-				Method:  method,
-				Url:     "",
-				Details: event,
-			},
-		},
+type emptyResponse struct {
+}
+
+const emptyMethod = "empty"
+
+func getIdent(reader api.TcpReader, methodFrame *MethodFrame) (ident string) {
+	tcpID := reader.GetTcpID()
+	// To match methods to their Ok(s)
+	methodId := methodFrame.MethodId - methodFrame.MethodId%10
+
+	if reader.GetIsClient() {
+		ident = fmt.Sprintf(
+			"%s_%s_%s_%s_%d_%d_%d",
+			tcpID.SrcIP,
+			tcpID.DstIP,
+			tcpID.SrcPort,
+			tcpID.DstPort,
+			methodFrame.ChannelId,
+			methodFrame.ClassId,
+			methodId,
+		)
+	} else {
+		ident = fmt.Sprintf(
+			"%s_%s_%s_%s_%d_%d_%d",
+			tcpID.DstIP,
+			tcpID.SrcIP,
+			tcpID.DstPort,
+			tcpID.SrcPort,
+			methodFrame.ChannelId,
+			methodFrame.ClassId,
+			methodId,
+		)
 	}
-	item := &api.OutputChannelItem{
-		Protocol:       protocol,
-		Capture:        capture,
-		Timestamp:      captureTime.UnixNano() / int64(time.Millisecond),
-		ConnectionInfo: connectionInfo,
-		Pair: &api.RequestResponsePair{
-			Request:  *request,
-			Response: api.GenericMessage{},
-		},
-	}
-	emitter.Emit(item)
+
+	return
 }
 
 func representProperties(properties map[string]interface{}, rep []interface{}) ([]interface{}, string, string) {
@@ -460,6 +472,36 @@ func representQueueDeclare(event map[string]interface{}) []interface{} {
 	return rep
 }
 
+func representQueueDeclareOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Queue",
+			Value:    event["queue"].(string),
+			Selector: `response.queue`,
+		},
+		{
+			Name:     "Message Count",
+			Value:    fmt.Sprintf("%g", event["messageCount"].(float64)),
+			Selector: `response.messageCount`,
+		},
+		{
+			Name:     "Consumer Count",
+			Value:    fmt.Sprintf("%g", event["consumerCount"].(float64)),
+			Selector: `response.consumerCount`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
 func representExchangeDeclare(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
@@ -571,7 +613,7 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 				x, _ := json.Marshal(value)
 				outcome = string(x)
 			default:
-				panic("Unknown data type for the server property!")
+				logger.Log.Info("Unknown data type for the server property!")
 			}
 			headers = append(headers, api.TableData{
 				Name:     name,
@@ -593,6 +635,65 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 	return rep
 }
 
+func representConnectionStartOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Mechanism",
+			Value:    event["mechanism"].(string),
+			Selector: `response.mechanism`,
+		},
+		{
+			Name:     "Mechanism",
+			Value:    event["mechanism"].(string),
+			Selector: `response.response`,
+		},
+		{
+			Name:     "Locale",
+			Value:    event["locale"].(string),
+			Selector: `response.locale`,
+		},
+	})
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	if event["clientProperties"] != nil {
+		headers := make([]api.TableData, 0)
+		for name, value := range event["clientProperties"].(map[string]interface{}) {
+			var outcome string
+			switch v := value.(type) {
+			case string:
+				outcome = v
+			case map[string]interface{}:
+				x, _ := json.Marshal(value)
+				outcome = string(x)
+			default:
+				logger.Log.Info("Unknown data type for the client property!")
+			}
+			headers = append(headers, api.TableData{
+				Name:     name,
+				Value:    outcome,
+				Selector: fmt.Sprintf(`response.clientProperties["%s"]`, name),
+			})
+		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Client Properties",
+			Data:  string(headersMarshaled),
+		})
+	}
+
+	return rep
+}
+
 func representConnectionClose(event map[string]interface{}) []interface{} {
 	replyCode := ""
 
@@ -750,3 +851,122 @@ func representBasicConsume(event map[string]interface{}) []interface{} {
 
 	return rep
 }
+
+func representBasicConsumeOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `response.consumerTag`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representConnectionOpen(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Virtual Host",
+			Value:    event["virtualHost"].(string),
+			Selector: `request.virtualHost`,
+		},
+	})
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representConnectionTune(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Channel Max",
+			Value:    fmt.Sprintf("%g", event["channelMax"].(float64)),
+			Selector: `request.channelMax`,
+		},
+		{
+			Name:     "Frame Max",
+			Value:    fmt.Sprintf("%g", event["frameMax"].(float64)),
+			Selector: `request.frameMax`,
+		},
+		{
+			Name:     "Heartbeat",
+			Value:    fmt.Sprintf("%g", event["heartbeat"].(float64)),
+			Selector: `request.heartbeat`,
+		},
+	})
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representBasicCancel(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `response.consumerTag`,
+		},
+		{
+			Name:     "NoWait",
+			Value:    strconv.FormatBool(event["noWait"].(bool)),
+			Selector: `request.noWait`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representBasicCancelOk(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "Consumer Tag",
+			Value:    event["consumerTag"].(string),
+			Selector: `response.consumerTag`,
+		},
+	})
+
+	rep = append(rep, api.SectionData{
+		Type:  api.TABLE,
+		Title: "Details",
+		Data:  string(details),
+	})
+
+	return rep
+}
+
+func representEmpty(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	return rep
+}

tap/extensions/amqp/main.go

@@ -46,22 +46,12 @@ func (d dissecting) Ping() {
 	log.Printf("pong %s", protocol.Name)
 }
 
-const amqpRequest string = "amqp_request"
-
 func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.TrafficFilteringOptions) error {
 	r := AmqpReader{b}
 
 	var remaining int
 	var header *HeaderFrame
 
-	connectionInfo := &api.ConnectionInfo{
-		ClientIP:   reader.GetTcpID().SrcIP,
-		ClientPort: reader.GetTcpID().SrcPort,
-		ServerIP:   reader.GetTcpID().DstIP,
-		ServerPort: reader.GetTcpID().DstPort,
-		IsOutgoing: true,
-	}
-
 	eventBasicPublish := &BasicPublish{
 		Exchange:   "",
 		RoutingKey: "",
@@ -83,6 +73,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 
 	var lastMethodFrameMessage Message
 
+	var ident string
+	isClient := reader.GetIsClient()
+	reqResMatcher := reader.GetReqResMatcher().(*requestResponseMatcher)
+
 	for {
 		frameVal, err := r.readFrame()
 		if err == io.EOF {
@@ -121,16 +115,22 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 			switch lastMethodFrameMessage.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Body = f.Body
-				emitAMQP(*eventBasicPublish, amqpRequest, basicMethodMap[40], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[40], *eventBasicPublish, reader)
+				reqResMatcher.emitEvent(!isClient, ident, emptyMethod, &emptyResponse{}, reader)
+
 			case *BasicDeliver:
 				eventBasicDeliver.Body = f.Body
-				emitAMQP(*eventBasicDeliver, amqpRequest, basicMethodMap[60], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(!isClient, ident, basicMethodMap[60], *eventBasicDeliver, reader)
+				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
 			}
 
 		case *MethodFrame:
 			reader.GetParent().SetProtocol(&protocol)
 
 			lastMethodFrameMessage = f.Method
+
+			ident = getIdent(reader, f)
+
 			switch m := f.Method.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Exchange = m.Exchange
@@ -146,7 +146,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				emitAMQP(*eventQueueBind, amqpRequest, queueMethodMap[20], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[20], *eventQueueBind, reader)
+
+			case *QueueBindOk:
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[21], m, reader)
 
 			case *BasicConsume:
 				eventBasicConsume := &BasicConsume{
@@ -158,7 +161,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:      m.NoWait,
 					Arguments:   m.Arguments,
 				}
-				emitAMQP(*eventBasicConsume, amqpRequest, basicMethodMap[20], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[20], *eventBasicConsume, reader)
+
+			case *BasicConsumeOk:
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[21], m, reader)
 
 			case *BasicDeliver:
 				eventBasicDeliver.ConsumerTag = m.ConsumerTag
@@ -177,7 +183,10 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				emitAMQP(*eventQueueDeclare, amqpRequest, queueMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[10], *eventQueueDeclare, reader)
+
+			case *QueueDeclareOk:
+				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[11], m, reader)
 
 			case *ExchangeDeclare:
 				eventExchangeDeclare := &ExchangeDeclare{
@@ -190,17 +199,19 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				emitAMQP(*eventExchangeDeclare, amqpRequest, exchangeMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, exchangeMethodMap[10], *eventExchangeDeclare, reader)
+
+			case *ExchangeDeclareOk:
+				reqResMatcher.emitEvent(isClient, ident, exchangeMethodMap[11], m, reader)
 
 			case *ConnectionStart:
-				eventConnectionStart := &ConnectionStart{
-					VersionMajor:     m.VersionMajor,
-					VersionMinor:     m.VersionMinor,
-					ServerProperties: m.ServerProperties,
-					Mechanisms:       m.Mechanisms,
-					Locales:          m.Locales,
-				}
-				emitAMQP(*eventConnectionStart, amqpRequest, connectionMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				// In our tests, *ConnectionStart does not result in *ConnectionStartOk
+				reqResMatcher.emitEvent(!isClient, ident, connectionMethodMap[10], m, reader)
+				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
+
+			case *ConnectionStartOk:
+				// In our tests, *ConnectionStart does not result in *ConnectionStartOk
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[11], m, reader)
 
 			case *ConnectionClose:
 				eventConnectionClose := &ConnectionClose{
@@ -209,7 +220,40 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					ClassId:   m.ClassId,
 					MethodId:  m.MethodId,
 				}
-				emitAMQP(*eventConnectionClose, amqpRequest, connectionMethodMap[50], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[50], *eventConnectionClose, reader)
+
+			case *ConnectionCloseOk:
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[51], m, reader)
+
+			case *connectionOpen:
+				eventConnectionOpen := &connectionOpen{
+					VirtualHost: m.VirtualHost,
+				}
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[40], *eventConnectionOpen, reader)
+
+			case *connectionOpenOk:
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[41], m, reader)
+
+			case *channelOpen:
+				reqResMatcher.emitEvent(isClient, ident, channelMethodMap[10], m, reader)
+
+			case *channelOpenOk:
+				reqResMatcher.emitEvent(isClient, ident, channelMethodMap[11], m, reader)
+
+			case *connectionTune:
+				// In our tests, *connectionTune does not result in *connectionTuneOk
+				reqResMatcher.emitEvent(!isClient, ident, connectionMethodMap[30], m, reader)
+				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
+
+			case *connectionTuneOk:
+				// In our tests, *connectionTune does not result in *connectionTuneOk
+				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[31], m, reader)
+
+			case *basicCancel:
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[30], m, reader)
+
+			case *basicCancelOk:
+				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[31], m, reader)
 			}
 
 		default:
@@ -220,9 +264,17 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 
 func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
+	response := item.Pair.Response.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
+	resDetails := response["details"].(map[string]interface{})
+
+	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
+	if elapsedTime < 0 {
+		elapsedTime = 0
+	}
 
 	reqDetails["method"] = request["method"]
+	resDetails["method"] = response["method"]
 	return &api.Entry{
 		Protocol: protocol.ProtocolSummary,
 		Capture:  item.Capture,
@@ -236,13 +288,15 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 			IP:   item.ConnectionInfo.ServerIP,
 			Port: item.ConnectionInfo.ServerPort,
 		},
-		Namespace:   namespace,
-		Outgoing:    item.ConnectionInfo.IsOutgoing,
-		Request:     reqDetails,
-		RequestSize: item.Pair.Request.CaptureSize,
-		Timestamp:   item.Timestamp,
-		StartTime:   item.Pair.Request.CaptureTime,
-		ElapsedTime: 0,
+		Namespace:    namespace,
+		Outgoing:     item.ConnectionInfo.IsOutgoing,
+		Request:      reqDetails,
+		Response:     resDetails,
+		RequestSize:  item.Pair.Request.CaptureSize,
+		ResponseSize: item.Pair.Response.CaptureSize,
+		Timestamp:    item.Timestamp,
+		StartTime:    item.Pair.Request.CaptureTime,
+		ElapsedTime:  elapsedTime,
 	}
 
 }
@@ -283,6 +337,21 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	case basicMethodMap[20]:
 		summary = entry.Request["queue"].(string)
 		summaryQuery = fmt.Sprintf(`request.queue == "%s"`, summary)
+	case connectionMethodMap[40]:
+		summary = entry.Request["virtualHost"].(string)
+		summaryQuery = fmt.Sprintf(`request.virtualHost == "%s"`, summary)
+	case connectionMethodMap[30]:
+		summary = fmt.Sprintf("%g", entry.Request["channelMax"].(float64))
+		summaryQuery = fmt.Sprintf(`request.channelMax == "%s"`, summary)
+	case connectionMethodMap[31]:
+		summary = fmt.Sprintf("%g", entry.Request["channelMax"].(float64))
+		summaryQuery = fmt.Sprintf(`request.channelMax == "%s"`, summary)
+	case basicMethodMap[30]:
+		summary = entry.Request["consumerTag"].(string)
+		summaryQuery = fmt.Sprintf(`request.consumerTag == "%s"`, summary)
+	case basicMethodMap[31]:
+		summary = entry.Request["consumerTag"].(string)
+		summaryQuery = fmt.Sprintf(`request.consumerTag == "%s"`, summary)
 	}
 
 	return &api.BaseEntry{
@@ -306,6 +375,8 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, err error) {
 	representation := make(map[string]interface{})
 	var repRequest []interface{}
+	var repResponse []interface{}
+
 	switch request["method"].(string) {
 	case basicMethodMap[40]:
 		repRequest = representBasicPublish(request)
@@ -323,9 +394,45 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 		repRequest = representQueueBind(request)
 	case basicMethodMap[20]:
 		repRequest = representBasicConsume(request)
+	case connectionMethodMap[40]:
+		repRequest = representConnectionOpen(request)
+	case channelMethodMap[10]:
+		repRequest = representEmpty(request)
+	case connectionMethodMap[30]:
+		repRequest = representConnectionTune(request)
+	case basicMethodMap[30]:
+		repRequest = representBasicCancel(request)
 	}
+
+	switch response["method"].(string) {
+	case queueMethodMap[11]:
+		repResponse = representQueueDeclareOk(response)
+	case exchangeMethodMap[11]:
+		repResponse = representEmpty(response)
+	case connectionMethodMap[11]:
+		repResponse = representConnectionStartOk(response)
+	case connectionMethodMap[51]:
+		repResponse = representEmpty(response)
+	case basicMethodMap[21]:
+		repResponse = representBasicConsumeOk(response)
+	case queueMethodMap[21]:
+		repResponse = representEmpty(response)
+	case connectionMethodMap[41]:
+		repResponse = representEmpty(response)
+	case channelMethodMap[11]:
+		repResponse = representEmpty(request)
+	case connectionMethodMap[31]:
+		repResponse = representConnectionTune(request)
+	case basicMethodMap[31]:
+		repResponse = representBasicCancelOk(request)
+	case emptyMethod:
+		repResponse = representEmpty(response)
+	}
+
 	representation["request"] = repRequest
+	representation["response"] = repResponse
 	object, err = json.Marshal(representation)
+
 	return
 }
 
@@ -336,7 +443,7 @@ func (d dissecting) Macros() map[string]string {
 }
 
 func (d dissecting) NewResponseRequestMatcher() api.RequestResponseMatcher {
-	return nil
+	return createResponseRequestMatcher()
 }
 
 var Dissector dissecting

tap/extensions/amqp/matcher.go

@@ -0,0 +1,113 @@
+package amqp
+
+import (
+	"sync"
+	"time"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+// Key is {client_addr}_{client_port}_{dest_addr}_{dest_port}_{channel_id}_{class_id}_{method_id}
+type requestResponseMatcher struct {
+	openMessagesMap *sync.Map
+}
+
+func createResponseRequestMatcher() api.RequestResponseMatcher {
+	return &requestResponseMatcher{openMessagesMap: &sync.Map{}}
+}
+
+func (matcher *requestResponseMatcher) GetMap() *sync.Map {
+	return matcher.openMessagesMap
+}
+
+func (matcher *requestResponseMatcher) SetMaxTry(value int) {
+}
+
+func (matcher *requestResponseMatcher) emitEvent(isRequest bool, ident string, method string, event interface{}, reader api.TcpReader) {
+	reader.GetParent().SetProtocol(&protocol)
+
+	var item *api.OutputChannelItem
+	if isRequest {
+		item = matcher.registerRequest(ident, method, event, reader.GetCaptureTime(), reader.GetReadProgress().Current())
+	} else {
+		item = matcher.registerResponse(ident, method, event, reader.GetCaptureTime(), reader.GetReadProgress().Current())
+	}
+
+	if item != nil {
+		item.ConnectionInfo = &api.ConnectionInfo{
+			ClientIP:   reader.GetTcpID().SrcIP,
+			ClientPort: reader.GetTcpID().SrcPort,
+			ServerIP:   reader.GetTcpID().DstIP,
+			ServerPort: reader.GetTcpID().DstPort,
+			IsOutgoing: true,
+		}
+		item.Capture = reader.GetParent().GetOrigin()
+		reader.GetEmitter().Emit(item)
+	}
+}
+
+func (matcher *requestResponseMatcher) registerRequest(ident string, method string, request interface{}, captureTime time.Time, captureSize int) *api.OutputChannelItem {
+	requestAMQPMessage := api.GenericMessage{
+		IsRequest:   true,
+		CaptureTime: captureTime,
+		CaptureSize: captureSize,
+		Payload: AMQPPayload{
+			Data: &AMQPWrapper{
+				Method:  method,
+				Url:     "",
+				Details: request,
+			},
+		},
+	}
+
+	if response, found := matcher.openMessagesMap.LoadAndDelete(ident); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		responseAMQPMessage := response.(*api.GenericMessage)
+		if responseAMQPMessage.IsRequest {
+			return nil
+		}
+		return matcher.preparePair(&requestAMQPMessage, responseAMQPMessage)
+	}
+
+	matcher.openMessagesMap.Store(ident, &requestAMQPMessage)
+	return nil
+}
+
+func (matcher *requestResponseMatcher) registerResponse(ident string, method string, response interface{}, captureTime time.Time, captureSize int) *api.OutputChannelItem {
+	responseAMQPMessage := api.GenericMessage{
+		IsRequest:   false,
+		CaptureTime: captureTime,
+		CaptureSize: captureSize,
+		Payload: AMQPPayload{
+			Data: &AMQPWrapper{
+				Method:  method,
+				Url:     "",
+				Details: response,
+			},
+		},
+	}
+
+	if request, found := matcher.openMessagesMap.LoadAndDelete(ident); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		requestAMQPMessage := request.(*api.GenericMessage)
+		if !requestAMQPMessage.IsRequest {
+			return nil
+		}
+		return matcher.preparePair(requestAMQPMessage, &responseAMQPMessage)
+	}
+
+	matcher.openMessagesMap.Store(ident, &responseAMQPMessage)
+	return nil
+}
+
+func (matcher *requestResponseMatcher) preparePair(requestAMQPMessage *api.GenericMessage, responseAMQPMessage *api.GenericMessage) *api.OutputChannelItem {
+	return &api.OutputChannelItem{
+		Protocol:       protocol,
+		Timestamp:      requestAMQPMessage.CaptureTime.UnixNano() / int64(time.Millisecond),
+		ConnectionInfo: nil,
+		Pair: &api.RequestResponsePair{
+			Request:  *requestAMQPMessage,
+			Response: *responseAMQPMessage,
+		},
+	}
+}

tap/extensions/amqp/spec091.go

@@ -81,10 +81,10 @@ func (msg *ConnectionStart) read(r io.Reader) (err error) {
 }
 
 type ConnectionStartOk struct {
-	ClientProperties Table
-	Mechanism        string
-	Response         string
-	Locale           string
+	ClientProperties Table  `json:"clientProperties"`
+	Mechanism        string `json:"mechanism"`
+	Response         string `json:"response"`
+	Locale           string `json:"locale"`
 }
 
 func (msg *ConnectionStartOk) read(r io.Reader) (err error) {
@@ -135,9 +135,9 @@ func (msg *connectionSecureOk) read(r io.Reader) (err error) {
 }
 
 type connectionTune struct {
-	ChannelMax uint16
-	FrameMax   uint32
-	Heartbeat  uint16
+	ChannelMax uint16 `json:"channelMax"`
+	FrameMax   uint32 `json:"frameMax"`
+	Heartbeat  uint16 `json:"heartbeat"`
 }
 
 func (msg *connectionTune) read(r io.Reader) (err error) {
@@ -181,7 +181,7 @@ func (msg *connectionTuneOk) read(r io.Reader) (err error) {
 }
 
 type connectionOpen struct {
-	VirtualHost string
+	VirtualHost string `json:"virtualHost"`
 	reserved1   string
 	reserved2   bool
 }
@@ -580,9 +580,9 @@ func (msg *QueueDeclare) read(r io.Reader) (err error) {
 }
 
 type QueueDeclareOk struct {
-	Queue         string
-	MessageCount  uint32
-	ConsumerCount uint32
+	Queue         string `json:"queue"`
+	MessageCount  uint32 `json:"messageCount"`
+	ConsumerCount uint32 `json:"consumerCount"`
 }
 
 func (msg *QueueDeclareOk) read(r io.Reader) (err error) {
@@ -840,7 +840,7 @@ func (msg *BasicConsume) read(r io.Reader) (err error) {
 }
 
 type BasicConsumeOk struct {
-	ConsumerTag string
+	ConsumerTag string `json:"consumerTag"`
 }
 
 func (msg *BasicConsumeOk) read(r io.Reader) (err error) {
@@ -853,8 +853,8 @@ func (msg *BasicConsumeOk) read(r io.Reader) (err error) {
 }
 
 type basicCancel struct {
-	ConsumerTag string
-	NoWait      bool
+	ConsumerTag string `json:"consumerTag"`
+	NoWait      bool   `json:"noWait"`
 }
 
 func (msg *basicCancel) read(r io.Reader) (err error) {
@@ -873,7 +873,7 @@ func (msg *basicCancel) read(r io.Reader) (err error) {
 }
 
 type basicCancelOk struct {
-	ConsumerTag string
+	ConsumerTag string `json:"consumerTag"`
 }
 
 func (msg *basicCancelOk) read(r io.Reader) (err error) {

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect16/http/\* expect

tap/extensions/http/handlers.go

@@ -18,10 +18,6 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
 		return
 	}
 
-	if options.EnableRedaction {
-		FilterSensitiveData(item, options)
-	}
-
 	replaceForwardedFor(item)
 
 	emitter.Emit(item)

tap/extensions/http/helpers.go

@@ -6,13 +6,16 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/up9inc/mizu/tap/api"
 )
 
 func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) {
 	newMap = make(map[string]interface{})
-	for _, item := range mapSlice {
+
+	mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice)
+	for _, item := range mergedMapSlice {
 		h := item.(map[string]interface{})
 		newMap[h["name"].(string)] = h["value"]
 	}
@@ -20,6 +23,28 @@ func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}
 	return
 }
 
+func mapSliceRebuildAsMergedMap(mapSlice []interface{}) (newMap map[string]interface{}) {
+	newMap = make(map[string]interface{})
+
+	mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice)
+	for _, item := range mergedMapSlice {
+		h := item.(map[string]interface{})
+
+		if valuesInterface, ok := h["value"].([]interface{}); ok {
+			var values []string
+			for _, valueInterface := range valuesInterface {
+				values = append(values, valueInterface.(string))
+			}
+
+			newMap[h["name"].(string)] = strings.Join(values, ",")
+		} else {
+			newMap[h["name"].(string)] = h["value"]
+		}
+	}
+
+	return
+}
+
 func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{}) {
 	newMapSlice = make([]interface{}, 0)
 	valuesMap := make(map[string][]interface{})
@@ -47,6 +72,24 @@ func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{
 	return
 }
 
+func representMapAsTable(mapToTable map[string]interface{}, selectorPrefix string) (representation string) {
+	var table []api.TableData
+
+	keys := make([]string, 0, len(mapToTable))
+	for k := range mapToTable {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		table = append(table, createTableForKey(key, mapToTable[key], selectorPrefix)...)
+	}
+
+	obj, _ := json.Marshal(table)
+	representation = string(obj)
+	return
+}
+
 func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
 	for _, item := range mapSlice {
@@ -54,34 +97,7 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re
 		key := h["name"].(string)
 		value := h["value"]
 
-		var reflectKind reflect.Kind
-		reflectType := reflect.TypeOf(value)
-		if reflectType == nil {
-			reflectKind = reflect.Interface
-		} else {
-			reflectKind = reflect.TypeOf(value).Kind()
-		}
-
-		switch reflectKind {
-		case reflect.Slice:
-			fallthrough
-		case reflect.Array:
-			for i, el := range value.([]interface{}) {
-				selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i)
-				table = append(table, api.TableData{
-					Name:     fmt.Sprintf("%s [%d]", key, i),
-					Value:    el,
-					Selector: selector,
-				})
-			}
-		default:
-			selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
-			table = append(table, api.TableData{
-				Name:     key,
-				Value:    value,
-				Selector: selector,
-			})
-		}
+		table = append(table, createTableForKey(key, value, selectorPrefix)...)
 	}
 
 	obj, _ := json.Marshal(table)
@@ -89,6 +105,41 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re
 	return
 }
 
+func createTableForKey(key string, value interface{}, selectorPrefix string) []api.TableData {
+	var table []api.TableData
+
+	var reflectKind reflect.Kind
+	reflectType := reflect.TypeOf(value)
+	if reflectType == nil {
+		reflectKind = reflect.Interface
+	} else {
+		reflectKind = reflect.TypeOf(value).Kind()
+	}
+
+	switch reflectKind {
+	case reflect.Slice:
+		fallthrough
+	case reflect.Array:
+		for i, el := range value.([]interface{}) {
+			selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i)
+			table = append(table, api.TableData{
+				Name:     fmt.Sprintf("%s [%d]", key, i),
+				Value:    el,
+				Selector: selector,
+			})
+		}
+	default:
+		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
+		table = append(table, api.TableData{
+			Name:     key,
+			Value:    value,
+			Selector: selector,
+		})
+	}
+
+	return table
+}
+
 func representSliceAsTable(slice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
 	for i, item := range slice {

tap/extensions/http/main.go

@@ -286,19 +286,13 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	reqDetails["pathSegments"] = strings.Split(path, "/")[1:]
 
 	// Rearrange the maps for the querying
-	reqDetails["_headers"] = reqDetails["headers"]
-	reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{}))
-	resDetails["_headers"] = resDetails["headers"]
-	resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{}))
+	reqDetails["headers"] = mapSliceRebuildAsMergedMap(reqDetails["headers"].([]interface{}))
+	resDetails["headers"] = mapSliceRebuildAsMergedMap(resDetails["headers"].([]interface{}))
 
-	reqDetails["_cookies"] = reqDetails["cookies"]
-	reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{}))
-	resDetails["_cookies"] = resDetails["cookies"]
-	resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{}))
+	reqDetails["cookies"] = mapSliceRebuildAsMergedMap(reqDetails["cookies"].([]interface{}))
+	resDetails["cookies"] = mapSliceRebuildAsMergedMap(resDetails["cookies"].([]interface{}))
 
-	reqDetails["_queryString"] = reqDetails["queryString"]
-	reqDetails["_queryStringMerged"] = mapSliceMergeRepeatedKeys(reqDetails["_queryString"].([]interface{}))
-	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryStringMerged"].([]interface{}))
+	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["queryString"].([]interface{}))
 
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
 	if elapsedTime < 0 {
@@ -397,19 +391,19 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Headers",
-		Data:  representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`),
+		Data:  representMapAsTable(request["headers"].(map[string]interface{}), `request.headers`),
 	})
 
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Cookies",
-		Data:  representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`),
+		Data:  representMapAsTable(request["cookies"].(map[string]interface{}), `request.cookies`),
 	})
 
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Query String",
-		Data:  representMapSliceAsTable(request["_queryStringMerged"].([]interface{}), `request.queryString`),
+		Data:  representMapAsTable(request["queryString"].(map[string]interface{}), `request.queryString`),
 	})
 
 	postData, _ := request["postData"].(map[string]interface{})
@@ -485,13 +479,13 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 	repResponse = append(repResponse, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Headers",
-		Data:  representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`),
+		Data:  representMapAsTable(response["headers"].(map[string]interface{}), `response.headers`),
 	})
 
 	repResponse = append(repResponse, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Cookies",
-		Data:  representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`),
+		Data:  representMapAsTable(response["cookies"].(map[string]interface{}), `response.cookies`),
 	})
 
 	content, _ := response["content"].(map[string]interface{})

tap/extensions/http/sensitive_data_cleaner.go

@@ -1,30 +1,14 @@
 package http
 
 import (
-	"bytes"
-	"encoding/json"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 
-	"github.com/beevik/etree"
 	"github.com/up9inc/mizu/tap/api"
 )
 
-const maskedFieldPlaceholderValue = "[REDACTED]"
 const userAgent = "user-agent"
 
-//these values MUST be all lower case and contain no `-` or `_` characters
-var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
-	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
-	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
-	"zip", "zipcode", "address", "country", "firstname", "lastname",
-	"middlename", "fname", "lname", "birthdate"}
-
 func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool {
 	if item.Protocol.Name != "http" {
 		return false
@@ -48,192 +32,3 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 
 	return false
 }
-
-func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
-	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
-	response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)
-
-	filterHeaders(&request.Header)
-	filterHeaders(&response.Header)
-	filterUrl(request.URL)
-	filterRequestBody(request, options)
-	filterResponseBody(response, options)
-}
-
-func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) {
-	contenType := getContentTypeHeaderValue(request.Header)
-	body, err := ioutil.ReadAll(request.Body)
-	if err != nil {
-		return
-	}
-	filteredBody, err := filterHttpBody(body, contenType, options)
-	if err == nil {
-		request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
-	} else {
-		request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	}
-}
-
-func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) {
-	contentType := getContentTypeHeaderValue(response.Header)
-	body, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		return
-	}
-	filteredBody, err := filterHttpBody(body, contentType, options)
-	if err == nil {
-		response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
-	} else {
-		response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	}
-}
-
-func filterHeaders(headers *http.Header) {
-	for key := range *headers {
-		if strings.ToLower(key) == userAgent {
-			continue
-		}
-
-		if strings.ToLower(key) == "cookie" {
-			headers.Del(key)
-		} else if isFieldNameSensitive(key) {
-			headers.Set(key, maskedFieldPlaceholderValue)
-		}
-	}
-}
-
-func getContentTypeHeaderValue(headers http.Header) string {
-	for key := range headers {
-		if strings.ToLower(key) == "content-type" {
-			return headers.Get(key)
-		}
-	}
-	return ""
-}
-
-func isFieldNameSensitive(fieldName string) bool {
-	if fieldName == ":authority" {
-		return false
-	}
-
-	name := strings.ToLower(fieldName)
-	name = strings.ReplaceAll(name, "_", "")
-	name = strings.ReplaceAll(name, "-", "")
-	name = strings.ReplaceAll(name, " ", "")
-
-	for _, sensitiveField := range personallyIdentifiableDataFields {
-		if strings.Contains(name, sensitiveField) {
-			return true
-		}
-	}
-
-	return false
-}
-
-func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) {
-	mimeType := strings.Split(contentType, ";")[0]
-	switch strings.ToLower(mimeType) {
-	case "application/json":
-		return filterJsonBody(bytes)
-	case "text/html":
-		fallthrough
-	case "application/xhtml+xml":
-		fallthrough
-	case "text/xml":
-		fallthrough
-	case "application/xml":
-		return filterXmlEtree(bytes)
-	case "text/plain":
-		if options != nil && options.PlainTextMaskingRegexes != nil {
-			return filterPlainText(bytes, options), nil
-		}
-	}
-	return bytes, nil
-}
-
-func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte {
-	for _, regex := range options.PlainTextMaskingRegexes {
-		bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
-	}
-	return bytes
-}
-
-func filterXmlEtree(bytes []byte) ([]byte, error) {
-	if !IsValidXML(bytes) {
-		return nil, errors.New("Invalid XML")
-	}
-	xmlDoc := etree.NewDocument()
-	err := xmlDoc.ReadFromBytes(bytes)
-	if err != nil {
-		return nil, err
-	} else {
-		filterXmlElement(xmlDoc.Root())
-	}
-	return xmlDoc.WriteToBytes()
-}
-
-func IsValidXML(data []byte) bool {
-	return xml.Unmarshal(data, new(interface{})) == nil
-}
-
-func filterXmlElement(element *etree.Element) {
-	for i, attribute := range element.Attr {
-		if isFieldNameSensitive(attribute.Key) {
-			element.Attr[i].Value = maskedFieldPlaceholderValue
-		}
-	}
-	if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
-		if isFieldNameSensitive(element.Tag) {
-			element.SetText(maskedFieldPlaceholderValue)
-		}
-	} else {
-		for _, element := range element.ChildElements() {
-			filterXmlElement(element)
-		}
-	}
-}
-
-func filterJsonBody(bytes []byte) ([]byte, error) {
-	var bodyJsonMap map[string]interface{}
-	err := json.Unmarshal(bytes, &bodyJsonMap)
-	if err != nil {
-		return nil, err
-	}
-	filterJsonMap(bodyJsonMap)
-	return json.Marshal(bodyJsonMap)
-}
-
-func filterJsonMap(jsonMap map[string]interface{}) {
-	for key, value := range jsonMap {
-		// Do not replace nil values with maskedFieldPlaceholderValue
-		if value == nil {
-			continue
-		}
-
-		nestedMap, isNested := value.(map[string]interface{})
-		if isNested {
-			filterJsonMap(nestedMap)
-		} else {
-			if isFieldNameSensitive(key) {
-				jsonMap[key] = maskedFieldPlaceholderValue
-			}
-		}
-	}
-}
-
-func filterUrl(url *url.URL) {
-	if len(url.RawQuery) > 0 {
-		newQueryArgs := make([]string, 0)
-		for urlQueryParamName, urlQueryParamValues := range url.Query() {
-			newValues := urlQueryParamValues
-			if isFieldNameSensitive(urlQueryParamName) {
-				newValues = []string{maskedFieldPlaceholderValue}
-			}
-			for _, paramValue := range newValues {
-				newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
-			}
-		}
-
-		url.RawQuery = strings.Join(newQueryArgs, "&")
-	}
-}

tap/tlstapper/bpf-builder/build.sh

@@ -9,7 +9,7 @@ docker build -t mizu-ebpf-builder . || exit 1
 BPF_TARGET=amd64
 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
 ARCH=$(uname -m)
-if [[ $ARCH == "aarch64" ]]; then
+if [[ $ARCH == "aarch64" || $ARCH == "arm64" ]]; then
     BPF_TARGET=arm64
     BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
 fi
@@ -18,7 +18,7 @@ docker run --rm \
 	--name mizu-ebpf-builder \
 	-v $MIZU_HOME:/mizu \
 	-v $(go env GOPATH):/root/go \
-	-it mizu-ebpf-builder \
+	mizu-ebpf-builder \
 	sh -c "
 		BPF_TARGET=\"$BPF_TARGET\" BPF_CFLAGS=\"$BPF_CFLAGS\" go generate tap/tlstapper/tls_tapper.go
         chown $(id -u):$(id -g) tap/tlstapper/tlstapper*_bpf*

tap/tlstapper/bpf/common.c

@@ -16,34 +16,16 @@ static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_
     __u32 pid = id >> 32;
     __u64 key = (__u64) pid << 32 | fd;
 
-    struct fd_info *fdinfo = bpf_map_lookup_elem(&file_descriptor_to_ipv4, &key);
+    conn_flags *flags = bpf_map_lookup_elem(&connection_context, &key);
 
-    if (fdinfo == NULL) {
+    // Happens when we don't catch the connect / accept (if the connection is created before tapping is started)
+    if (flags == NULL) {
         return 0;
     }
 
-    int err;
+    chunk->flags |= (*flags & FLAGS_IS_CLIENT_BIT);
 
-    switch (info->address_info.mode) {
-        case ADDRESS_INFO_MODE_UNDEFINED:
-            chunk->address_info.mode = ADDRESS_INFO_MODE_SINGLE;
-            err = bpf_probe_read(&chunk->address_info.sport, sizeof(chunk->address_info.sport), &fdinfo->ipv4_addr[2]);
-            if (err != 0) {
-                log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
-                return 0;
-            }
-
-            err = bpf_probe_read(&chunk->address_info.saddr, sizeof(chunk->address_info.saddr), &fdinfo->ipv4_addr[4]);
-            if (err != 0) {
-                log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
-                return 0;
-            }
-            break;
-        default:
-            bpf_probe_read(&chunk->address_info, sizeof(chunk->address_info), &info->address_info);
-    }
-
-    chunk->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT);
+    bpf_probe_read(&chunk->address_info, sizeof(chunk->address_info), &info->address_info);
 
     return 1;
 }

tap/tlstapper/bpf/fd_to_address_tracepoints.c

@@ -14,7 +14,6 @@ Copyright (C) UP9 Inc.
 #define IPV4_ADDR_LEN (16)
 
 struct accept_info {
-	__u64* sockaddr;
 	__u32* addrlen;
 };
 
@@ -39,7 +38,6 @@ void sys_enter_accept4(struct sys_enter_accept4_ctx *ctx) {
 	
 	struct accept_info info = {};
 	
-	info.sockaddr = ctx->sockaddr;
 	info.addrlen = ctx->addrlen;
 	
 	long err = bpf_map_update_elem(&accept_syscall_context, &id, &info, BPF_ANY);
@@ -94,26 +92,21 @@ void sys_exit_accept4(struct sys_exit_accept4_ctx *ctx) {
 		return;
 	}
 	
-	struct fd_info fdinfo = {
-		.flags = 0
-	};
-	
-	bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr);
+	conn_flags flags = 0;
 	
 	__u32 pid = id >> 32;
 	__u32 fd = (__u32) ctx->ret;
 	
 	__u64 key = (__u64) pid << 32 | fd;
-	err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY);
+	err = bpf_map_update_elem(&connection_context, &key, &flags, BPF_ANY);
 	
 	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FD_MAPPING, id, err, ORIGIN_SYS_EXIT_ACCEPT4_CODE);
+		log_error(ctx, LOG_ERROR_PUTTING_CONNECTION_CONTEXT, id, err, ORIGIN_SYS_EXIT_ACCEPT4_CODE);
 	}
 }
 
 struct connect_info {
 	__u64 fd;
-	__u64* sockaddr;
 	__u32 addrlen;
 };
 
@@ -138,7 +131,6 @@ void sys_enter_connect(struct sys_enter_connect_ctx *ctx) {
 	
 	struct connect_info info = {};
 	
-	info.sockaddr = ctx->sockaddr;
 	info.addrlen = ctx->addrlen;
 	info.fd = ctx->fd;
 	
@@ -193,19 +185,15 @@ void sys_exit_connect(struct sys_exit_connect_ctx *ctx) {
 		return;
 	}
 	
-	struct fd_info fdinfo = {
-		.flags = FLAGS_IS_CLIENT_BIT
-	};
-	
-	bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr);
+	conn_flags flags = FLAGS_IS_CLIENT_BIT;
 	
 	__u32 pid = id >> 32;
 	__u32 fd = (__u32) info.fd;
 	
 	__u64 key = (__u64) pid << 32 | fd;
-	err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY);
+	err = bpf_map_update_elem(&connection_context, &key, &flags, BPF_ANY);
 	
 	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FD_MAPPING, id, err, ORIGIN_SYS_EXIT_CONNECT_CODE);
+		log_error(ctx, LOG_ERROR_PUTTING_CONNECTION_CONTEXT, id, err, ORIGIN_SYS_EXIT_CONNECT_CODE);
 	}
 }

tap/tlstapper/bpf/fd_tracepoints.c

@@ -10,8 +10,9 @@ Copyright (C) UP9 Inc.
 #include "include/log.h"
 #include "include/logger_messages.h"
 #include "include/pids.h"
+#include "include/common.h"
 
-struct sys_enter_read_ctx {
+struct sys_enter_read_write_ctx {
 	__u64 __unused_syscall_header;
 	__u32 __unused_syscall_nr;
 	
@@ -20,8 +21,46 @@ struct sys_enter_read_ctx {
 	__u64 count;
 };
 
+struct sys_exit_read_write_ctx {
+	__u64 __unused_syscall_header;
+	__u32 __unused_syscall_nr;
+
+	__u64 ret;
+};
+
+
+static __always_inline void fd_tracepoints_handle_openssl(struct sys_enter_read_write_ctx *ctx, __u64 id, struct ssl_info *infoPtr, struct bpf_map_def *map_fd, __u64 origin_code) {
+	struct ssl_info info;
+	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, origin_code);
+		return;
+	}
+	
+	info.fd = ctx->fd;
+	
+	err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, origin_code);
+		return;
+	}
+}
+
+static __always_inline void fd_tracepoints_handle_go(struct sys_enter_read_write_ctx *ctx, __u64 id, struct bpf_map_def *map_fd, __u64 origin_code) {
+	__u32 fd = ctx->fd;
+
+	long err = bpf_map_update_elem(map_fd, &id, &fd, BPF_ANY);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, origin_code);
+		return;
+	}
+}
+
 SEC("tracepoint/syscalls/sys_enter_read")
-void sys_enter_read(struct sys_enter_read_ctx *ctx) {
+void sys_enter_read(struct sys_enter_read_write_ctx *ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -30,38 +69,15 @@ void sys_enter_read(struct sys_enter_read_ctx *ctx) {
 	
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_read_context, &id);
 	
-	if (infoPtr == NULL) {
-		return;
-	}
-	
-	struct ssl_info info;
-	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, ORIGIN_SYS_ENTER_READ_CODE);
-		return;
-	}
-	
-	info.fd = ctx->fd;
-	
-	err = bpf_map_update_elem(&openssl_read_context, &id, &info, BPF_ANY);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_READ_CODE);
+	if (infoPtr != NULL) {
+		fd_tracepoints_handle_openssl(ctx, id, infoPtr, &openssl_read_context, ORIGIN_SYS_ENTER_READ_CODE);
 	}
+
+	fd_tracepoints_handle_go(ctx, id, &go_kernel_read_context, ORIGIN_SYS_ENTER_READ_CODE);
 }
-
-struct sys_enter_write_ctx {
-	__u64 __unused_syscall_header;
-	__u32 __unused_syscall_nr;
 	
-	__u64 fd;
-	__u64* buf;
-	__u64 count;
-};
-
 SEC("tracepoint/syscalls/sys_enter_write")
-void sys_enter_write(struct sys_enter_write_ctx *ctx) {
+void sys_enter_write(struct sys_enter_read_write_ctx *ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -70,23 +86,25 @@ void sys_enter_write(struct sys_enter_write_ctx *ctx) {
 	
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_write_context, &id);
 	
-	if (infoPtr == NULL) {
-		return;
-	}
-	
-	struct ssl_info info;
-	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, ORIGIN_SYS_ENTER_WRITE_CODE);
-		return;
-	}
-	
-	info.fd = ctx->fd;
-	
-	err = bpf_map_update_elem(&openssl_write_context, &id, &info, BPF_ANY);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_WRITE_CODE);
+	if (infoPtr != NULL) {
+		fd_tracepoints_handle_openssl(ctx, id, infoPtr, &openssl_write_context, ORIGIN_SYS_ENTER_WRITE_CODE);
 	}
+
+	fd_tracepoints_handle_go(ctx, id, &go_kernel_write_context, ORIGIN_SYS_ENTER_WRITE_CODE);
+}
+
+SEC("tracepoint/syscalls/sys_exit_read")
+void sys_exit_read(struct sys_exit_read_write_ctx *ctx) {
+	__u64 id = bpf_get_current_pid_tgid();
+	// Delete from go map. The value is not used after exiting this syscall.
+	// Keep value in openssl map.
+	bpf_map_delete_elem(&go_kernel_read_context, &id);
+}
+
+SEC("tracepoint/syscalls/sys_exit_write")
+void sys_exit_write(struct sys_exit_read_write_ctx *ctx) {
+	__u64 id = bpf_get_current_pid_tgid();
+	// Delete from go map. The value is not used after exiting this syscall.
+	// Keep value in openssl map.
+	bpf_map_delete_elem(&go_kernel_write_context, &id);
 }

tap/tlstapper/bpf/go_uprobes.c

@@ -220,7 +220,7 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf
     return;
 }
 
-static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, __u32 flags, enum ABI abi) {
+static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, struct bpf_map_def* go_user_kernel_context, __u32 flags, enum ABI abi) {
     __u64 pid_tgid = bpf_get_current_pid_tgid();
     __u64 pid = pid_tgid >> 32;
     if (!should_tap(pid)) {
@@ -285,6 +285,21 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct
         }
     }
 
+    __u64 key = (__u64) pid << 32 | info_ptr->fd;
+    struct address_info *address_info = bpf_map_lookup_elem(go_user_kernel_context, &key);
+    // Ideally we would delete the entry from the map after reading it,
+    // but sometimes the uprobe is called twice in a row without the tcp kprobes in between to fill in
+    // the entry again. Keeping it in the map and rely on LRU logic.
+    if (address_info == NULL) {
+        log_error(ctx, LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT, pid_tgid, info_ptr->fd, err);
+        return;
+    }
+
+    info.address_info.daddr = address_info->daddr;
+    info.address_info.dport = address_info->dport;
+    info.address_info.saddr = address_info->saddr;
+    info.address_info.sport = address_info->sport;
+
     output_ssl_chunk(ctx, &info, info.buffer_len, pid_tgid, flags);
 
     return;
@@ -298,7 +313,7 @@ int BPF_KPROBE(go_crypto_tls_abi0_write) {
 
 SEC("uprobe/go_crypto_tls_abi0_write_ex")
 int BPF_KPROBE(go_crypto_tls_abi0_write_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0, ABI0);
+    go_crypto_tls_ex_uprobe(ctx, &go_write_context, &go_user_kernel_write_context, 0, ABI0);
     return 1;
 }
 
@@ -310,7 +325,7 @@ int BPF_KPROBE(go_crypto_tls_abi0_read) {
 
 SEC("uprobe/go_crypto_tls_abi0_read_ex")
 int BPF_KPROBE(go_crypto_tls_abi0_read_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT, ABI0);
+    go_crypto_tls_ex_uprobe(ctx, &go_read_context, &go_user_kernel_read_context, FLAGS_IS_READ_BIT, ABI0);
     return 1;
 }
 
@@ -322,7 +337,7 @@ int BPF_KPROBE(go_crypto_tls_abi_internal_write) {
 
 SEC("uprobe/go_crypto_tls_abi_internal_write_ex")
 int BPF_KPROBE(go_crypto_tls_abi_internal_write_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0, ABIInternal);
+    go_crypto_tls_ex_uprobe(ctx, &go_write_context, &go_user_kernel_write_context, 0, ABIInternal);
     return 1;
 }
 
@@ -334,6 +349,6 @@ int BPF_KPROBE(go_crypto_tls_abi_internal_read) {
 
 SEC("uprobe/go_crypto_tls_abi_internal_read_ex")
 int BPF_KPROBE(go_crypto_tls_abi_internal_read_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT, ABIInternal);
+    go_crypto_tls_ex_uprobe(ctx, &go_read_context, &go_user_kernel_read_context, FLAGS_IS_READ_BIT, ABIInternal);
     return 1;
 }

tap/tlstapper/bpf/include/logger_messages.h

@@ -10,27 +10,28 @@ Copyright (C) UP9 Inc.
 // Must be synced with bpf_logger_messages.go
 //
 #define LOG_ERROR_READING_BYTES_COUNT (0)
-#define LOG_ERROR_READING_FD_ADDRESS (1)
-#define LOG_ERROR_READING_FROM_SSL_BUFFER (2)
-#define LOG_ERROR_BUFFER_TOO_BIG (3)
-#define LOG_ERROR_ALLOCATING_CHUNK (4)
-#define LOG_ERROR_READING_SSL_CONTEXT (5)
-#define LOG_ERROR_PUTTING_SSL_CONTEXT (6)
-#define LOG_ERROR_GETTING_SSL_CONTEXT (7)
-#define LOG_ERROR_MISSING_FILE_DESCRIPTOR (8)
-#define LOG_ERROR_PUTTING_FILE_DESCRIPTOR (9)
-#define LOG_ERROR_PUTTING_ACCEPT_INFO (10)
-#define LOG_ERROR_GETTING_ACCEPT_INFO (11)
-#define LOG_ERROR_READING_ACCEPT_INFO (12)
-#define LOG_ERROR_PUTTING_FD_MAPPING (13)
-#define LOG_ERROR_PUTTING_CONNECT_INFO (14)
-#define LOG_ERROR_GETTING_CONNECT_INFO (15)
-#define LOG_ERROR_READING_CONNECT_INFO (16)
-#define LOG_ERROR_READING_SOCKET_FAMILY (17)
-#define LOG_ERROR_READING_SOCKET_DADDR (18)
-#define LOG_ERROR_READING_SOCKET_SADDR (19)
-#define LOG_ERROR_READING_SOCKET_DPORT (20)
-#define LOG_ERROR_READING_SOCKET_SPORT (21)
+#define LOG_ERROR_READING_FROM_SSL_BUFFER (1)
+#define LOG_ERROR_BUFFER_TOO_BIG (2)
+#define LOG_ERROR_ALLOCATING_CHUNK (3)
+#define LOG_ERROR_READING_SSL_CONTEXT (4)
+#define LOG_ERROR_PUTTING_SSL_CONTEXT (5)
+#define LOG_ERROR_GETTING_SSL_CONTEXT (6)
+#define LOG_ERROR_MISSING_FILE_DESCRIPTOR (7)
+#define LOG_ERROR_PUTTING_FILE_DESCRIPTOR (8)
+#define LOG_ERROR_PUTTING_ACCEPT_INFO (9)
+#define LOG_ERROR_GETTING_ACCEPT_INFO (10)
+#define LOG_ERROR_READING_ACCEPT_INFO (11)
+#define LOG_ERROR_PUTTING_CONNECTION_CONTEXT (12)
+#define LOG_ERROR_PUTTING_CONNECT_INFO (13)
+#define LOG_ERROR_GETTING_CONNECT_INFO (14)
+#define LOG_ERROR_READING_CONNECT_INFO (15)
+#define LOG_ERROR_READING_SOCKET_FAMILY (16)
+#define LOG_ERROR_READING_SOCKET_DADDR (17)
+#define LOG_ERROR_READING_SOCKET_SADDR (18)
+#define LOG_ERROR_READING_SOCKET_DPORT (19)
+#define LOG_ERROR_READING_SOCKET_SPORT (20)
+#define LOG_ERROR_PUTTING_GO_USER_KERNEL_CONTEXT (21)
+#define LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT (22)
 
 // Sometimes we have the same error, happening from different locations.
 // 	in order to be able to distinct between them in the log, we add an 

tap/tlstapper/bpf/include/maps.h

@@ -25,14 +25,7 @@ Copyright (C) UP9 Inc.
 //  Be careful when editing, alignment and padding should be exactly the same in go/c.
 //
 
-typedef enum {
-    ADDRESS_INFO_MODE_UNDEFINED,
-    ADDRESS_INFO_MODE_SINGLE,
-    ADDRESS_INFO_MODE_PAIR,
-} address_info_mode;
-
 struct address_info {
-    address_info_mode mode;
     __be32 saddr;
     __be32 daddr;
     __be16 sport;
@@ -64,10 +57,7 @@ struct ssl_info {
     size_t *count_ptr;
 };
 
-struct fd_info {
-    __u8 ipv4_addr[16]; // struct sockaddr (linux-src/include/linux/socket.h)
-    __u8 flags;
-};
+typedef __u8 conn_flags;
 
 struct goid_offsets {
     __u64 g_addr_offset;
@@ -105,7 +95,7 @@ struct {
 
 // Generic
 BPF_HASH(pids_map, __u32, __u32);
-BPF_LRU_HASH(file_descriptor_to_ipv4, __u64, struct fd_info);
+BPF_LRU_HASH(connection_context, __u64, conn_flags);
 BPF_PERF_OUTPUT(chunks_buffer);
 BPF_PERF_OUTPUT(log_buffer);
 
@@ -117,5 +107,9 @@ BPF_LRU_HASH(openssl_read_context, __u64, struct ssl_info);
 BPF_HASH(goid_offsets_map, __u32, struct goid_offsets);
 BPF_LRU_HASH(go_write_context, __u64, struct ssl_info);
 BPF_LRU_HASH(go_read_context, __u64, struct ssl_info);
+BPF_LRU_HASH(go_kernel_write_context, __u64, __u32);
+BPF_LRU_HASH(go_kernel_read_context, __u64, __u32);
+BPF_LRU_HASH(go_user_kernel_write_context, __u64, struct address_info);
+BPF_LRU_HASH(go_user_kernel_read_context, __u64, struct address_info);
 
 #endif /* __MAPS__ */

tap/tlstapper/bpf/tcp_kprobes.c

@@ -5,32 +5,19 @@
 #include "include/pids.h"
 #include "include/common.h"
 
-static __always_inline void tcp_kprobe(struct pt_regs *ctx, struct bpf_map_def *map_fd, _Bool is_send) {
+
+static __always_inline int tcp_kprobes_get_address_pair_from_ctx(struct pt_regs *ctx, __u64 id, struct address_info *address_info_ptr) {
 	long err;
-
-	__u64 id = bpf_get_current_pid_tgid();
-	__u32 pid = id >> 32;
-
-	if (!should_tap(id >> 32)) {
-		return;
-	}
-
-	struct ssl_info *info_ptr = bpf_map_lookup_elem(map_fd, &id);
-	// Happens when the connection is not tls
-	if (info_ptr == NULL) {
-		return;
-	}
-
 	struct sock *sk = (struct sock *) PT_REGS_PARM1(ctx);
 
 	short unsigned int family;
 	err = bpf_probe_read(&family, sizeof(family), (void *)&sk->__sk_common.skc_family);
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_READING_SOCKET_FAMILY, id, err, 0l);
-		return;
+		return -1;
 	}
 	if (family != AF_INET) {
-		return;
+		return -1;
 	}
 
 	// daddr, saddr and dport are in network byte order (big endian)
@@ -43,37 +30,89 @@ static __always_inline void tcp_kprobe(struct pt_regs *ctx, struct bpf_map_def *
 	err = bpf_probe_read(&saddr, sizeof(saddr), (void *)&sk->__sk_common.skc_rcv_saddr);
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_READING_SOCKET_SADDR, id, err, 0l);
-		return;
+		return -1;
 	}
 	err = bpf_probe_read(&daddr, sizeof(daddr), (void *)&sk->__sk_common.skc_daddr);
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_READING_SOCKET_DADDR, id, err, 0l);
-		return;
+		return -1;
 	}
 	err = bpf_probe_read(&dport, sizeof(dport), (void *)&sk->__sk_common.skc_dport);
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_READING_SOCKET_DPORT, id, err, 0l);
-		return;
+		return -1;
 	}
 	err = bpf_probe_read(&sport, sizeof(sport), (void *)&sk->__sk_common.skc_num);
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_READING_SOCKET_SPORT, id, err, 0l);
+		return -1;
+	}
+
+	address_info_ptr->daddr = daddr;
+	address_info_ptr->saddr = saddr;
+	address_info_ptr->dport = dport;
+	address_info_ptr->sport = bpf_htons(sport);
+
+	return 0;
+}
+
+static __always_inline void tcp_kprobes_forward_go(struct pt_regs *ctx, __u64 id, __u32 fd, struct address_info address_info, struct bpf_map_def *map_fd_go_user_kernel) {
+		__u32 pid = id >> 32;
+		__u64 key = (__u64) pid << 32 | fd;
+
+		long err = bpf_map_update_elem(map_fd_go_user_kernel, &key, &address_info, BPF_ANY);
+    if (err != 0) {
+        log_error(ctx, LOG_ERROR_PUTTING_GO_USER_KERNEL_CONTEXT, id, fd, err);
+				return;
+    }
+}
+
+static void __always_inline tcp_kprobes_forward_openssl(struct ssl_info *info_ptr, struct address_info address_info) {
+		info_ptr->address_info.daddr = address_info.daddr;
+		info_ptr->address_info.saddr = address_info.saddr;
+		info_ptr->address_info.dport = address_info.dport;
+		info_ptr->address_info.sport = address_info.sport;
+}
+
+static __always_inline void tcp_kprobe(struct pt_regs *ctx, struct bpf_map_def *map_fd_openssl, struct bpf_map_def *map_fd_go_kernel, struct bpf_map_def *map_fd_go_user_kernel) {
+	long err;
+
+	__u64 id = bpf_get_current_pid_tgid();
+
+	if (!should_tap(id >> 32)) {
 		return;
 	}
 
-	info_ptr->address_info.mode = ADDRESS_INFO_MODE_PAIR;
-	info_ptr->address_info.daddr = daddr;
-	info_ptr->address_info.saddr = saddr;
-	info_ptr->address_info.dport = dport;
-	info_ptr->address_info.sport = bpf_htons(sport);
+	struct address_info address_info;
+	if (0 != tcp_kprobes_get_address_pair_from_ctx(ctx, id, &address_info)) {
+		return;
+	}
+
+	struct ssl_info *info_ptr = bpf_map_lookup_elem(map_fd_openssl, &id);
+	__u32 *fd_ptr;
+	if (info_ptr == NULL) {
+		fd_ptr = bpf_map_lookup_elem(map_fd_go_kernel, &id);
+		// Connection is used by a Go program
+		if (fd_ptr == NULL) {
+			// Connection was not created by a Go program or by openssl lib
+			return;
+		}
+		tcp_kprobes_forward_go(ctx, id, *fd_ptr, address_info, map_fd_go_user_kernel);
+	} else {
+		// Connection is used by openssl lib
+		tcp_kprobes_forward_openssl(info_ptr, address_info);
+	}
+
 }
 
 SEC("kprobe/tcp_sendmsg")
 void BPF_KPROBE(tcp_sendmsg) {
-	tcp_kprobe(ctx, &openssl_write_context, true);
+	__u64 id = bpf_get_current_pid_tgid();
+	tcp_kprobe(ctx, &openssl_write_context, &go_kernel_write_context, &go_user_kernel_write_context);
 }
 
 SEC("kprobe/tcp_recvmsg")
 void BPF_KPROBE(tcp_recvmsg) {
-	tcp_kprobe(ctx, &openssl_read_context, false);
+	__u64 id = bpf_get_current_pid_tgid();
+	tcp_kprobe(ctx, &openssl_read_context, &go_kernel_read_context, &go_user_kernel_read_context);
 }

tap/tlstapper/bpf_logger_messages.go

@@ -4,25 +4,26 @@ package tlstapper
 //
 var bpfLogMessages = []string{
 	/*0000*/ "[%d] Unable to read bytes count from _ex methods [err: %d]",
-	/*0001*/ "[%d] Unable to read ipv4 address [err: %d]",
-	/*0002*/ "[%d] Unable to read ssl buffer [err: %d]",
-	/*0003*/ "[%d] Buffer is too big [size: %d]",
-	/*0004*/ "[%d] Unable to allocate chunk in bpf heap",
-	/*0005*/ "[%d] Unable to read ssl context [err: %d] [origin: %d]",
-	/*0006*/ "[%d] Unable to put ssl context [err: %d]",
-	/*0007*/ "[%d] Unable to get ssl context",
-	/*0008*/ "[%d] File descriptor is missing for tls chunk",
-	/*0009*/ "[%d] Unable to put file descriptor [err: %d] [origin: %d]",
-	/*0010*/ "[%d] Unable to put accept info [err: %d]",
-	/*0011*/ "[%d] Unable to get accept info",
-	/*0012*/ "[%d] Unable to read accept info [err: %d]",
-	/*0013*/ "[%d] Unable to put file descriptor to address mapping [err: %d] [origin: %d]",
-	/*0014*/ "[%d] Unable to put connect info [err: %d]",
-	/*0015*/ "[%d] Unable to get connect info",
-	/*0016*/ "[%d] Unable to read connect info [err: %d]",
-	/*0017*/ "[%d] Unable to read socket family [err: %d]",
-	/*0018*/ "[%d] Unable to read socket daddr [err: %d]",
-	/*0019*/ "[%d] Unable to read socket saddr [err: %d]",
+	/*0001*/ "[%d] Unable to read ssl buffer [err: %d] [origin: %d]",
+	/*0002*/ "[%d] Buffer is too big [size: %d]",
+	/*0003*/ "[%d] Unable to allocate chunk in bpf heap",
+	/*0004*/ "[%d] Unable to read ssl context [err: %d] [origin: %d]",
+	/*0005*/ "[%d] Unable to put ssl context [err: %d]",
+	/*0006*/ "[%d] Unable to get ssl context",
+	/*0007*/ "[%d] File descriptor is missing for tls chunk",
+	/*0008*/ "[%d] Unable to put file descriptor [err: %d] [origin: %d]",
+	/*0009*/ "[%d] Unable to put accept info [err: %d]",
+	/*0010*/ "[%d] Unable to get accept info",
+	/*0011*/ "[%d] Unable to read accept info [err: %d]",
+	/*0012*/ "[%d] Unable to put connection info to connection context [err: %d] [origin: %d]",
+	/*0013*/ "[%d] Unable to put connect info [err: %d]",
+	/*0014*/ "[%d] Unable to get connect info",
+	/*0015*/ "[%d] Unable to read connect info [err: %d]",
+	/*0016*/ "[%d] Unable to read socket family [err: %d]",
+	/*0017*/ "[%d] Unable to read socket daddr [err: %d]",
+	/*0018*/ "[%d] Unable to read socket saddr [err: %d]",
 	/*0019*/ "[%d] Unable to read socket dport [err: %d]",
-	/*0021*/ "[%d] Unable to read socket sport [err: %d]",
+	/*0020*/ "[%d] Unable to read socket sport [err: %d]",
+	/*0021*/ "[%d] Unable to put go user-kernel context [fd: %d] [err: %d]",
+	/*0022*/ "[%d] Unable to get go user-kernel context [fd: %d]]",
 }

tap/tlstapper/chunk.go

@@ -4,17 +4,17 @@ import (
 	"encoding/binary"
 	"net"
 	"unsafe"
-
-	"github.com/up9inc/mizu/tap/api"
 )
 
 const FlagsIsClientBit uint32 = 1 << 0
 const FlagsIsReadBit uint32 = 1 << 1
-const (
-	addressInfoModeUndefined = iota
-	addressInfoModeSingle
-	addressInfoModePair
-)
+
+type addressPair struct {
+	srcIp   net.IP
+	srcPort uint16
+	dstIp   net.IP
+	dstPort uint16
+}
 
 func (c *tlsTapperTlsChunk) getSrcAddress() (net.IP, uint16) {
 	ip := intToIP(c.AddressInfo.Saddr)
@@ -54,36 +54,18 @@ func (c *tlsTapperTlsChunk) isRequest() bool {
 	return (c.isClient() && c.isWrite()) || (c.isServer() && c.isRead())
 }
 
-func (c *tlsTapperTlsChunk) getAddressPair() (addressPair, bool) {
+func (c *tlsTapperTlsChunk) getAddressPair() addressPair {
 	var (
 		srcIp, dstIp     net.IP
 		srcPort, dstPort uint16
-		full             bool
 	)
 
-	switch c.AddressInfo.Mode {
-	case addressInfoModeSingle:
-		if c.isRequest() {
-			srcIp, srcPort = api.UnknownIp, api.UnknownPort
-			dstIp, dstPort = c.getSrcAddress()
-		} else {
-			srcIp, srcPort = c.getSrcAddress()
-			dstIp, dstPort = api.UnknownIp, api.UnknownPort
-		}
-		full = false
-	case addressInfoModePair:
-		if c.isRequest() {
-			srcIp, srcPort = c.getSrcAddress()
-			dstIp, dstPort = c.getDstAddress()
-		} else {
-			srcIp, srcPort = c.getDstAddress()
-			dstIp, dstPort = c.getSrcAddress()
-		}
-		full = true
-	case addressInfoModeUndefined:
-		srcIp, srcPort = api.UnknownIp, api.UnknownPort
-		dstIp, dstPort = api.UnknownIp, api.UnknownPort
-		full = false
+	if c.isRequest() {
+		srcIp, srcPort = c.getSrcAddress()
+		dstIp, dstPort = c.getDstAddress()
+	} else {
+		srcIp, srcPort = c.getDstAddress()
+		dstIp, dstPort = c.getSrcAddress()
 	}
 
 	return addressPair{
@@ -91,7 +73,7 @@ func (c *tlsTapperTlsChunk) getAddressPair() (addressPair, bool) {
 		srcPort: srcPort,
 		dstIp:   dstIp,
 		dstPort: dstPort,
-	}, full
+	}
 }
 
 // intToIP converts IPv4 number to net.IP

tap/tlstapper/sockfd_info.go

@@ -1,122 +0,0 @@
-package tlstapper
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"github.com/go-errors/errors"
-)
-
-var socketInodeRegex = regexp.MustCompile(`socket:\[(\d+)\]`)
-
-const (
-	SRC_ADDRESS_FILED_INDEX = 1
-	DST_ADDRESS_FILED_INDEX = 2
-	INODE_FILED_INDEX       = 9
-)
-
-type addressPair struct {
-	srcIp   net.IP
-	srcPort uint16
-	dstIp   net.IP
-	dstPort uint16
-}
-
-// This file helps to extract Ip and Port out of a Socket file descriptor.
-//
-// The equivalent bash commands are:
-//
-//  > ls -l /proc/<pid>/fd/<fd>
-// 	    Output something like "socket:[1234]" for sockets - 1234 is the inode of the socket
-//  > cat /proc/<pid>/net/tcp | grep <inode>
-//      Output a line per ipv4 socket, the 9th field is the inode of the socket
-//      The 1st and 2nd fields are the source and dest ip and ports in a Hex format
-//      0100007F:50 is 127.0.0.1:80
-
-func getAddressBySockfd(procfs string, pid uint32, fd uint32) (addressPair, error) {
-	inode, err := getSocketInode(procfs, pid, fd)
-
-	if err != nil {
-		return addressPair{}, err
-	}
-
-	tcppath := fmt.Sprintf("%s/%d/net/tcp", procfs, pid)
-	tcp, err := ioutil.ReadFile(tcppath)
-
-	if err != nil {
-		return addressPair{}, errors.Wrap(err, 0)
-	}
-
-	for _, line := range strings.Split(string(tcp), "\n") {
-		parts := strings.Fields(line)
-
-		if len(parts) < 10 {
-			continue
-		}
-
-		if inode == parts[INODE_FILED_INDEX] {
-			srcIp, srcPort, srcErr := parseHexAddress(parts[SRC_ADDRESS_FILED_INDEX])
-
-			if srcErr != nil {
-				return addressPair{}, srcErr
-			}
-
-			dstIp, dstPort, dstErr := parseHexAddress(parts[DST_ADDRESS_FILED_INDEX])
-
-			if dstErr != nil {
-				return addressPair{}, dstErr
-			}
-
-			return addressPair{
-				srcIp:   srcIp,
-				srcPort: srcPort,
-				dstIp:   dstIp,
-				dstPort: dstPort,
-			}, nil
-		}
-	}
-
-	return addressPair{}, errors.Errorf("address not found [pid: %d] [sockfd: %d] [inode: %s]", pid, fd, inode)
-}
-
-func getSocketInode(procfs string, pid uint32, fd uint32) (string, error) {
-	fdlinkPath := fmt.Sprintf("%s/%d/fd/%d", procfs, pid, fd)
-	fdlink, err := os.Readlink(fdlinkPath)
-
-	if err != nil {
-		return "", errors.Wrap(err, 0)
-	}
-
-	tokens := socketInodeRegex.FindStringSubmatch(fdlink)
-
-	if tokens == nil || len(tokens) < 1 {
-		return "", errors.Errorf("socket inode not found [pid: %d] [sockfd: %d] [link: %s]", pid, fd, fdlink)
-	}
-
-	return tokens[1], nil
-}
-
-// Format looks like 0100007F:50 for 127.0.0.1:80
-//
-func parseHexAddress(addr string) (net.IP, uint16, error) {
-	addrParts := strings.Split(addr, ":")
-
-	port, err := strconv.ParseUint(addrParts[1], 16, 16)
-
-	if err != nil {
-		return nil, 0, errors.Wrap(err, 0)
-	}
-
-	ip, err := strconv.ParseUint(addrParts[0], 16, 32)
-
-	if err != nil {
-		return nil, 0, errors.Wrap(err, 0)
-	}
-
-	return net.IP{uint8(ip), uint8(ip >> 8), uint8(ip >> 16), uint8(ip >> 24)}, uint16(port), nil
-}

tap/tlstapper/ssllib_hooks.go

@@ -14,8 +14,6 @@ type sslHooks struct {
 	sslWriteExRetProbe link.Link
 	sslReadExProbe     link.Link
 	sslReadExRetProbe  link.Link
-	tcpSendmsg         link.Link
-	tcpRecvmsg         link.Link
 }
 
 func (s *sslHooks) installUprobes(bpfObjects *tlsTapperObjects, sslLibraryPath string) error {
@@ -105,16 +103,6 @@ func (s *sslHooks) installSslHooks(bpfObjects *tlsTapperObjects, sslLibrary *lin
 		}
 	}
 
-	s.tcpSendmsg, err = link.Kprobe("tcp_sendmsg", bpfObjects.TcpSendmsg, nil)
-	if err != nil {
-		return errors.Wrap(err, 0)
-	}
-
-	s.tcpRecvmsg, err = link.Kprobe("tcp_recvmsg", bpfObjects.TcpRecvmsg, nil)
-	if err != nil {
-		return errors.Wrap(err, 0)
-	}
-
 	return nil
 }
 
@@ -161,17 +149,5 @@ func (s *sslHooks) close() []error {
 		}
 	}
 
-	if s.tcpSendmsg != nil {
-		if err := s.tcpSendmsg.Close(); err != nil {
-			returnValue = append(returnValue, err)
-		}
-	}
-
-	if s.tcpRecvmsg != nil {
-		if err := s.tcpRecvmsg.Close(); err != nil {
-			returnValue = append(returnValue, err)
-		}
-	}
-
 	return returnValue
 }

tap/tlstapper/syscall_hooks.go

@@ -8,6 +8,8 @@ import (
 type syscallHooks struct {
 	sysEnterRead    link.Link
 	sysEnterWrite   link.Link
+	sysExitRead     link.Link
+	sysExitWrite    link.Link
 	sysEnterAccept4 link.Link
 	sysExitAccept4  link.Link
 	sysEnterConnect link.Link
@@ -29,6 +31,18 @@ func (s *syscallHooks) installSyscallHooks(bpfObjects *tlsTapperObjects) error {
 		return errors.Wrap(err, 0)
 	}
 
+	s.sysExitRead, err = link.Tracepoint("syscalls", "sys_exit_read", bpfObjects.SysExitRead, nil)
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	s.sysExitWrite, err = link.Tracepoint("syscalls", "sys_exit_write", bpfObjects.SysExitWrite, nil)
+
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
 	s.sysEnterAccept4, err = link.Tracepoint("syscalls", "sys_enter_accept4", bpfObjects.SysEnterAccept4, nil)
 
 	if err != nil {
@@ -67,6 +81,14 @@ func (s *syscallHooks) close() []error {
 		returnValue = append(returnValue, err)
 	}
 
+	if err := s.sysExitRead.Close(); err != nil {
+		returnValue = append(returnValue, err)
+	}
+
+	if err := s.sysExitWrite.Close(); err != nil {
+		returnValue = append(returnValue, err)
+	}
+
 	if err := s.sysEnterAccept4.Close(); err != nil {
 		returnValue = append(returnValue, err)
 	}

tap/tlstapper/tcp_kprobe_hooks.go

@@ -0,0 +1,45 @@
+package tlstapper
+
+import (
+	"github.com/cilium/ebpf/link"
+	"github.com/go-errors/errors"
+)
+
+type tcpKprobeHooks struct {
+	tcpSendmsg         link.Link
+	tcpRecvmsg         link.Link
+}
+
+func (s *tcpKprobeHooks) installTcpKprobeHooks(bpfObjects *tlsTapperObjects) error {
+	var err error
+
+	s.tcpSendmsg, err = link.Kprobe("tcp_sendmsg", bpfObjects.TcpSendmsg, nil)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	s.tcpRecvmsg, err = link.Kprobe("tcp_recvmsg", bpfObjects.TcpRecvmsg, nil)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	return nil
+}
+
+func (s *tcpKprobeHooks) close() []error {
+	returnValue := make([]error, 0)
+
+	if s.tcpSendmsg != nil {
+		if err := s.tcpSendmsg.Close(); err != nil {
+			returnValue = append(returnValue, err)
+		}
+	}
+
+	if s.tcpRecvmsg != nil {
+		if err := s.tcpRecvmsg.Close(); err != nil {
+			returnValue = append(returnValue, err)
+		}
+	}
+
+	return returnValue
+}

tap/tlstapper/tls_poller.go

@@ -134,10 +134,7 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsTapperTlsChunk) {
 
 func (p *tlsPoller) handleTlsChunk(chunk *tlsTapperTlsChunk, extension *api.Extension, emitter api.Emitter,
 	options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) error {
-	address, err := p.getAddressPair(chunk)
-	if err != nil {
-		return err
-	}
+	address := chunk.getAddressPair()
 
 	key := buildTlsKey(address)
 	reader, exists := p.readers[key]
@@ -156,22 +153,6 @@ func (p *tlsPoller) handleTlsChunk(chunk *tlsTapperTlsChunk, extension *api.Exte
 	return nil
 }
 
-func (p *tlsPoller) getAddressPair(chunk *tlsTapperTlsChunk) (addressPair, error) {
-	addrPairFromChunk, full := chunk.getAddressPair()
-	if full {
-		return addrPairFromChunk, nil
-	}
-
-	addrPairFromSockfd, err := p.getSockfdAddressPair(chunk)
-	if err == nil {
-		return addrPairFromSockfd, nil
-	} else {
-		logger.Log.Error("failed to get address from sock fd:", err)
-	}
-
-	return addrPairFromChunk, err
-}
-
 func (p *tlsPoller) startNewTlsReader(chunk *tlsTapperTlsChunk, address *addressPair, key string,
 	emitter api.Emitter, extension *api.Extension, options *api.TrafficFilteringOptions,
 	streamsMap api.TcpStreamMap) *tlsReader {
@@ -227,41 +208,6 @@ func (p *tlsPoller) closeReader(key string, r *tlsReader) {
 	p.closedReaders <- key
 }
 
-func (p *tlsPoller) getSockfdAddressPair(chunk *tlsTapperTlsChunk) (addressPair, error) {
-	address, err := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd)
-	fdCacheKey := fmt.Sprintf("%d:%d", chunk.Pid, chunk.Fd)
-
-	if err == nil {
-		if !chunk.isRequest() {
-			switchedAddress := addressPair{
-				srcIp:   address.dstIp,
-				srcPort: address.dstPort,
-				dstIp:   address.srcIp,
-				dstPort: address.srcPort,
-			}
-			p.fdCache.Add(fdCacheKey, switchedAddress)
-			return switchedAddress, nil
-		} else {
-			p.fdCache.Add(fdCacheKey, address)
-			return address, nil
-		}
-	}
-
-	fromCacheIfc, ok := p.fdCache.Get(fdCacheKey)
-
-	if !ok {
-		return addressPair{}, err
-	}
-
-	fromCache, ok := fromCacheIfc.(addressPair)
-
-	if !ok {
-		return address, errors.Errorf("Unable to cast %T to addressPair", fromCacheIfc)
-	}
-
-	return fromCache, nil
-}
-
 func buildTlsKey(address addressPair) string {
 	return fmt.Sprintf("%s:%d>%s:%d", address.srcIp, address.srcPort, address.dstIp, address.dstPort)
 }

tap/tlstapper/tls_tapper.go

@@ -22,6 +22,7 @@ const GlobalTapPid = 0
 type TlsTapper struct {
 	bpfObjects      tlsTapperObjects
 	syscallHooks    syscallHooks
+	tcpKprobeHooks  tcpKprobeHooks
 	sslHooksStructs []sslHooks
 	goHooksStructs  []goHooks
 	poller          *tlsPoller
@@ -63,6 +64,11 @@ func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string,
 		return err
 	}
 
+	t.tcpKprobeHooks = tcpKprobeHooks{}
+	if err := t.tcpKprobeHooks.installTcpKprobeHooks(&t.bpfObjects); err != nil {
+		return err
+	}
+
 	t.sslHooksStructs = make([]sslHooks, 0)
 
 	t.bpfLogger = newBpfLogger()
@@ -152,6 +158,8 @@ func (t *TlsTapper) Close() []error {
 
 	returnValue = append(returnValue, t.syscallHooks.close()...)
 
+	returnValue = append(returnValue, t.tcpKprobeHooks.close()...)
+
 	for _, sslHooks := range t.sslHooksStructs {
 		returnValue = append(returnValue, sslHooks.close()...)
 	}

tap/tlstapper/tlstapper46_bpfel_x86.go

@@ -27,7 +27,6 @@ type tlsTapper46TlsChunk struct {
 	Fd          uint32
 	Flags       uint32
 	AddressInfo struct {
-		Mode  int32
 		Saddr uint32
 		Daddr uint32
 		Sport uint16
@@ -99,6 +98,8 @@ type tlsTapper46ProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.ProgramSpec `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.ProgramSpec `ebpf:"sys_exit_write"`
 	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
 	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
@@ -107,18 +108,22 @@ type tlsTapper46ProgramSpecs struct {
 //
 // It can be passed ebpf.CollectionSpec.Assign.
 type tlsTapper46MapSpecs struct {
-	AcceptSyscallContext *ebpf.MapSpec `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.MapSpec `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.MapSpec `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"`
-	GoReadContext        *ebpf.MapSpec `ebpf:"go_read_context"`
-	GoWriteContext       *ebpf.MapSpec `ebpf:"go_write_context"`
-	GoidOffsetsMap       *ebpf.MapSpec `ebpf:"goid_offsets_map"`
-	Heap                 *ebpf.MapSpec `ebpf:"heap"`
-	LogBuffer            *ebpf.MapSpec `ebpf:"log_buffer"`
-	OpensslReadContext   *ebpf.MapSpec `ebpf:"openssl_read_context"`
-	OpensslWriteContext  *ebpf.MapSpec `ebpf:"openssl_write_context"`
-	PidsMap              *ebpf.MapSpec `ebpf:"pids_map"`
+	AcceptSyscallContext     *ebpf.MapSpec `ebpf:"accept_syscall_context"`
+	ChunksBuffer             *ebpf.MapSpec `ebpf:"chunks_buffer"`
+	ConnectSyscallInfo       *ebpf.MapSpec `ebpf:"connect_syscall_info"`
+	ConnectionContext        *ebpf.MapSpec `ebpf:"connection_context"`
+	GoKernelReadContext      *ebpf.MapSpec `ebpf:"go_kernel_read_context"`
+	GoKernelWriteContext     *ebpf.MapSpec `ebpf:"go_kernel_write_context"`
+	GoReadContext            *ebpf.MapSpec `ebpf:"go_read_context"`
+	GoUserKernelReadContext  *ebpf.MapSpec `ebpf:"go_user_kernel_read_context"`
+	GoUserKernelWriteContext *ebpf.MapSpec `ebpf:"go_user_kernel_write_context"`
+	GoWriteContext           *ebpf.MapSpec `ebpf:"go_write_context"`
+	GoidOffsetsMap           *ebpf.MapSpec `ebpf:"goid_offsets_map"`
+	Heap                     *ebpf.MapSpec `ebpf:"heap"`
+	LogBuffer                *ebpf.MapSpec `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.MapSpec `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.MapSpec `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.MapSpec `ebpf:"pids_map"`
 }
 
 // tlsTapper46Objects contains all objects after they have been loaded into the kernel.
@@ -140,18 +145,22 @@ func (o *tlsTapper46Objects) Close() error {
 //
 // It can be passed to loadTlsTapper46Objects or ebpf.CollectionSpec.LoadAndAssign.
 type tlsTapper46Maps struct {
-	AcceptSyscallContext *ebpf.Map `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.Map `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.Map `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"`
-	GoReadContext        *ebpf.Map `ebpf:"go_read_context"`
-	GoWriteContext       *ebpf.Map `ebpf:"go_write_context"`
-	GoidOffsetsMap       *ebpf.Map `ebpf:"goid_offsets_map"`
-	Heap                 *ebpf.Map `ebpf:"heap"`
-	LogBuffer            *ebpf.Map `ebpf:"log_buffer"`
-	OpensslReadContext   *ebpf.Map `ebpf:"openssl_read_context"`
-	OpensslWriteContext  *ebpf.Map `ebpf:"openssl_write_context"`
-	PidsMap              *ebpf.Map `ebpf:"pids_map"`
+	AcceptSyscallContext     *ebpf.Map `ebpf:"accept_syscall_context"`
+	ChunksBuffer             *ebpf.Map `ebpf:"chunks_buffer"`
+	ConnectSyscallInfo       *ebpf.Map `ebpf:"connect_syscall_info"`
+	ConnectionContext        *ebpf.Map `ebpf:"connection_context"`
+	GoKernelReadContext      *ebpf.Map `ebpf:"go_kernel_read_context"`
+	GoKernelWriteContext     *ebpf.Map `ebpf:"go_kernel_write_context"`
+	GoReadContext            *ebpf.Map `ebpf:"go_read_context"`
+	GoUserKernelReadContext  *ebpf.Map `ebpf:"go_user_kernel_read_context"`
+	GoUserKernelWriteContext *ebpf.Map `ebpf:"go_user_kernel_write_context"`
+	GoWriteContext           *ebpf.Map `ebpf:"go_write_context"`
+	GoidOffsetsMap           *ebpf.Map `ebpf:"goid_offsets_map"`
+	Heap                     *ebpf.Map `ebpf:"heap"`
+	LogBuffer                *ebpf.Map `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.Map `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.Map `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.Map `ebpf:"pids_map"`
 }
 
 func (m *tlsTapper46Maps) Close() error {
@@ -159,8 +168,12 @@ func (m *tlsTapper46Maps) Close() error {
 		m.AcceptSyscallContext,
 		m.ChunksBuffer,
 		m.ConnectSyscallInfo,
-		m.FileDescriptorToIpv4,
+		m.ConnectionContext,
+		m.GoKernelReadContext,
+		m.GoKernelWriteContext,
 		m.GoReadContext,
+		m.GoUserKernelReadContext,
+		m.GoUserKernelWriteContext,
 		m.GoWriteContext,
 		m.GoidOffsetsMap,
 		m.Heap,
@@ -197,6 +210,8 @@ type tlsTapper46Programs struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.Program `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.Program `ebpf:"sys_exit_write"`
 	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
 	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
@@ -225,6 +240,8 @@ func (p *tlsTapper46Programs) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.SysExitRead,
+		p.SysExitWrite,
 		p.TcpRecvmsg,
 		p.TcpSendmsg,
 	)

tap/tlstapper/tlstapper_bpfel_x86.go

@@ -27,7 +27,6 @@ type tlsTapperTlsChunk struct {
 	Fd          uint32
 	Flags       uint32
 	AddressInfo struct {
-		Mode  int32
 		Saddr uint32
 		Daddr uint32
 		Sport uint16
@@ -99,6 +98,8 @@ type tlsTapperProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.ProgramSpec `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.ProgramSpec `ebpf:"sys_exit_write"`
 	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
 	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
@@ -107,18 +108,22 @@ type tlsTapperProgramSpecs struct {
 //
 // It can be passed ebpf.CollectionSpec.Assign.
 type tlsTapperMapSpecs struct {
-	AcceptSyscallContext *ebpf.MapSpec `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.MapSpec `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.MapSpec `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"`
-	GoReadContext        *ebpf.MapSpec `ebpf:"go_read_context"`
-	GoWriteContext       *ebpf.MapSpec `ebpf:"go_write_context"`
-	GoidOffsetsMap       *ebpf.MapSpec `ebpf:"goid_offsets_map"`
-	Heap                 *ebpf.MapSpec `ebpf:"heap"`
-	LogBuffer            *ebpf.MapSpec `ebpf:"log_buffer"`
-	OpensslReadContext   *ebpf.MapSpec `ebpf:"openssl_read_context"`
-	OpensslWriteContext  *ebpf.MapSpec `ebpf:"openssl_write_context"`
-	PidsMap              *ebpf.MapSpec `ebpf:"pids_map"`
+	AcceptSyscallContext     *ebpf.MapSpec `ebpf:"accept_syscall_context"`
+	ChunksBuffer             *ebpf.MapSpec `ebpf:"chunks_buffer"`
+	ConnectSyscallInfo       *ebpf.MapSpec `ebpf:"connect_syscall_info"`
+	ConnectionContext        *ebpf.MapSpec `ebpf:"connection_context"`
+	GoKernelReadContext      *ebpf.MapSpec `ebpf:"go_kernel_read_context"`
+	GoKernelWriteContext     *ebpf.MapSpec `ebpf:"go_kernel_write_context"`
+	GoReadContext            *ebpf.MapSpec `ebpf:"go_read_context"`
+	GoUserKernelReadContext  *ebpf.MapSpec `ebpf:"go_user_kernel_read_context"`
+	GoUserKernelWriteContext *ebpf.MapSpec `ebpf:"go_user_kernel_write_context"`
+	GoWriteContext           *ebpf.MapSpec `ebpf:"go_write_context"`
+	GoidOffsetsMap           *ebpf.MapSpec `ebpf:"goid_offsets_map"`
+	Heap                     *ebpf.MapSpec `ebpf:"heap"`
+	LogBuffer                *ebpf.MapSpec `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.MapSpec `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.MapSpec `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.MapSpec `ebpf:"pids_map"`
 }
 
 // tlsTapperObjects contains all objects after they have been loaded into the kernel.
@@ -140,18 +145,22 @@ func (o *tlsTapperObjects) Close() error {
 //
 // It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign.
 type tlsTapperMaps struct {
-	AcceptSyscallContext *ebpf.Map `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.Map `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.Map `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"`
-	GoReadContext        *ebpf.Map `ebpf:"go_read_context"`
-	GoWriteContext       *ebpf.Map `ebpf:"go_write_context"`
-	GoidOffsetsMap       *ebpf.Map `ebpf:"goid_offsets_map"`
-	Heap                 *ebpf.Map `ebpf:"heap"`
-	LogBuffer            *ebpf.Map `ebpf:"log_buffer"`
-	OpensslReadContext   *ebpf.Map `ebpf:"openssl_read_context"`
-	OpensslWriteContext  *ebpf.Map `ebpf:"openssl_write_context"`
-	PidsMap              *ebpf.Map `ebpf:"pids_map"`
+	AcceptSyscallContext     *ebpf.Map `ebpf:"accept_syscall_context"`
+	ChunksBuffer             *ebpf.Map `ebpf:"chunks_buffer"`
+	ConnectSyscallInfo       *ebpf.Map `ebpf:"connect_syscall_info"`
+	ConnectionContext        *ebpf.Map `ebpf:"connection_context"`
+	GoKernelReadContext      *ebpf.Map `ebpf:"go_kernel_read_context"`
+	GoKernelWriteContext     *ebpf.Map `ebpf:"go_kernel_write_context"`
+	GoReadContext            *ebpf.Map `ebpf:"go_read_context"`
+	GoUserKernelReadContext  *ebpf.Map `ebpf:"go_user_kernel_read_context"`
+	GoUserKernelWriteContext *ebpf.Map `ebpf:"go_user_kernel_write_context"`
+	GoWriteContext           *ebpf.Map `ebpf:"go_write_context"`
+	GoidOffsetsMap           *ebpf.Map `ebpf:"goid_offsets_map"`
+	Heap                     *ebpf.Map `ebpf:"heap"`
+	LogBuffer                *ebpf.Map `ebpf:"log_buffer"`
+	OpensslReadContext       *ebpf.Map `ebpf:"openssl_read_context"`
+	OpensslWriteContext      *ebpf.Map `ebpf:"openssl_write_context"`
+	PidsMap                  *ebpf.Map `ebpf:"pids_map"`
 }
 
 func (m *tlsTapperMaps) Close() error {
@@ -159,8 +168,12 @@ func (m *tlsTapperMaps) Close() error {
 		m.AcceptSyscallContext,
 		m.ChunksBuffer,
 		m.ConnectSyscallInfo,
-		m.FileDescriptorToIpv4,
+		m.ConnectionContext,
+		m.GoKernelReadContext,
+		m.GoKernelWriteContext,
 		m.GoReadContext,
+		m.GoUserKernelReadContext,
+		m.GoUserKernelWriteContext,
 		m.GoWriteContext,
 		m.GoidOffsetsMap,
 		m.Heap,
@@ -197,6 +210,8 @@ type tlsTapperPrograms struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	SysExitRead                   *ebpf.Program `ebpf:"sys_exit_read"`
+	SysExitWrite                  *ebpf.Program `ebpf:"sys_exit_write"`
 	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
 	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
@@ -225,6 +240,8 @@ func (p *tlsTapperPrograms) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.SysExitRead,
+		p.SysExitWrite,
 		p.TcpRecvmsg,
 		p.TcpSendmsg,
 	)

ui-common/package.json

@@ -26,14 +26,15 @@
     "@craco/craco": "^6.4.3",
     "@types/jest": "^26.0.24",
     "@types/node": "^12.20.54",
-    "sass": "^1.52.3",
     "react": "^17.0.2",
     "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "^17.0.2",
-    "recoil": "^0.7.2"
+    "recoil": "^0.7.2",
+    "sass": "^1.52.3"
   },
   "dependencies": {
     "@craco/craco": "^6.4.3",
+    "@elastic/eui": "^60.2.0",
     "@emotion/react": "^11.9.0",
     "@emotion/styled": "^11.8.1",
     "@mui/icons-material": "^5.8.2",
@@ -72,6 +73,7 @@
   "devDependencies": {
     "@rollup/plugin-node-resolve": "^13.3.0",
     "@svgr/rollup": "^6.2.1",
+    "@types/ace": "^0.0.48",
     "cross-env": "^7.0.3",
     "env-cmd": "^10.1.0",
     "gh-pages": "^4.0.0",
@@ -84,7 +86,7 @@
     "rollup-plugin-postcss": "^4.0.2",
     "rollup-plugin-sass": "^1.2.12",
     "rollup-plugin-scss": "^3.0.0",
-    "typescript": "^4.7.2"
+    "typescript": "^4.5.3"
   },
   "eslintConfig": {
     "extends": [
@@ -93,6 +95,7 @@
     ]
   },
   "files": [
+    "src/*.scss",
     "dist"
   ]
 }

ui-common/src/components/EntryDetailed/EntryViewer/AutoRepresentation.tsx

@@ -32,7 +32,7 @@ export const AutoRepresentation: React.FC<any> = ({ representation, color, opene
                 badge: null
             }]
 
-        if (response) {
+        if (response && response.length > 0) {
             arr.push({
                 tab: 'Response',
                 badge: null
@@ -71,7 +71,7 @@ export const AutoRepresentation: React.FC<any> = ({ representation, color, opene
             {getOpenedTabIndex() === TabsEnum.Request && <React.Fragment>
                 <SectionsRepresentation data={request} color={color} requestRepresentation={request} />
             </React.Fragment>}
-            {response && getOpenedTabIndex() === TabsEnum.Response && <React.Fragment>
+            {response && response.length > 0 && getOpenedTabIndex() === TabsEnum.Response && <React.Fragment>
                 <SectionsRepresentation data={response} color={color} />
             </React.Fragment>}
         </div>}

ui-common/src/components/modals/ServiceMapModal/ServiceMapModal.module.sass

@@ -97,4 +97,7 @@ $modalMargin-from-edge : 35px
   overflow: hidden
 
 .servicesFilterList
-  height: calc(100% - 30px - 52px)
+  height: calc(100% - 30px)
+
+.protocolsFilterList
+  height: 100%

ui-common/src/components/modals/ServiceMapModal/ServiceMapModal.tsx

@@ -228,7 +228,7 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
                                 <div className={styles.filterWrapper}>
                                     <div className={styles.card}>
                                         <SelectList items={getProtocolsForFilter} checkBoxWidth="5%" tableName={"PROTOCOLS"} multiSelect={true}
-                                            checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters}
+                                            checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters + ` ${styles.protocolsFilterList}`}
                                             inputSearchClass={styles.servicesFilterSearch} isFilterable={false} />
                                     </div>
                                     <div className={styles.servicesFilterWrapper + ` ${styles.card}`}>

ui-common/src/components/modals/TrafficStatsModal/TimelineBarChart/TimeRangePicker/TimeTangePicker.tsx

@@ -0,0 +1,96 @@
+import React, { useState, Fragment } from 'react';
+import { EuiProvider } from '@elastic/eui';
+
+import {
+    EuiSuperDatePicker,
+    EuiSpacer,
+} from '@elastic/eui';
+import dateMath from '@elastic/datemath';
+import '@elastic/eui/dist/eui_theme_light.css';
+
+interface TimeRangePickerProps {
+    refreshStats: (startTime, endTime) => void;
+}
+
+export const TimeRangePicker: React.FC<TimeRangePickerProps> = ({ refreshStats }) => {
+    const [recentlyUsedRanges, setRecentlyUsedRanges] = useState([]);
+    const [isLoading, setIsLoading] = useState(false);
+    const [start, setStart] = useState('now-30m');
+    const [end, setEnd] = useState('now');
+    const [isPaused, setIsPaused] = useState(true);
+    const [refreshInterval, setRefreshInterval] = useState();
+
+    const dateConvertor = (inputStart, inputEnd) => {
+        const startMoment = dateMath.parse(inputStart);
+        if (!startMoment || !startMoment.isValid()) {
+            console.error("Unable to parse start string");
+        }
+        const endMoment = dateMath.parse(inputEnd, { roundUp: true });
+        if (!endMoment || !endMoment.isValid()) {
+            console.error("Unable to parse end string");
+        }
+        return { startMoment: startMoment.format("x"), endMoment: endMoment.format("x") }
+    }
+
+    const onTimeChange = ({ start, end }) => {
+        const recentlyUsedRange = recentlyUsedRanges.filter(recentlyUsedRange => {
+            const isDuplicate =
+                recentlyUsedRange.start === start && recentlyUsedRange.end === end;
+            return !isDuplicate;
+        });
+        recentlyUsedRange.unshift({ start, end });
+        setStart(start);
+        setEnd(end);
+        setRecentlyUsedRanges(
+            recentlyUsedRange.length > 10
+                ? recentlyUsedRange.slice(0, 9)
+                : recentlyUsedRange
+        );
+        const { startMoment, endMoment } = dateConvertor(start, end)
+        refreshStats(startMoment, endMoment)
+        setIsLoading(true);
+        startLoading();
+    };
+
+    const onRefresh = ({ start, end, refreshInterval }) => {
+        return new Promise(resolve => {
+            setTimeout(resolve, 100);
+        }).then(() => {
+            const { startMoment, endMoment } = dateConvertor(start, end)
+            refreshStats(startMoment, endMoment)
+        });
+    };
+
+    const startLoading = () => {
+        setTimeout(stopLoading, 1000);
+    };
+    const stopLoading = () => {
+        setIsLoading(false);
+    };
+
+    const onRefreshChange = ({ isPaused, refreshInterval }) => {
+        setIsPaused(isPaused);
+        setRefreshInterval(refreshInterval);
+    };
+
+    return (
+        <EuiProvider>
+            <Fragment>
+                <EuiSpacer />
+                <EuiSuperDatePicker
+                    width='auto'
+                    isLoading={isLoading}
+                    start={start}
+                    end={end}
+                    onTimeChange={onTimeChange}
+                    onRefresh={onRefresh}
+                    isPaused={isPaused}
+                    refreshInterval={refreshInterval}
+                    onRefreshChange={onRefreshChange}
+                    recentlyUsedRanges={recentlyUsedRanges}
+                />
+                <EuiSpacer />
+            </Fragment>
+        </EuiProvider>
+    );
+};

ui-common/src/components/modals/TrafficStatsModal/TimelineBarChart/TimelineBarChart.module.sass

@@ -1,3 +1,5 @@
+@import  "../../../../variables.module"
+
 .barChartContainer
     width: 100%
     display: flex 
@@ -6,3 +8,4 @@
 .axisText
     font-size: 12px
     opacity: 0.9
+    font-family: $text-font-family

ui-common/src/components/modals/TrafficStatsModal/TrafficPieChart/TrafficPieChart.module.sass

@@ -0,0 +1,5 @@
+@import  "../../../../variables.module"
+
+.entryName
+    margin-left: 5px
+    font-family: $text-font-family

ui-common/src/components/modals/TrafficStatsModal/TrafficPieChart/TrafficPieChart.tsx

@@ -2,6 +2,7 @@ import React, { useEffect, useMemo, useState } from "react";
 import { Cell, Legend, Pie, PieChart, Tooltip } from "recharts";
 import { Utils } from "../../../../helpers/Utils";
 import { ALL_PROTOCOLS ,StatsMode as PieChartMode } from "../consts"
+import styles from "./TrafficPieChart.module.sass";
 
 const RADIAN = Math.PI / 180;
 const renderCustomizedLabel = ({
@@ -76,7 +77,7 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
     if (selectedProtocol === ALL_PROTOCOLS) {
       legend = data.map(protocol => <div style={{ marginBottom: 5, display: "flex" }}>
         <div style={{ height: 15, width: 30, background: protocol?.color }} />
-        <span style={{ marginLeft: 5 }}>
+        <span className={styles.entryName}>
           {protocol.name}
         </span>
       </div>)
@@ -84,7 +85,7 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
       legend = data.find(protocol => protocol.name === selectedProtocol)?.methods.map((method) => <div
         style={{ marginBottom: 5, display: "flex" }}>
         <div style={{ height: 15, width: 30, background: method.color}} />
-        <span style={{ marginLeft: 5 }}>
+        <span className={styles.entryName}>
           {method.name}
         </span>
       </div>)

ui-common/src/components/modals/TrafficStatsModal/TrafficStatsModal.module.sass

@@ -1,9 +1,11 @@
+@import  "../../../variables.module"
+  
 .headlineContainer
   display: flex
 
 .title
   color: #494677
-  font-family: Source Sans Pro,Lucida Grande,Tahoma,sans-serif
+  font-family: $text-font-family
   font-size: 28px
   font-weight: 600
 
@@ -13,16 +15,21 @@
   top: 20px
 
 .mainContainer
-  padding: 30px
   text-align: center
 
 .selectContainer
   display: flex
   justify-content: space-evenly
+  align-items: center
   margin-bottom: 4%
 
+.selectTitle
+  font-family: $text-font-family
+  margin-right: 15px
+
 .select
   border: none
   border-bottom: 1px black solid
   outline: none
   width: 100px
+  font-family: $text-font-family

ui-common/src/components/modals/TrafficStatsModal/TrafficStatsModal.tsx

@@ -1,13 +1,12 @@
 import React, { useCallback, useEffect, useState } from "react";
-import { Backdrop, Box, Button, debounce, Fade, Modal } from "@mui/material";
+import { Backdrop, Box, debounce, Fade, Modal } from "@mui/material";
 import styles from "./TrafficStatsModal.module.sass";
 import closeIcon from "assets/close.svg";
 import { TrafficPieChart } from "./TrafficPieChart/TrafficPieChart";
 import { TimelineBarChart } from "./TimelineBarChart/TimelineBarChart";
-import refreshIcon from "assets/refresh.svg";
-import { useCommonStyles } from "../../../helpers/commonStyle";
 import { LoadingWrapper } from "../../UI/withLoading/withLoading";
 import { ALL_PROTOCOLS, StatsMode } from "./consts";
+import { TimeRangePicker } from "./TimelineBarChart/TimeRangePicker/TimeTangePicker";
 
 const modalStyle = {
   position: 'absolute',
@@ -15,7 +14,7 @@ const modalStyle = {
   left: '50%',
   transform: 'translate(-50%, 0%)',
   width: '60vw',
-  height: '82vh',
+  height: '90vh',
   bgcolor: 'background.paper',
   borderRadius: '5px',
   boxShadow: 24,
@@ -26,11 +25,10 @@ const modalStyle = {
 interface TrafficStatsModalProps {
   isOpen: boolean;
   onClose: () => void;
-  getTrafficStatsDataApi: () => Promise<any>
+  getTrafficStatsDataApi: (start?, end?) => Promise<any>
 }
 
 export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, onClose, getTrafficStatsDataApi }) => {
-
   const modes = Object.keys(StatsMode).filter(x => !(parseInt(x) >= 0));
   const [statsMode, setStatsMode] = useState(modes[0]);
   const [selectedProtocol, setSelectedProtocol] = useState(ALL_PROTOCOLS);
@@ -38,14 +36,13 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
   const [timelineStatsData, setTimelineStatsData] = useState(null);
   const [protocols, setProtocols] = useState([])
   const [isLoading, setIsLoading] = useState(false);
-  const commonClasses = useCommonStyles();
 
-  const getTrafficStats = useCallback(async () => {
+  const getTrafficStats = useCallback(async (startTime, endTime) => {
     if (isOpen && getTrafficStatsDataApi) {
       (async () => {
         try {
           setIsLoading(true);
-          const statsData = await getTrafficStatsDataApi();
+          const statsData = await getTrafficStatsDataApi(startTime, endTime);
           setPieStatsData(statsData.pie);
           setTimelineStatsData(statsData.timeline);
           setProtocols(statsData.protocols)
@@ -59,11 +56,13 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
   }, [isOpen, getTrafficStatsDataApi, setPieStatsData, setTimelineStatsData])
 
   useEffect(() => {
-    getTrafficStats();
+    const now = new Date().getTime();
+    const halfAnHourAgo = now - (30 * 60 * 1000);
+    getTrafficStats(halfAnHourAgo, now);
   }, [getTrafficStats])
 
-  const refreshStats = debounce(() => {
-    getTrafficStats();
+  const refreshStats = debounce((newStartTime, newEndTime) => {
+    getTrafficStats(newStartTime, newEndTime);
   }, 500);
 
   return (
@@ -82,30 +81,24 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
           </div>
           <div className={styles.headlineContainer}>
             <div className={styles.title}>Traffic Statistics</div>
-            <Button style={{ marginLeft: "2%", textTransform: 'unset' }}
-              startIcon={<img src={refreshIcon} className="custom" alt="refresh"></img>}
-              size="medium"
-              variant="contained"
-              className={commonClasses.outlinedButton + " " + commonClasses.imagedButton}
-              onClick={refreshStats}
-            >
-              Refresh
-            </Button>
           </div>
           <div className={styles.mainContainer}>
             <div className={styles.selectContainer}>
               <div>
-                <span style={{ marginRight: 15 }}>Breakdown By</span>
+                <span className={styles.selectTitle}>Breakdown By</span>
                 <select className={styles.select} value={statsMode} onChange={(e) => setStatsMode(e.target.value)}>
                   {modes.map(mode => <option key={mode} value={mode}>{mode}</option>)}
                 </select>
               </div>
               <div>
-                <span style={{ marginRight: 15 }}>Protocol</span>
+                <span className={styles.selectTitle}>Protocol</span>
                 <select className={styles.select} value={selectedProtocol} onChange={(e) => setSelectedProtocol(e.target.value)}>
                   {protocols.map(protocol => <option key={protocol} value={protocol}>{protocol}</option>)}
                 </select>
               </div>
+              <div>
+                <TimeRangePicker refreshStats={refreshStats} />
+              </div>
             </div>
             <div>
               <LoadingWrapper isLoading={isLoading} loaderMargin={20} loaderHeight={50}>

ui-common/src/variables.module.scss

@@ -14,6 +14,8 @@ $content-section-color: #f8f9fc;
 $blue-gray: #494677;
 $light-gray: #8F9BB2;
 
+$text-font-family: Source Sans Pro, Lucida Grande, Tahoma, sans-serif;
+
 :export {
   mainBackgroundColor: $main-background-color;
   headerBackgroundColor: $header-background-color;
@@ -25,4 +27,6 @@ $light-gray: #8F9BB2;
   failureColor: $failure-color;
   blueGray: $blue-gray;
   lightGray: $light-gray;
+  contentSectionColor: $content-section-color;
+  textFontFamily: $text-font-family
 }

ui/package.json

@@ -4,6 +4,8 @@
   "private": true,
   "dependencies": {
     "@craco/craco": "^6.4.3",
+    "@elastic/datemath": "^5.0.3",
+    "@elastic/eui": "^60.2.0",
     "@emotion/react": "^11.9.0",
     "@emotion/styled": "^11.8.1",
     "@mui/material": "^5.8.2",
@@ -23,7 +25,6 @@
     "mobx": "^6.6.0",
     "moment": "^2.29.3",
     "node-fetch": "^3.2.4",
-    "sass": "^1.52.3",
     "numeral": "^2.0.6",
     "react": "^17.0.2",
     "react-copy-to-clipboard": "^5.1.0",
@@ -35,8 +36,9 @@
     "react-syntax-highlighter": "^15.5.0",
     "react-toastify": "^8.2.0",
     "redoc": "^2.0.0-rc.71",
+    "sass": "^1.52.3",
     "styled-components": "^5.3.5",
-    "typescript": "^4.7.2",
+    "typescript": "^4.5.3",
     "web-vitals": "^2.1.4",
     "xml-formatter": "^2.6.1"
   },

ui/src/App.tsx

@@ -8,7 +8,7 @@ import oasModalOpenAtom from './recoil/oasModalOpen/atom';
 import trafficStatsModalOpenAtom from "./recoil/trafficStatsModalOpen";
 import { OasModal } from '@up9/mizu-common';
 import Api from './helpers/api';
-import {ThemeProvider, StyledEngineProvider, createTheme} from '@mui/material';
+import { ThemeProvider, StyledEngineProvider, createTheme } from '@mui/material';
 import { TrafficStatsModal } from '@up9/mizu-common';
 
 const api = Api.getInstance()
@@ -36,7 +36,7 @@ const App = () => {
                         openModal={oasModalOpen}
                         handleCloseModal={() => setOasModalOpen(false)}
                     />}
-                    <TrafficStatsModal isOpen={trafficStatsModalOpen} onClose={() => setTrafficStatsModalOpen(false)} getTrafficStatsDataApi={api.getTrafficStats}/>
+                    <TrafficStatsModal isOpen={trafficStatsModalOpen} onClose={() => setTrafficStatsModalOpen(false)} getTrafficStatsDataApi={api.getTrafficStats} />
                 </div>
             </ThemeProvider>
         </StyledEngineProvider>

ui/src/helpers/api.js

@@ -116,8 +116,8 @@ export default class Api {
         });
     }
 
-    getTrafficStats = async () => {
-        const response = await client.get("/status/trafficStats");
+    getTrafficStats = async (startTimeMs, endTimeMs) => {
+        const response = await client.get("/status/trafficStats", {params: {startTimeMs, endTimeMs}});
         return response.data;
     }
 }