Generate bpf files before running tests (#1194)

Do not track object files in git.
Generate the files with `make bpf` or during `make agent`.

.github/workflows/static_code_analysis.yml

@@ -32,6 +32,10 @@ jobs:
         id: agent_modified_files
         run: devops/check_modified_files.sh agent/
 
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
       - name: Go lint - agent
         uses: golangci/golangci-lint-action@v2
         if: steps.agent_modified_files.outputs.matched == 'true'

.github/workflows/test.yml

@@ -40,6 +40,10 @@ jobs:
         run: |
           ./devops/install-capstone.sh
 
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
       - name: Check CLI modified files
         id: cli_modified_files
         run: devops/check_modified_files.sh cli/

.gitignore

@@ -56,3 +56,6 @@ tap/extensions/*/expect
 
 # Ignore *.log files
 *.log
+
+# Object files
+*.o

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent agent-debug cli tap docker
+.PHONY: help ui agent agent-debug cli tap docker bpf clean-bpf
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -20,6 +20,13 @@ TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
 export VER?=0.0
+ARCH=$(shell uname -m)
+ifeq ($(ARCH),$(filter $(ARCH),aarch64 arm64))
+	BPF_O_ARCH_LABEL=arm64
+else
+	BPF_O_ARCH_LABEL=x86
+endif
+BPF_O_FILES = tap/tlstapper/tlstapper46_bpfel_$(BPF_O_ARCH_LABEL).o tap/tlstapper/tlstapper_bpfel_$(BPF_O_ARCH_LABEL).o
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -31,11 +38,17 @@ cli: ## Build CLI.
 cli-debug: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build-debug
 
-agent: ## Build agent.
+agent: bpf ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
 	@ls -l agent/build
 
+bpf: $(BPF_O_FILES)
+
+$(BPF_O_FILES): $(wildcard tap/tlstapper/bpf/**/*.[ch])
+	@(echo "building tlstapper bpf")
+	@(./tap/tlstapper/bpf-builder/build.sh)
+
 agent-debug: ## Build agent for debug.
 	@(echo "building mizu agent for debug.." )
 	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
@@ -76,6 +89,9 @@ clean-cli:  ## Clean CLI.
 clean-docker:  ## Run clean docker
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+clean-bpf:
+	@(rm $(BPF_O_FILES) ; echo "bpf cleanup done" )
+
 test-lint:  ## Run lint on all modules
 	cd agent && golangci-lint run
 	cd shared && golangci-lint run

agent/pkg/api/entry_streamer_socket_connector.go

@@ -22,7 +22,7 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
 	if params.EnableFullEntries {
 		message, _ = models.CreateFullEntryWebSocketMessage(entry)
 	} else {
-		protocol, ok := protocolsMap[entry.ProtocolId]
+		protocol, ok := protocolsMap[entry.Protocol.ToString()]
 		if !ok {
 			return fmt.Errorf("protocol not found, protocol: %v", protocol)
 		}

agent/pkg/api/main.go

@@ -126,7 +126,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
 
 		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
-		oasGenerator.HandleEntry(mizuEntry, &item.Protocol)
+		oasGenerator.HandleEntry(mizuEntry)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -97,7 +97,9 @@ func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool
 	websocketIdsLock.Unlock()
 
 	defer func() {
-		socketCleanup(socketId, connectedWebsockets[socketId])
+		if socketConnection := connectedWebsockets[socketId]; socketConnection != nil {
+			socketCleanup(socketId, socketConnection)
+		}
 	}()
 
 	eventHandlers.WebSocketConnect(c, socketId, isTapper)

agent/pkg/controllers/service_map_controller_test.go

@@ -36,11 +36,13 @@ var (
 )
 
 var ProtocolHttp = &tapApi.Protocol{
-	Name:            "http",
+	ProtocolSummary: tapApi.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.1",
+		Abbreviation: "HTTP",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
-	Abbreviation:    "HTTP",
 	Macro:           "http",
-	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,

agent/pkg/entries/entries_provider.go

@@ -38,7 +38,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 			return nil, nil, err
 		}
 
-		protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+		protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
 		if !ok {
 			return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
 		}
@@ -77,7 +77,7 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 		return nil, errors.New(string(bytes))
 	}
 
-	protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+	protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
 	if !ok {
 		return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
 	}

agent/pkg/oas/oas_generator.go

@@ -16,7 +16,7 @@ var (
 )
 
 type OasGeneratorSink interface {
-	HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol)
+	HandleEntry(mizuEntry *api.Entry)
 }
 
 type OasGenerator interface {
@@ -58,12 +58,12 @@ func (g *defaultOasGenerator) IsStarted() bool {
 	return g.started
 }
 
-func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol) {
+func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
 	if !g.started {
 		return
 	}
 
-	if protocol.Name == "http" {
+	if mizuEntry.Protocol.Name == "http" {
 		dest := mizuEntry.Destination.Name
 		if dest == "" {
 			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
@@ -85,7 +85,7 @@ func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry, protocol *api.Pr
 
 		g.handleHARWithSource(entryWSource)
 	} else {
-		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, protocol.Name)
+		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, mizuEntry.Protocol.Name)
 	}
 }
 

agent/pkg/providers/stats_provider_test.go

@@ -26,7 +26,7 @@ func TestEntryAddedCount(t *testing.T) {
 
 	entryBucketKey := time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC)
 	valueLessThanBucketThreshold := time.Second * 130
-	mockSummery := &api.BaseEntry{Protocol: api.Protocol{Name: "mock"}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
 			for i := 0; i < entriesCount; i++ {
@@ -61,7 +61,7 @@ func TestEntryAddedVolume(t *testing.T) {
 	var expectedEntriesCount int
 	var expectedVolumeInGB float64
 
-	mockSummery := &api.BaseEntry{Protocol: api.Protocol{Name: "mock"}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
 
 	for _, data := range tests {
 		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {

agent/pkg/servicemap/servicemap_test.go

@@ -50,11 +50,13 @@ var (
 		IP:   fmt.Sprintf("%s.%s", Ip, UnresolvedNodeName),
 	}
 	ProtocolHttp = &tapApi.Protocol{
-		Name:            "http",
+		ProtocolSummary: tapApi.ProtocolSummary{
+			Name:         "http",
+			Version:      "1.1",
+			Abbreviation: "HTTP",
+		},
 		LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
-		Abbreviation:    "HTTP",
 		Macro:           "http",
-		Version:         "1.1",
 		BackgroundColor: "#205cf5",
 		ForegroundColor: "#ffffff",
 		FontSize:        12,
@@ -63,11 +65,13 @@ var (
 		Priority:        0,
 	}
 	ProtocolRedis = &tapApi.Protocol{
-		Name:            "redis",
+		ProtocolSummary: tapApi.ProtocolSummary{
+			Name:         "redis",
+			Version:      "3.x",
+			Abbreviation: "REDIS",
+		},
 		LongName:        "Redis Serialization Protocol",
-		Abbreviation:    "REDIS",
 		Macro:           "redis",
-		Version:         "3.x",
 		BackgroundColor: "#a41e11",
 		ForegroundColor: "#ffffff",
 		FontSize:        11,

tap/api/api.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"bufio"
+	"fmt"
 	"net"
 	"sync"
 	"time"
@@ -14,12 +15,20 @@ const UnknownNamespace = ""
 var UnknownIp = net.IP{0, 0, 0, 0}
 var UnknownPort uint16 = 0
 
+type ProtocolSummary struct {
+	Name         string `json:"name"`
+	Version      string `json:"version"`
+	Abbreviation string `json:"abbr"`
+}
+
+func (protocol *ProtocolSummary) ToString() string {
+	return fmt.Sprintf("%s?%s?%s", protocol.Name, protocol.Version, protocol.Abbreviation)
+}
+
 type Protocol struct {
-	Name            string   `json:"name"`
+	ProtocolSummary
 	LongName        string   `json:"longName"`
-	Abbreviation    string   `json:"abbr"`
 	Macro           string   `json:"macro"`
-	Version         string   `json:"version"`
 	BackgroundColor string   `json:"backgroundColor"`
 	ForegroundColor string   `json:"foregroundColor"`
 	FontSize        int8     `json:"fontSize"`
@@ -151,7 +160,7 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 
 type Entry struct {
 	Id           string                 `json:"id"`
-	ProtocolId   string                 `json:"protocol"`
+	Protocol     ProtocolSummary        `json:"protocol"`
 	Capture      Capture                `json:"capture"`
 	Source       *TCP                   `json:"src"`
 	Destination  *TCP                   `json:"dst"`

tap/extensions/amqp/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/amqp/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/amqp/\* expect

tap/extensions/amqp/main.go

@@ -13,11 +13,13 @@ import (
 )
 
 var protocol = api.Protocol{
-	Name:            "amqp",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "amqp",
+		Version:      "0-9-1",
+		Abbreviation: "AMQP",
+	},
 	LongName:        "Advanced Message Queuing Protocol 0-9-1",
-	Abbreviation:    "AMQP",
 	Macro:           "amqp",
-	Version:         "0-9-1",
 	BackgroundColor: "#ff6600",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -27,7 +29,7 @@ var protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
+	protocol.ToString(): &protocol,
 }
 
 type dissecting string
@@ -222,8 +224,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 	reqDetails["method"] = request["method"]
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Protocol: protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -285,7 +287,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -329,7 +331,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`amqp`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
+		`amqp`: fmt.Sprintf(`protocol.name == "%s"`, protocol.Name),
 	}
 }
 

tap/extensions/amqp/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"amqp": `protocol == "amqp/0-9-1/AMQP"`,
+		"amqp": `protocol.name == "amqp"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/http/\* expect

tap/extensions/http/main.go

@@ -15,11 +15,13 @@ import (
 )
 
 var http10protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.0",
+		Abbreviation: "HTTP",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.0",
-	Abbreviation:    "HTTP",
 	Macro:           "http",
-	Version:         "1.0",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -29,11 +31,13 @@ var http10protocol = api.Protocol{
 }
 
 var http11protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.1",
+		Abbreviation: "HTTP",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
-	Abbreviation:    "HTTP",
 	Macro:           "http",
-	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -43,11 +47,13 @@ var http11protocol = api.Protocol{
 }
 
 var http2Protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "2.0",
+		Abbreviation: "HTTP/2",
+	},
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2)",
-	Abbreviation:    "HTTP/2",
 	Macro:           "http2",
-	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -57,11 +63,13 @@ var http2Protocol = api.Protocol{
 }
 
 var grpcProtocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "2.0",
+		Abbreviation: "gRPC",
+	},
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ gRPC over HTTP/2 ]",
-	Abbreviation:    "gRPC",
 	Macro:           "grpc",
-	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -71,11 +79,13 @@ var grpcProtocol = api.Protocol{
 }
 
 var graphQL1Protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.1",
+		Abbreviation: "GQL",
+	},
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1 [ GraphQL over HTTP/1.1 ]",
-	Abbreviation:    "GQL",
 	Macro:           "gql",
-	Version:         "1.1",
 	BackgroundColor: "#e10098",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -85,11 +95,13 @@ var graphQL1Protocol = api.Protocol{
 }
 
 var graphQL2Protocol = api.Protocol{
-	Name:            "http",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "http",
+		Version:      "2.0",
+		Abbreviation: "GQL",
+	},
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ GraphQL over HTTP/2 ]",
-	Abbreviation:    "GQL",
 	Macro:           "gql",
-	Version:         "2.0",
 	BackgroundColor: "#e10098",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -99,12 +111,12 @@ var graphQL2Protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation):       &http10protocol,
+	http10protocol.ToString():   &http10protocol,
-	fmt.Sprintf("%s/%s/%s", http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation):       &http11protocol,
+	http11protocol.ToString():   &http11protocol,
-	fmt.Sprintf("%s/%s/%s", http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation):          &http2Protocol,
+	http2Protocol.ToString():    &http2Protocol,
-	fmt.Sprintf("%s/%s/%s", grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation):             &grpcProtocol,
+	grpcProtocol.ToString():     &grpcProtocol,
-	fmt.Sprintf("%s/%s/%s", graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation): &graphQL1Protocol,
+	graphQL1Protocol.ToString(): &graphQL1Protocol,
-	fmt.Sprintf("%s/%s/%s", graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation): &graphQL2Protocol,
+	graphQL2Protocol.ToString(): &graphQL2Protocol,
 }
 
 const (
@@ -294,8 +306,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", item.Protocol.Name, item.Protocol.Version, item.Protocol.Abbreviation),
+		Protocol: item.Protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -328,7 +340,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -515,10 +527,10 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`http`:  fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation, http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation),
+		`http`:  fmt.Sprintf(`protocol.abbr == "%s"`, http11protocol.Abbreviation),
-		`http2`: fmt.Sprintf(`protocol == "%s/%s/%s"`, http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation),
+		`http2`: fmt.Sprintf(`protocol.abbr == "%s"`, http2Protocol.Abbreviation),
-		`grpc`:  fmt.Sprintf(`protocol == "%s/%s/%s"`, grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation),
+		`grpc`:  fmt.Sprintf(`protocol.abbr == "%s"`, grpcProtocol.Abbreviation),
-		`gql`:   fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation, graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation),
+		`gql`:   fmt.Sprintf(`protocol.abbr == "%s"`, graphQL1Protocol.Abbreviation),
 	}
 }
 

tap/extensions/http/main_test.go

@@ -44,10 +44,10 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"http":  `protocol == "http/1.0/HTTP" or protocol == "http/1.1/HTTP"`,
+		"http":  `protocol.abbr == "HTTP"`,
-		"http2": `protocol == "http/2.0/HTTP/2"`,
+		"http2": `protocol.abbr == "HTTP/2"`,
-		"grpc":  `protocol == "http/2.0/gRPC"`,
+		"grpc":  `protocol.abbr == "gRPC"`,
-		"gql":   `protocol == "http/1.1/GQL" or protocol == "http/2.0/GQL"`,
+		"gql":   `protocol.abbr == "GQL"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/kafka/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/kafka/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/kafka/\* expect

tap/extensions/kafka/main.go

@@ -11,11 +11,13 @@ import (
 )
 
 var _protocol = api.Protocol{
-	Name:            "kafka",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "kafka",
+		Version:      "12",
+		Abbreviation: "KAFKA",
+	},
 	LongName:        "Apache Kafka Protocol",
-	Abbreviation:    "KAFKA",
 	Macro:           "kafka",
-	Version:         "12",
 	BackgroundColor: "#000000",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -25,7 +27,7 @@ var _protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation): &_protocol,
+	_protocol.ToString(): &_protocol,
 }
 
 type dissecting string
@@ -70,8 +72,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation),
+		Protocol: _protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -195,7 +197,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -250,7 +252,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`kafka`: fmt.Sprintf(`protocol == "%s/%s/%s"`, _protocol.Name, _protocol.Version, _protocol.Abbreviation),
+		`kafka`: fmt.Sprintf(`protocol.name == "%s"`, _protocol.Name),
 	}
 }
 

tap/extensions/kafka/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"kafka": `protocol == "kafka/12/KAFKA"`,
+		"kafka": `protocol.name == "kafka"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/redis/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/redis/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/redis/\* expect

tap/extensions/redis/main.go

@@ -11,11 +11,13 @@ import (
 )
 
 var protocol = api.Protocol{
-	Name:            "redis",
+	ProtocolSummary: api.ProtocolSummary{
+		Name:         "redis",
+		Version:      "3.x",
+		Abbreviation: "REDIS",
+	},
 	LongName:        "Redis Serialization Protocol",
-	Abbreviation:    "REDIS",
 	Macro:           "redis",
-	Version:         "3.x",
 	BackgroundColor: "#a41e11",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -25,7 +27,7 @@ var protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
+	protocol.ToString(): &protocol,
 }
 
 type dissecting string
@@ -78,8 +80,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Protocol: protocol.ProtocolSummary,
-		Capture:    item.Capture,
+		Capture:  item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -123,7 +125,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.ProtocolId],
+		Protocol:     *protocolsMap[entry.Protocol.ToString()],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -151,7 +153,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`redis`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
+		`redis`: fmt.Sprintf(`protocol.name == "%s"`, protocol.Name),
 	}
 }
 

tap/extensions/redis/main_test.go

@@ -45,7 +45,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"redis": `protocol == "redis/3.x/REDIS"`,
+		"redis": `protocol.name == "redis"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/tlstapper/bpf-builder/build.sh

@@ -9,7 +9,7 @@ docker build -t mizu-ebpf-builder . || exit 1
 BPF_TARGET=amd64
 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
 ARCH=$(uname -m)
-if [[ $ARCH == "aarch64" ]]; then
+if [[ $ARCH == "aarch64" || $ARCH == "arm64" ]]; then
     BPF_TARGET=arm64
     BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
 fi
@@ -18,7 +18,7 @@ docker run --rm \
 	--name mizu-ebpf-builder \
 	-v $MIZU_HOME:/mizu \
 	-v $(go env GOPATH):/root/go \
-	-it mizu-ebpf-builder \
+	mizu-ebpf-builder \
 	sh -c "
 		BPF_TARGET=\"$BPF_TARGET\" BPF_CFLAGS=\"$BPF_CFLAGS\" go generate tap/tlstapper/tls_tapper.go
         chown $(id -u):$(id -g) tap/tlstapper/tlstapper*_bpf*

tap/tlstapper/bpf/common.c

@@ -12,7 +12,7 @@ Copyright (C) UP9 Inc.
 #include "include/common.h"
 
 
-static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd) {
+static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd, struct ssl_info* info) {
     __u32 pid = id >> 32;
     __u64 key = (__u64) pid << 32 | fd;
 
@@ -22,14 +22,29 @@ static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_
         return 0;
     }
 
-    int err = bpf_probe_read(chunk->address, sizeof(chunk->address), fdinfo->ipv4_addr);
+    int err;
-    chunk->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT);
 
-    if (err != 0) {
+    switch (info->address_info.mode) {
-        log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
+        case ADDRESS_INFO_MODE_UNDEFINED:
-        return 0;
+            chunk->address_info.mode = ADDRESS_INFO_MODE_SINGLE;
+            err = bpf_probe_read(&chunk->address_info.sport, sizeof(chunk->address_info.sport), &fdinfo->ipv4_addr[2]);
+            if (err != 0) {
+                log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
+                return 0;
+            }
+
+            err = bpf_probe_read(&chunk->address_info.saddr, sizeof(chunk->address_info.saddr), &fdinfo->ipv4_addr[4]);
+            if (err != 0) {
+                log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
+                return 0;
+            }
+            break;
+        default:
+            bpf_probe_read(&chunk->address_info, sizeof(chunk->address_info), &info->address_info);
     }
 
+    chunk->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT);
+
     return 1;
 }
 
@@ -104,7 +119,7 @@ static __always_inline void output_ssl_chunk(struct pt_regs *ctx, struct ssl_inf
     chunk->len = count_bytes;
     chunk->fd = info->fd;
 
-    if (!add_address_to_chunk(ctx, chunk, id, chunk->fd)) {
+    if (!add_address_to_chunk(ctx, chunk, id, chunk->fd, info)) {
         // Without an address, we drop the chunk because there is not much to do with it in Go
         //
         return;

tap/tlstapper/bpf/include/common.h

@@ -7,9 +7,11 @@ Copyright (C) UP9 Inc.
 #ifndef __COMMON__
 #define __COMMON__
 
+#define AF_INET	2	/* Internet IP Protocol */
+
 const __s32 invalid_fd = -1;
 
-static int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd);
+static int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd, struct ssl_info* info);
 static void send_chunk_part(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk, int start, int end);
 static void send_chunk(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk);
 static void output_ssl_chunk(struct pt_regs *ctx, struct ssl_info* info, int count_bytes, __u64 id, __u32 flags);

tap/tlstapper/bpf/include/headers.h

@@ -15,6 +15,7 @@ Copyright (C) UP9 Inc.
 
 #include "legacy_kernel.h"
 
+#include <bpf/bpf_endian.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 #include <bpf/bpf_core_read.h>

tap/tlstapper/bpf/include/logger_messages.h

@@ -26,6 +26,11 @@ Copyright (C) UP9 Inc.
 #define LOG_ERROR_PUTTING_CONNECT_INFO (14)
 #define LOG_ERROR_GETTING_CONNECT_INFO (15)
 #define LOG_ERROR_READING_CONNECT_INFO (16)
+#define LOG_ERROR_READING_SOCKET_FAMILY (17)
+#define LOG_ERROR_READING_SOCKET_DADDR (18)
+#define LOG_ERROR_READING_SOCKET_SADDR (19)
+#define LOG_ERROR_READING_SOCKET_DPORT (20)
+#define LOG_ERROR_READING_SOCKET_SPORT (21)
 
 // Sometimes we have the same error, happening from different locations.
 // 	in order to be able to distinct between them in the log, we add an 

tap/tlstapper/bpf/include/maps.h

@@ -24,6 +24,21 @@ Copyright (C) UP9 Inc.
 //  
 //  Be careful when editing, alignment and padding should be exactly the same in go/c.
 //
+
+typedef enum {
+    ADDRESS_INFO_MODE_UNDEFINED,
+    ADDRESS_INFO_MODE_SINGLE,
+    ADDRESS_INFO_MODE_PAIR,
+} address_info_mode;
+
+struct address_info {
+    address_info_mode mode;
+    __be32 saddr;
+    __be32 daddr;
+    __be16 sport;
+    __be16 dport;
+};
+
 struct tls_chunk {
     __u32 pid;
     __u32 tgid;
@@ -32,7 +47,7 @@ struct tls_chunk {
     __u32 recorded;
     __u32 fd;
     __u32 flags;
-    __u8 address[16];
+    struct address_info address_info;
     __u8 data[CHUNK_SIZE]; // Must be N^2
 };
 
@@ -41,6 +56,7 @@ struct ssl_info {
     __u32 buffer_len;
     __u32 fd;
     __u64 created_at_nano;
+    struct address_info address_info;
     
     // for ssl_write and ssl_read must be zero
     // for ssl_write_ex and ssl_read_ex save the *written/*readbytes pointer. 

tap/tlstapper/bpf/openssl_uprobes.c

@@ -42,6 +42,8 @@ static __always_inline int get_count_bytes(struct pt_regs *ctx, struct ssl_info*
 }
 
 static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buffer, int num, struct bpf_map_def* map_fd, size_t *count_ptr) {
+	long err;
+
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -53,7 +55,7 @@ static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buf
 	info.count_ptr = count_ptr;
 	info.buffer = buffer;
 	
-	long err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
+	err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
 	
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, id, err, 0l);
@@ -66,7 +68,7 @@ static __always_inline void ssl_uretprobe(struct pt_regs *ctx, struct bpf_map_de
 	if (!should_tap(id >> 32)) {
 		return;
 	}
-	
+
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(map_fd, &id);
 	
 	if (infoPtr == NULL) {
@@ -99,10 +101,10 @@ static __always_inline void ssl_uretprobe(struct pt_regs *ctx, struct bpf_map_de
 		return;
 	}
 
-    int count_bytes = get_count_bytes(ctx, &info, id);
+	int count_bytes = get_count_bytes(ctx, &info, id);
-    if (count_bytes <= 0) {
+	if (count_bytes <= 0) {
-        return;
+		return;
-    }
+	}
 
 	output_ssl_chunk(ctx, &info, count_bytes, id, flags);
 }

tap/tlstapper/bpf/tcp_kprobes.c

@@ -0,0 +1,79 @@
+#include "include/headers.h"
+#include "include/maps.h"
+#include "include/log.h"
+#include "include/logger_messages.h"
+#include "include/pids.h"
+#include "include/common.h"
+
+static __always_inline void tcp_kprobe(struct pt_regs *ctx, struct bpf_map_def *map_fd, _Bool is_send) {
+	long err;
+
+	__u64 id = bpf_get_current_pid_tgid();
+	__u32 pid = id >> 32;
+
+	if (!should_tap(id >> 32)) {
+		return;
+	}
+
+	struct ssl_info *info_ptr = bpf_map_lookup_elem(map_fd, &id);
+	// Happens when the connection is not tls
+	if (info_ptr == NULL) {
+		return;
+	}
+
+	struct sock *sk = (struct sock *) PT_REGS_PARM1(ctx);
+
+	short unsigned int family;
+	err = bpf_probe_read(&family, sizeof(family), (void *)&sk->__sk_common.skc_family);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_FAMILY, id, err, 0l);
+		return;
+	}
+	if (family != AF_INET) {
+		return;
+	}
+
+	// daddr, saddr and dport are in network byte order (big endian)
+	// sport is in host byte order
+	__be32 saddr;
+	__be32 daddr;
+	__be16 dport;
+	__u16 sport;
+
+	err = bpf_probe_read(&saddr, sizeof(saddr), (void *)&sk->__sk_common.skc_rcv_saddr);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_SADDR, id, err, 0l);
+		return;
+	}
+	err = bpf_probe_read(&daddr, sizeof(daddr), (void *)&sk->__sk_common.skc_daddr);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_DADDR, id, err, 0l);
+		return;
+	}
+	err = bpf_probe_read(&dport, sizeof(dport), (void *)&sk->__sk_common.skc_dport);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_DPORT, id, err, 0l);
+		return;
+	}
+	err = bpf_probe_read(&sport, sizeof(sport), (void *)&sk->__sk_common.skc_num);
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SOCKET_SPORT, id, err, 0l);
+		return;
+	}
+
+	info_ptr->address_info.mode = ADDRESS_INFO_MODE_PAIR;
+	info_ptr->address_info.daddr = daddr;
+	info_ptr->address_info.saddr = saddr;
+	info_ptr->address_info.dport = dport;
+	info_ptr->address_info.sport = bpf_htons(sport);
+}
+
+SEC("kprobe/tcp_sendmsg")
+void BPF_KPROBE(tcp_sendmsg) {
+	tcp_kprobe(ctx, &openssl_write_context, true);
+}
+
+SEC("kprobe/tcp_recvmsg")
+void BPF_KPROBE(tcp_recvmsg) {
+	tcp_kprobe(ctx, &openssl_read_context, false);
+}

tap/tlstapper/bpf/tls_tapper.c

@@ -15,6 +15,7 @@ Copyright (C) UP9 Inc.
 //
 #include "common.c"
 #include "openssl_uprobes.c"
+#include "tcp_kprobes.c"
 #include "go_uprobes.c"
 #include "fd_tracepoints.c"
 #include "fd_to_address_tracepoints.c"

tap/tlstapper/bpf_logger_messages.go

@@ -20,4 +20,9 @@ var bpfLogMessages = []string{
 	/*0014*/ "[%d] Unable to put connect info [err: %d]",
 	/*0015*/ "[%d] Unable to get connect info",
 	/*0016*/ "[%d] Unable to read connect info [err: %d]",
+	/*0017*/ "[%d] Unable to read socket family [err: %d]",
+	/*0018*/ "[%d] Unable to read socket daddr [err: %d]",
+	/*0019*/ "[%d] Unable to read socket saddr [err: %d]",
+	/*0019*/ "[%d] Unable to read socket dport [err: %d]",
+	/*0021*/ "[%d] Unable to read socket sport [err: %d]",
 }

tap/tlstapper/chunk.go

@@ -1,38 +1,33 @@
 package tlstapper
 
 import (
-	"bytes"
 	"encoding/binary"
 	"net"
+	"unsafe"
 
-	"github.com/go-errors/errors"
 	"github.com/up9inc/mizu/tap/api"
 )
 
 const FlagsIsClientBit uint32 = 1 << 0
 const FlagsIsReadBit uint32 = 1 << 1
+const (
+	addressInfoModeUndefined = iota
+	addressInfoModeSingle
+	addressInfoModePair
+)
 
-func (c *tlsTapperTlsChunk) getAddress() (net.IP, uint16, error) {
+func (c *tlsTapperTlsChunk) getSrcAddress() (net.IP, uint16) {
-	address := bytes.NewReader(c.Address[:])
+	ip := intToIP(c.AddressInfo.Saddr)
-	var family uint16
+	port := ntohs(c.AddressInfo.Sport)
-	var port uint16
-	var ip32 uint32
 
-	if err := binary.Read(address, binary.BigEndian, &family); err != nil {
+	return ip, port
-		return nil, 0, errors.Wrap(err, 0)
+}
-	}
 
-	if err := binary.Read(address, binary.BigEndian, &port); err != nil {
+func (c *tlsTapperTlsChunk) getDstAddress() (net.IP, uint16) {
-		return nil, 0, errors.Wrap(err, 0)
+	ip := intToIP(c.AddressInfo.Daddr)
-	}
+	port := ntohs(c.AddressInfo.Dport)
 
-	if err := binary.Read(address, binary.BigEndian, &ip32); err != nil {
+	return ip, port
-		return nil, 0, errors.Wrap(err, 0)
-	}
-
-	ip := net.IP{uint8(ip32 >> 24), uint8(ip32 >> 16), uint8(ip32 >> 8), uint8(ip32)}
-
-	return ip, port, nil
 }
 
 func (c *tlsTapperTlsChunk) isClient() bool {
@@ -59,26 +54,54 @@ func (c *tlsTapperTlsChunk) isRequest() bool {
 	return (c.isClient() && c.isWrite()) || (c.isServer() && c.isRead())
 }
 
-func (c *tlsTapperTlsChunk) getAddressPair() (addressPair, error) {
+func (c *tlsTapperTlsChunk) getAddressPair() (addressPair, bool) {
-	ip, port, err := c.getAddress()
+	var (
+		srcIp, dstIp     net.IP
+		srcPort, dstPort uint16
+		full             bool
+	)
 
-	if err != nil {
+	switch c.AddressInfo.Mode {
-		return addressPair{}, err
+	case addressInfoModeSingle:
+		if c.isRequest() {
+			srcIp, srcPort = api.UnknownIp, api.UnknownPort
+			dstIp, dstPort = c.getSrcAddress()
+		} else {
+			srcIp, srcPort = c.getSrcAddress()
+			dstIp, dstPort = api.UnknownIp, api.UnknownPort
+		}
+		full = false
+	case addressInfoModePair:
+		if c.isRequest() {
+			srcIp, srcPort = c.getSrcAddress()
+			dstIp, dstPort = c.getDstAddress()
+		} else {
+			srcIp, srcPort = c.getDstAddress()
+			dstIp, dstPort = c.getSrcAddress()
+		}
+		full = true
+	case addressInfoModeUndefined:
+		srcIp, srcPort = api.UnknownIp, api.UnknownPort
+		dstIp, dstPort = api.UnknownIp, api.UnknownPort
+		full = false
 	}
 
-	if c.isRequest() {
+	return addressPair{
-		return addressPair{
+		srcIp:   srcIp,
-			srcIp:   api.UnknownIp,
+		srcPort: srcPort,
-			srcPort: api.UnknownPort,
+		dstIp:   dstIp,
-			dstIp:   ip,
+		dstPort: dstPort,
-			dstPort: port,
+	}, full
-		}, nil
+}
-	} else {
+
-		return addressPair{
+// intToIP converts IPv4 number to net.IP
-			srcIp:   ip,
+func intToIP(ip32be uint32) net.IP {
-			srcPort: port,
+	return net.IPv4(uint8(ip32be), uint8(ip32be>>8), uint8(ip32be>>16), uint8(ip32be>>24))
-			dstIp:   api.UnknownIp,
+}
-			dstPort: api.UnknownPort,
+
-		}, nil
+// ntohs converts big endian (network byte order) to little endian (assuming that's the host byte order)
-	}
+func ntohs(i16be uint16) uint16 {
+	b := make([]byte, 2)
+	binary.BigEndian.PutUint16(b, i16be)
+	return *(*uint16)(unsafe.Pointer(&b[0]))
 }

tap/tlstapper/ssllib_hooks.go

@@ -14,6 +14,8 @@ type sslHooks struct {
 	sslWriteExRetProbe link.Link
 	sslReadExProbe     link.Link
 	sslReadExRetProbe  link.Link
+	tcpSendmsg         link.Link
+	tcpRecvmsg         link.Link
 }
 
 func (s *sslHooks) installUprobes(bpfObjects *tlsTapperObjects, sslLibraryPath string) error {
@@ -103,6 +105,16 @@ func (s *sslHooks) installSslHooks(bpfObjects *tlsTapperObjects, sslLibrary *lin
 		}
 	}
 
+	s.tcpSendmsg, err = link.Kprobe("tcp_sendmsg", bpfObjects.TcpSendmsg, nil)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	s.tcpRecvmsg, err = link.Kprobe("tcp_recvmsg", bpfObjects.TcpRecvmsg, nil)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
 	return nil
 }
 
@@ -149,5 +161,17 @@ func (s *sslHooks) close() []error {
 		}
 	}
 
+	if s.tcpSendmsg != nil {
+		if err := s.tcpSendmsg.Close(); err != nil {
+			returnValue = append(returnValue, err)
+		}
+	}
+
+	if s.tcpRecvmsg != nil {
+		if err := s.tcpRecvmsg.Close(); err != nil {
+			returnValue = append(returnValue, err)
+		}
+	}
+
 	return returnValue
 }

tap/tlstapper/tls_poller.go

@@ -134,14 +134,9 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsTapperTlsChunk) {
 
 func (p *tlsPoller) handleTlsChunk(chunk *tlsTapperTlsChunk, extension *api.Extension, emitter api.Emitter,
 	options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) error {
-	address, err := p.getSockfdAddressPair(chunk)
+	address, err := p.getAddressPair(chunk)
-
 	if err != nil {
-		address, err = chunk.getAddressPair()
+		return err
-
-		if err != nil {
-			return err
-		}
 	}
 
 	key := buildTlsKey(address)
@@ -161,6 +156,22 @@ func (p *tlsPoller) handleTlsChunk(chunk *tlsTapperTlsChunk, extension *api.Exte
 	return nil
 }
 
+func (p *tlsPoller) getAddressPair(chunk *tlsTapperTlsChunk) (addressPair, error) {
+	addrPairFromChunk, full := chunk.getAddressPair()
+	if full {
+		return addrPairFromChunk, nil
+	}
+
+	addrPairFromSockfd, err := p.getSockfdAddressPair(chunk)
+	if err == nil {
+		return addrPairFromSockfd, nil
+	} else {
+		logger.Log.Error("failed to get address from sock fd:", err)
+	}
+
+	return addrPairFromChunk, err
+}
+
 func (p *tlsPoller) startNewTlsReader(chunk *tlsTapperTlsChunk, address *addressPair, key string,
 	emitter api.Emitter, extension *api.Extension, options *api.TrafficFilteringOptions,
 	streamsMap api.TcpStreamMap) *tlsReader {

tap/tlstapper/tlstapper46_bpfel_arm64.go

@@ -19,15 +19,21 @@ type tlsTapper46GoidOffsets struct {
 }
 
 type tlsTapper46TlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Mode  int32
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper46 returns the embedded CollectionSpec for tlsTapper46.
@@ -93,6 +99,8 @@ type tlsTapper46ProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapper46MapSpecs contains maps before they are loaded into the kernel.
@@ -189,6 +197,8 @@ type tlsTapper46Programs struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapper46Programs) Close() error {
@@ -215,6 +225,8 @@ func (p *tlsTapper46Programs) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

tap/tlstapper/tlstapper46_bpfel_x86.go

@@ -19,15 +19,21 @@ type tlsTapper46GoidOffsets struct {
 }
 
 type tlsTapper46TlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Mode  int32
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper46 returns the embedded CollectionSpec for tlsTapper46.
@@ -93,6 +99,8 @@ type tlsTapper46ProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapper46MapSpecs contains maps before they are loaded into the kernel.
@@ -189,6 +197,8 @@ type tlsTapper46Programs struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapper46Programs) Close() error {
@@ -215,6 +225,8 @@ func (p *tlsTapper46Programs) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

tap/tlstapper/tlstapper_bpfel_arm64.go

@@ -19,15 +19,21 @@ type tlsTapperGoidOffsets struct {
 }
 
 type tlsTapperTlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Mode  int32
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper returns the embedded CollectionSpec for tlsTapper.
@@ -93,6 +99,8 @@ type tlsTapperProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapperMapSpecs contains maps before they are loaded into the kernel.
@@ -189,6 +197,8 @@ type tlsTapperPrograms struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapperPrograms) Close() error {
@@ -215,6 +225,8 @@ func (p *tlsTapperPrograms) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

tap/tlstapper/tlstapper_bpfel_x86.go

@@ -19,15 +19,21 @@ type tlsTapperGoidOffsets struct {
 }
 
 type tlsTapperTlsChunk struct {
-	Pid      uint32
+	Pid         uint32
-	Tgid     uint32
+	Tgid        uint32
-	Len      uint32
+	Len         uint32
-	Start    uint32
+	Start       uint32
-	Recorded uint32
+	Recorded    uint32
-	Fd       uint32
+	Fd          uint32
-	Flags    uint32
+	Flags       uint32
-	Address  [16]uint8
+	AddressInfo struct {
-	Data     [4096]uint8
+		Mode  int32
+		Saddr uint32
+		Daddr uint32
+		Sport uint16
+		Dport uint16
+	}
+	Data [4096]uint8
 }
 
 // loadTlsTapper returns the embedded CollectionSpec for tlsTapper.
@@ -93,6 +99,8 @@ type tlsTapperProgramSpecs struct {
 	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
 }
 
 // tlsTapperMapSpecs contains maps before they are loaded into the kernel.
@@ -189,6 +197,8 @@ type tlsTapperPrograms struct {
 	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
 	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
 	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
+	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
+	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
 }
 
 func (p *tlsTapperPrograms) Close() error {
@@ -215,6 +225,8 @@ func (p *tlsTapperPrograms) Close() error {
 		p.SysEnterWrite,
 		p.SysExitAccept4,
 		p.SysExitConnect,
+		p.TcpRecvmsg,
+		p.TcpSendmsg,
 	)
 }
 

ui-common/package.json

@@ -65,6 +65,7 @@
     "recharts": "^2.1.10",
     "redoc": "^2.0.0-rc.71",
     "styled-components": "^5.3.5",
+    "use-file-picker": "^1.4.2",
     "web-vitals": "^2.1.4",
     "xml-formatter": "^2.6.1"
   },

ui-common/src/components.scss

@@ -17,6 +17,6 @@
       width: 100%;
       width: -moz-available;
       width: -webkit-fill-available;
-      width: strech;
+      width: stretch;
     }
-}
+}

ui-common/src/components/EntriesList/EntriesList.tsx

@@ -1,20 +1,20 @@
-import React, {useCallback, useEffect, useMemo, useState} from "react";
+import React, { useCallback, useEffect, useMemo, useState } from "react";
 import styles from './EntriesList.module.sass';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
-import Moment from 'moment';
+import { EntryItem } from "../EntryListItem/EntryListItem";
-import {EntryItem} from "../EntryListItem/EntryListItem";
 import down from "assets/downImg.svg";
 import spinner from 'assets/spinner.svg';
-import {RecoilState, useRecoilState, useRecoilValue, useSetRecoilState} from "recoil";
+import { RecoilState, useRecoilState, useRecoilValue, useSetRecoilState } from "recoil";
 import entriesAtom from "../../recoil/entries";
 import queryAtom from "../../recoil/query";
 import TrafficViewerApiAtom from "../../recoil/TrafficViewerApi";
 import TrafficViewerApi from "../TrafficViewer/TrafficViewerApi";
 import focusedEntryIdAtom from "../../recoil/focusedEntryId";
-import {toast} from "react-toastify";
+import { toast } from "react-toastify";
-import {MAX_ENTRIES, TOAST_CONTAINER_ID} from "../../configs/Consts";
+import { MAX_ENTRIES, TOAST_CONTAINER_ID } from "../../configs/Consts";
 import tappingStatusAtom from "../../recoil/tappingStatus";
 import leftOffTopAtom from "../../recoil/leftOffTop";
+import Moment from "moment";
 
 interface EntriesListProps {
   listEntryREF: any;

ui-common/src/components/UI/CodeEditor/CodeEditor.tsx

@@ -37,11 +37,6 @@ const CodeEditor: React.FC<CodeEditorProps> = ({
             theme="github"
             onChange={onChange}
             editorProps={{ $blockScrolling: true }}
-            setOptions={{
-                enableBasicAutocompletion: true,
-                enableLiveAutocompletion: true,
-                enableSnippets: true
-            }}
             showPrintMargin={false}
             value={code}
             width="100%"

ui-common/src/components/UI/FilePicker/FilePicker.tsx

@@ -0,0 +1,33 @@
+import React from 'react';
+import { useEffect } from 'react';
+import { useFilePicker } from 'use-file-picker';
+import { FileContent } from 'use-file-picker/dist/interfaces';
+
+interface IFilePickerProps {
+    onLoadingComplete: (file: FileContent) => void;
+    elem: any
+}
+
+const FilePicker = ({ elem, onLoadingComplete }: IFilePickerProps) => {
+    const [openFileSelector, { filesContent }] = useFilePicker({
+        accept: ['.json'],
+        limitFilesConfig: { max: 1 },
+        maxFileSize: 1
+    });
+
+    const onFileSelectorClick = (e) => {
+        e.preventDefault();
+        e.stopPropagation();
+        openFileSelector();
+    }
+
+    useEffect(() => {
+        filesContent.length && onLoadingComplete(filesContent[0])
+    }, [filesContent, onLoadingComplete]);
+
+    return (<React.Fragment>
+        {React.cloneElement(elem, { onClick: onFileSelectorClick })}
+    </React.Fragment>)
+}
+
+export default FilePicker;

ui-common/src/components/UI/KeyValueTable/KeyValueTable.tsx

@@ -75,5 +75,6 @@ const KeyValueTable: React.FC<KeyValueTableProps> = ({ data, onDataChange, keyPl
         })}
     </div>
 }
-
+export const convertParamsToArr = (paramsObj) => Object.entries(paramsObj).map(([key, value]) => { return { key, value } })
+export const convertArrToKeyValueObject = (arr) => arr.reduce((acc, curr) => { acc[curr.key] = curr.value; return acc }, {})
 export default KeyValueTable

ui-common/src/components/modals/ReplayRequestModal/ReplayRequestModal.module.sass

@@ -79,5 +79,12 @@
     overflow: hidden
     
     b::after
-       content: '\b'
+        content: '\b'
         display: inline
+
+.icon
+    width: 24px
+    height: 26px
+    stroke-width: 0px
+    fill: $blue-color
+    stroke: $blue-color

ui-common/src/components/modals/ReplayRequestModal/ReplayRequestModal.tsx

@@ -1,25 +1,30 @@
-import { Accordion, AccordionDetails, AccordionSummary, Backdrop, Box, Button, Fade, Modal } from "@mui/material";
 import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
+import DownloadIcon from '@mui/icons-material/FileDownloadOutlined';
+import UploadIcon from '@mui/icons-material/UploadFile';
+import closeIcon from "assets/close.svg";
+import refreshImg from "assets/refresh.svg";
+import { Accordion, AccordionDetails, AccordionSummary, Backdrop, Box, Button, Fade, Modal } from "@mui/material";
 import React, { Fragment, useCallback, useEffect, useState } from "react";
-import { useCommonStyles } from "../../../helpers/commonStyle";
-import { Tabs } from "../../UI";
-import KeyValueTable from "../../UI/KeyValueTable/KeyValueTable";
-import CodeEditor from "../../UI/CodeEditor/CodeEditor";
-import { useRecoilValue, RecoilState, useRecoilState } from "recoil";
-import TrafficViewerApiAtom from "../../../recoil/TrafficViewerApi/atom";
-import TrafficViewerApi from "../../TrafficViewer/TrafficViewerApi";
 import { toast } from "react-toastify";
+import { RecoilState, useRecoilState, useRecoilValue } from "recoil";
+import { FileContent } from "use-file-picker/dist/interfaces";
 import { TOAST_CONTAINER_ID } from "../../../configs/Consts";
-import styles from './ReplayRequestModal.module.sass'
+import { useCommonStyles } from "../../../helpers/commonStyle";
-import closeIcon from "assets/close.svg"
-import refreshImg from "assets/refresh.svg"
-import { formatRequestWithOutError } from "../../EntryDetailed/EntrySections/EntrySections";
-import entryDataAtom from "../../../recoil/entryData";
-import { AutoRepresentation, TabsEnum } from "../../EntryDetailed/EntryViewer/AutoRepresentation";
-import useDebounce from "../../../hooks/useDebounce"
-import replayRequestModalOpenAtom from "../../../recoil/replayRequestModalOpen";
 import { Utils } from "../../../helpers/Utils";
+import useDebounce from "../../../hooks/useDebounce";
+import entryDataAtom from "../../../recoil/entryData";
+import replayRequestModalOpenAtom from "../../../recoil/replayRequestModalOpen";
+import TrafficViewerApiAtom from "../../../recoil/TrafficViewerApi/atom";
+import { formatRequestWithOutError } from "../../EntryDetailed/EntrySections/EntrySections";
+import { AutoRepresentation, TabsEnum } from "../../EntryDetailed/EntryViewer/AutoRepresentation";
+import TrafficViewerApi from "../../TrafficViewer/TrafficViewerApi";
+import { Tabs } from "../../UI";
+import CodeEditor from "../../UI/CodeEditor/CodeEditor";
+import FilePicker from '../../UI/FilePicker/FilePicker';
+import KeyValueTable, { convertArrToKeyValueObject, convertParamsToArr } from "../../UI/KeyValueTable/KeyValueTable";
 import { LoadingWrapper } from "../../UI/withLoading/withLoading";
+import { IReplayRequestData, KeyValuePair } from './interfaces';
+import styles from './ReplayRequestModal.module.sass';
 
 const modalStyle = {
     position: 'absolute',
@@ -37,11 +42,6 @@ const modalStyle = {
     paddingBottom: "15px"
 };
 
-interface ReplayRequestModalProps {
-    isOpen: boolean;
-    onClose: () => void;
-}
-
 enum RequestTabs {
     Params = "params",
     Headers = "headers",
@@ -51,8 +51,6 @@ enum RequestTabs {
 const HTTP_METHODS = ["get", "post", "put", "head", "options", "delete"]
 const TABS = [{ tab: RequestTabs.Headers }, { tab: RequestTabs.Params }, { tab: RequestTabs.Body }];
 
-const convertParamsToArr = (paramsObj) => Object.entries(paramsObj).map(([key, value]) => { return { key, value } })
-
 const getQueryStringParams = (link: String) => {
 
     if (link) {
@@ -69,43 +67,61 @@ const decodeQueryParam = (p) => {
     return decodeURIComponent(p.replace(/\+/g, ' '));
 }
 
+interface ReplayRequestModalProps {
+    isOpen: boolean;
+    onClose: () => void;
+}
+
 const ReplayRequestModal: React.FC<ReplayRequestModalProps> = ({ isOpen, onClose }) => {
     const entryData = useRecoilValue(entryDataAtom)
     const request = entryData.data.request
-    const [method, setMethod] = useState(request?.method?.toLowerCase() as string)
     const getHostUrl = useCallback(() => {
         return entryData.data.dst.name ? entryData.data?.dst?.name : entryData.data.dst.ip
     }, [entryData.data.dst.ip, entryData.data.dst.name])
-    const [hostPortInput, setHostPortInput] = useState(`${entryData.base.proto.name}://${getHostUrl()}:${entryData.data.dst.port}`)
+    const getHostPortVal = useCallback(() => {
+        return `${entryData.base.proto.name}://${getHostUrl()}:${entryData.data.dst.port}`
+    }, [entryData.base.proto.name, entryData.data.dst.port, getHostUrl])
+    const [hostPortInput, setHostPortInput] = useState(getHostPortVal())
     const [pathInput, setPathInput] = useState(request.path);
     const commonClasses = useCommonStyles();
     const [currentTab, setCurrentTab] = useState(TABS[0].tab);
     const [response, setResponse] = useState(null);
-    const [postData, setPostData] = useState(request?.postData?.text || JSON.stringify(request?.postData?.params));
-    const [params, setParams] = useState(convertParamsToArr(request?.queryString || {}))
-    const [headers, setHeaders] = useState(convertParamsToArr(request?.headers || {}))
     const trafficViewerApi = useRecoilValue(TrafficViewerApiAtom as RecoilState<TrafficViewerApi>)
     const [isLoading, setIsLoading] = useState(false)
     const [requestExpanded, setRequestExpanded] = useState(true)
     const [responseExpanded, setResponseExpanded] = useState(false)
 
+    const getInitialRequestData = useCallback((): IReplayRequestData => {
+        return {
+            method: request?.method?.toLowerCase() as string,
+            hostPort: `${entryData.base.proto.name}://${getHostUrl()}:${entryData.data.dst.port}`,
+            path: request.path,
+            postData: request.postData?.text || JSON.stringify(request.postData?.params),
+            headers: convertParamsToArr(request.headers || {}),
+            params: convertParamsToArr(request.queryString || {})
+        }
+    }, [entryData.base.proto.name, entryData.data.dst.port, getHostUrl, request.headers, request?.method, request.path, request.postData?.params, request.postData?.text, request.queryString])
+
+    const [requestDataModel, setRequestData] = useState<IReplayRequestData>(getInitialRequestData())
+
     const debouncedPath = useDebounce(pathInput, 500);
 
+    const addParamsToUrl = useCallback((url: string, params: KeyValuePair[]) => {
+        const urlParams = new URLSearchParams("");
+        params.forEach(param => urlParams.append(param.key, param.value as string))
+        return `${url}?${urlParams.toString()}`
+    }, [])
+
     const onParamsChange = useCallback((newParams) => {
-        setParams(newParams);
         let newUrl = `${debouncedPath ? debouncedPath.split('?')[0] : ""}`
-        newParams.forEach(({ key, value }, index) => {
+        newUrl = addParamsToUrl(newUrl, newParams)
-            newUrl += index > 0 ? '&' : '?'
-            newUrl += `${key}` + (value ? `=${value}` : "")
-        })
-
         setPathInput(newUrl)
-
+    }, [addParamsToUrl, debouncedPath])
-    }, [debouncedPath])
 
     useEffect(() => {
-        const newParams = getQueryStringParams(debouncedPath);
+        const params = convertParamsToArr(getQueryStringParams(debouncedPath));
-        setParams(convertParamsToArr(newParams))
+        setRequestData({ ...requestDataModel, params })
+        // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [debouncedPath])
 
     const onModalClose = () => {
@@ -114,33 +130,28 @@ const ReplayRequestModal: React.FC<ReplayRequestModalProps> = ({ isOpen, onClose
         onClose()
     }
 
-    const resetModel = useCallback(() => {
+    const resetModal = useCallback((requestDataModel: IReplayRequestData, hostPortInputVal, pathVal) => {
-        setMethod(request?.method?.toLowerCase() as string)
+        setRequestData(requestDataModel)
-        setHostPortInput(`${entryData.base.proto.name}://${getHostUrl()}:${entryData.data.dst.port}`)
+        setHostPortInput(hostPortInputVal)
-        setPathInput(request.path);
+        setPathInput(addParamsToUrl(pathVal, requestDataModel.params));
         setResponse(null);
-        setPostData(request?.postData?.text || JSON.stringify(request?.postData?.params));
+        setRequestExpanded(true);
-        setParams(convertParamsToArr(request?.queryString || {}))
+    }, [addParamsToUrl])
-        setHeaders(convertParamsToArr(request?.headers || {}))
-        setRequestExpanded(true)
-    }, [entryData.base.proto.name, entryData.data.dst.port, getHostUrl, request?.headers, request?.method, request.path, request?.postData?.params, request?.postData?.text, request?.queryString])
 
     const onRefreshRequest = useCallback((event) => {
-        event.stopPropagation()
+        event.stopPropagation();
-        resetModel()
+        const hostPortInputVal = getHostPortVal();
-    }, [resetModel])
+        resetModal(getInitialRequestData(), hostPortInputVal, request.path);
+    }, [getHostPortVal, getInitialRequestData, request.path, resetModal])
 
 
     const sendRequest = useCallback(async () => {
         setResponse(null)
-        const headersData = headers.reduce((prev, corrent) => {
+        const headersData = convertArrToKeyValueObject(requestDataModel.headers)
-            prev[corrent.key] = corrent.value
+
-            return prev
-        }, {})
-        const buildUrl = `${hostPortInput}${pathInput}`
-        const requestData = { url: buildUrl, headers: headersData, data: postData, method }
         try {
             setIsLoading(true)
+            const requestData = { url: `${hostPortInput}${pathInput}`, headers: headersData, data: requestDataModel.postData, method: requestDataModel.method }
             const response = await trafficViewerApi.replayRequest(requestData)
             setResponse(response?.data?.representation)
             if (response.errorMessage) {
@@ -150,7 +161,6 @@ const ReplayRequestModal: React.FC<ReplayRequestModalProps> = ({ isOpen, onClose
                 setRequestExpanded(false)
                 setResponseExpanded(true)
             }
-
         } catch (error) {
             setRequestExpanded(true)
             toast.error("Error occurred while fetching response", { containerId: TOAST_CONTAINER_ID });
@@ -159,27 +169,37 @@ const ReplayRequestModal: React.FC<ReplayRequestModalProps> = ({ isOpen, onClose
         finally {
             setIsLoading(false)
         }
+    }, [hostPortInput, pathInput, requestDataModel.headers, requestDataModel.method, requestDataModel.postData, trafficViewerApi])
 
-    }, [headers, hostPortInput, method, pathInput, postData, trafficViewerApi])
+    const onDownloadRequest = useCallback((e) => {
+        e.stopPropagation()
+        const date = Utils.getNow()
+        Utils.exportToJson(requestDataModel, `${getHostUrl()} - ${date}`)
+    }, [getHostUrl, requestDataModel])
+
+    const onLoadingComplete = useCallback((fileContent: FileContent) => {
+        const requestData = JSON.parse(fileContent.content) as IReplayRequestData
+        resetModal(requestData, requestData.hostPort, requestData.path)
+    }, [resetModal])
 
     let innerComponent
     switch (currentTab) {
         case RequestTabs.Params:
-            innerComponent = <div className={styles.keyValueContainer}><KeyValueTable data={params} onDataChange={onParamsChange} key={"params"} valuePlaceholder="New Param Value" keyPlaceholder="New param Key" /></div>
+            innerComponent = <div className={styles.keyValueContainer}><KeyValueTable data={requestDataModel.params} onDataChange={onParamsChange} key={"params"} valuePlaceholder="New Param Value" keyPlaceholder="New param Key" /></div>
             break;
         case RequestTabs.Headers:
             innerComponent = <Fragment>
-                <div className={styles.keyValueContainer}><KeyValueTable data={headers} onDataChange={(heaedrs) => setHeaders(heaedrs)} key={"Header"} valuePlaceholder="New Headers Value" keyPlaceholder="New Headers Key" />
+                <div className={styles.keyValueContainer}><KeyValueTable data={requestDataModel.headers} onDataChange={(headers) => setRequestData({ ...requestDataModel, headers: headers })} key={"Header"} valuePlaceholder="New Headers Value" keyPlaceholder="New Headers Key" />
                 </div>
-                <span className={styles.note}><b>* </b> X-Mizu Header added to reuqests</span>
+                <span className={styles.note}><b>* </b> X-Mizu Header added to requests</span>
             </Fragment>
             break;
         case RequestTabs.Body:
-            const formatedCode = formatRequestWithOutError(postData || "", request?.postData?.mimeType)
+            const formattedCode = formatRequestWithOutError(requestDataModel.postData || "", request?.postData?.mimeType)
             innerComponent = <div className={styles.codeEditor}>
                 <CodeEditor language={request?.postData?.mimeType.split("/")[1]}
-                    code={Utils.isJson(formatedCode) ? JSON.stringify(JSON.parse(formatedCode || "{}"), null, 2) : formatedCode}
+                    code={Utils.isJson(formattedCode) ? JSON.stringify(JSON.parse(formattedCode || "{}"), null, 2) : formattedCode}
-                    onChange={setPostData} />
+                    onChange={(postData) => setRequestData({ ...requestDataModel, postData })} />
             </div>
             break;
         default:
@@ -204,17 +224,43 @@ const ReplayRequestModal: React.FC<ReplayRequestModalProps> = ({ isOpen, onClose
                     <div className={styles.headerContainer}>
                         <div className={styles.headerSection}>
                             <span className={styles.title}>Replay Request</span>
+                            <Button style={{ marginLeft: "2%", textTransform: 'unset' }}
+                                startIcon={<img src={refreshImg} className="custom" alt="Refresh Request"></img>}
+                                size="medium"
+                                variant="contained"
+                                className={commonClasses.outlinedButton + " " + commonClasses.imagedButton}
+                                onClick={onRefreshRequest}
+                            >
+                                Refresh
+                            </Button>
+                            <Button style={{ marginLeft: "2%", textTransform: 'unset' }}
+                                startIcon={<DownloadIcon className={`custom ${styles.icon}`} />}
+                                size="medium"
+                                variant="contained"
+                                className={commonClasses.outlinedButton + " " + commonClasses.imagedButton}
+                                onClick={onDownloadRequest}
+                            >
+                                Download
+                            </Button>
+                            <FilePicker onLoadingComplete={onLoadingComplete}
+                                elem={<Button style={{ marginLeft: "2%", textTransform: 'unset' }}
+                                    startIcon={<UploadIcon className={`custom ${styles.icon}`} />}
+                                    size="medium"
+                                    variant="contained"
+                                    className={commonClasses.outlinedButton + " " + commonClasses.imagedButton}>
+                                    Upload
+                                </Button>}
+                            />
                         </div>
                     </div>
                     <div className={styles.modalContainer}>
                         <Accordion TransitionProps={{ unmountOnExit: true }} expanded={requestExpanded} onChange={() => setRequestExpanded(!requestExpanded)}>
                             <AccordionSummary expandIcon={<ExpandMoreIcon />} aria-controls="response-content">
                                 <span className={styles.sectionHeader}>REQUEST</span>
-                                <img src={refreshImg} style={{ marginLeft: "10px" }} title="Refresh Reuqest" alt="Refresh Reuqest" onClick={onRefreshRequest} />
                             </AccordionSummary>
                             <AccordionDetails>
                                 <div className={styles.path}>
-                                    <select className={styles.select} value={method} onChange={(e) => setMethod(e.target.value)}>
+                                    <select className={styles.select} value={requestDataModel.method} onChange={(e) => setRequestData({ ...requestDataModel, method: e.target.value })}>
                                         {HTTP_METHODS.map(method => <option value={method} key={method}>{method}</option>)}
                                     </select>
                                     <input placeholder="Host:Port" value={hostPortInput} onChange={(event) => setHostPortInput(event.target.value)} className={`${commonClasses.textField} ${styles.hostPort}`} />
@@ -246,7 +292,7 @@ const ReplayRequestModal: React.FC<ReplayRequestModalProps> = ({ isOpen, onClose
                     </div>
                 </Box>
             </Fade>
-        </Modal>
+        </Modal >
     );
 }
 

ui-common/src/components/modals/ReplayRequestModal/interfaces.ts

@@ -0,0 +1,13 @@
+export interface KeyValuePair {
+    key: string;
+    value: unknown;
+}
+
+export interface IReplayRequestData {
+    method: string;
+    hostPort: string;
+    path: string;
+    postData: string;
+    headers: KeyValuePair[]
+    params: KeyValuePair[]
+}

ui-common/src/components/modals/ServiceMapModal/ServiceMapModal.module.sass

@@ -35,6 +35,7 @@ $modalMargin-from-edge : 35px
     color: $blue-gray
     font-weight: 600
     margin-right: 35px
+    white-space: nowrap
 
 .graphSection
   flex: 85%
@@ -74,6 +75,7 @@ $modalMargin-from-edge : 35px
   width: -moz-available
   width: -webkit-fill-available
   width: fill-available
+  width: strech
   max-width: 200px
   box-shadow: 0px 1px 5px #979797
   margin-left: 10px

ui-common/src/components/modals/TrafficStatsModal/TimelineBarChart/TimelineBarChart.module.sass

@@ -2,3 +2,7 @@
     width: 100%
     display: flex 
     justify-content: center
+
+.axisText
+    font-size: 12px
+    opacity: 0.9

ui-common/src/components/modals/TrafficStatsModal/TimelineBarChart/TimelineBarChart.tsx

@@ -1,5 +1,4 @@
 import styles from "./TimelineBarChart.module.sass";
-import { ALL_PROTOCOLS, StatsMode } from "../TrafficStatsModal"
 import React, { useEffect, useMemo, useState } from "react";
 import {
     BarChart,
@@ -9,6 +8,7 @@ import {
     Tooltip,
 } from "recharts";
 import { Utils } from "../../../../helpers/Utils";
+import { ALL_PROTOCOLS, StatsMode } from "../consts";
 
 interface TimelineBarChartProps {
     timeLineBarChartMode: string;
@@ -21,21 +21,21 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
     const [protocolsNamesAndColors, setProtocolsNamesAndColors] = useState([]);
     const [methodsStats, setMethodsStats] = useState(null);
     const [methodsNamesAndColors, setMethodsNamesAndColors] = useState(null);
-    
+
     useEffect(() => {
         if (!data) return;
         const protocolsBarsData = [];
         const prtcNames = [];
         data.sort((a, b) => a.timestamp < b.timestamp ? -1 : 1).forEach(protocolObj => {
             let newProtocolObj: { [k: string]: any } = {};
-            newProtocolObj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
+            newProtocolObj.timestamp = Utils.formatDate(protocolObj.timestamp);
             protocolObj.protocols.forEach(protocol => {
                 newProtocolObj[`${protocol.name}`] = protocol[StatsMode[timeLineBarChartMode]];
                 prtcNames.push({ name: protocol.name, color: protocol.color });
             })
             protocolsBarsData.push(newProtocolObj);
         })
-        const uniqueObjArray = Utils.creatUniqueObjArrayByProp(prtcNames, "name")
+        const uniqueObjArray = Utils.createUniqueObjArrayByProp(prtcNames, "name")
         setProtocolStats(protocolsBarsData);
         setProtocolsNamesAndColors(uniqueObjArray);
     }, [data, timeLineBarChartMode])
@@ -50,14 +50,14 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
         const protocolsMethods = [];
         data.sort((a, b) => a.timestamp < b.timestamp ? -1 : 1).forEach(protocolObj => {
             let newMethodObj: { [k: string]: any } = {};
-            newMethodObj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
+            newMethodObj.timestamp = Utils.formatDate(protocolObj.timestamp);
             protocolObj.protocols.find(protocol => protocol.name === selectedProtocol)?.methods.forEach(method => {
                 newMethodObj[`${method.name}`] = method[StatsMode[timeLineBarChartMode]]
-                protocolsMethodsNamesAndColors.push({name: method.name, color: method.color});
+                protocolsMethodsNamesAndColors.push({ name: method.name, color: method.color });
             })
             protocolsMethods.push(newMethodObj);
         })
-        const uniqueObjArray = Utils.creatUniqueObjArrayByProp(protocolsMethodsNamesAndColors, "name")
+        const uniqueObjArray = Utils.createUniqueObjArrayByProp(protocolsMethodsNamesAndColors, "name")
         setMethodsNamesAndColors(uniqueObjArray);
         setMethodsStats(protocolsMethods);
     }, [data, timeLineBarChartMode, selectedProtocol])
@@ -68,17 +68,13 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
 
     const renderTick = (tickProps) => {
         const { x, y, payload } = tickProps;
-        const { index, value } = payload;
+        const { offset, value } = payload;
+        const pathX = Math.floor(x - offset) + 0.5;
 
-        if (protocolStats.length > 5) {
+        return <React.Fragment>
-            if (index % 3 === 0) {
+            <text x={pathX} y={y + 10} textAnchor="middle" className={styles.axisText}>{`${value}`}</text>;
-                return <text x={x} y={y + 10} textAnchor="end">{`${value}`}</text>;
+            <path d={`M${pathX},${y - 4}v${-10}`} stroke="red" />;
-            }
+        </React.Fragment>
-            return null;
-        }
-        else {
-            return <text x={x} y={y + 10} textAnchor="end">{`${value}`}</text>;
-        }
     };
 
     return (
@@ -95,8 +91,8 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
                     bottom: 5
                 }}
             >
-                <XAxis dataKey="timestamp" tick={renderTick} tickLine={false} interval="preserveStart" />
+                <XAxis dataKey="timestamp" tickLine={false} tick={renderTick} interval="preserveStart"/>
-                <YAxis tickFormatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value} interval="preserveEnd"/>
+                <YAxis tickFormatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value} interval="preserveEnd" />
                 <Tooltip formatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value + " Requests"} />
                 {bars}
             </BarChart>}

ui-common/src/components/modals/TrafficStatsModal/TrafficPieChart/TrafficPieChart.tsx

@@ -1,7 +1,7 @@
 import React, { useEffect, useMemo, useState } from "react";
 import { Cell, Legend, Pie, PieChart, Tooltip } from "recharts";
 import { Utils } from "../../../../helpers/Utils";
-import { ALL_PROTOCOLS, StatsMode as PieChartMode } from "../TrafficStatsModal"
+import { ALL_PROTOCOLS ,StatsMode as PieChartMode } from "../consts"
 
 const RADIAN = Math.PI / 180;
 const renderCustomizedLabel = ({

ui-common/src/components/modals/TrafficStatsModal/TrafficStatsModal.tsx

@@ -7,6 +7,7 @@ import { TimelineBarChart } from "./TimelineBarChart/TimelineBarChart";
 import refreshIcon from "assets/refresh.svg";
 import { useCommonStyles } from "../../../helpers/commonStyle";
 import { LoadingWrapper } from "../../UI/withLoading/withLoading";
+import { ALL_PROTOCOLS, StatsMode } from "./consts";
 
 const modalStyle = {
   position: 'absolute',
@@ -22,19 +23,12 @@ const modalStyle = {
   color: '#000',
 };
 
-export enum StatsMode {
-  REQUESTS = "entriesCount",
-  VOLUME = "volumeSizeBytes"
-}
-
 interface TrafficStatsModalProps {
   isOpen: boolean;
   onClose: () => void;
   getTrafficStatsDataApi: () => Promise<any>
 }
 
-export const ALL_PROTOCOLS = "ALL";
-
 export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, onClose, getTrafficStatsDataApi }) => {
 
   const modes = Object.keys(StatsMode).filter(x => !(parseInt(x) >= 0));

ui-common/src/components/modals/TrafficStatsModal/consts.ts

@@ -0,0 +1,5 @@
+export const ALL_PROTOCOLS = "ALL";
+export enum StatsMode {
+    REQUESTS = "entriesCount",
+    VOLUME = "volumeSizeBytes"
+  }

ui-common/src/helpers/Utils.ts

@@ -1,5 +1,13 @@
+import Moment from 'moment';
+
 const IP_ADDRESS_REGEX = /([0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3})(:([0-9]{1,5}))?/
 
+type JSONValue =
+  | string
+  | number
+  | boolean
+  | Object
+
 
 export class Utils {
   static isIpAddress = (address: string): boolean => IP_ADDRESS_REGEX.test(address)
@@ -37,7 +45,21 @@ export class Utils {
     return hoursAndMinutes;
   }
 
-  static creatUniqueObjArrayByProp = (objArray, prop) => {
+  static formatDate = (date) => {
+    let d = new Date(date),
+        month = '' + (d.getMonth() + 1),
+        day = '' + d.getDate(),
+        year = d.getFullYear();
+    const hoursAndMinutes = Utils.getHoursAndMinutes(date);
+    if (month.length < 2) 
+        month = '0' + month;
+    if (day.length < 2) 
+        day = '0' + day;
+    const newDate = [year, month, day].join('-');
+    return [hoursAndMinutes, newDate].join(' ');
+}
+
+  static createUniqueObjArrayByProp = (objArray, prop) => {
     const map = new Map(objArray.map((item) => [item[prop], item])).values()
     return Array.from(map);
   }
@@ -51,4 +73,24 @@ export class Utils {
     return true;
   }
 
+  static downloadFile = (data: string, filename: string, fileType: string) => {
+    const blob = new Blob([data], { type: fileType })
+    const a = document.createElement('a');
+    a.href = window.URL.createObjectURL(blob);
+    a.download = filename;
+    a.click();
+    a.remove();
+  }
+
+  static exportToJson = (data: JSONValue, name) => {
+    Utils.downloadFile(JSON.stringify(data), `${name}.json`, 'text/json')
+  }
+
+  static getTimeFormatted = (time: Moment.MomentInput) => {
+    return Moment(time).utc().format('MM/DD/YYYY, h:mm:ss.SSS A')
+  }
+
+  static getNow = (format: string = 'MM/DD/YYYY, HH:mm:ss.SSS') => {
+    return Moment().format(format)
+  }
 }

ui-common/src/index.tsx

@@ -7,6 +7,5 @@ import { ServiceMapModal } from './components/modals/ServiceMapModal/ServiceMapM
 import { TrafficStatsModal } from './components/modals/TrafficStatsModal/TrafficStatsModal';
 
 export { CodeEditorWrap as QueryForm } from './components/Filters/Filters';
-export { UI, StatusBar, OasModal, ServiceMapModal, TrafficStatsModal }
+export { UI, StatusBar, OasModal, ServiceMapModal, TrafficStatsModal, TrafficViewer }
 export { useWS, DEFAULT_LEFTOFF }
-export default TrafficViewer;

ui/src/components/Pages/TrafficPage/TrafficPage.tsx

@@ -5,7 +5,7 @@ import debounce from 'lodash/debounce';
 import { useRecoilState } from "recoil";
 import { useCommonStyles } from "../../../helpers/commonStyle"
 import serviceMapModalOpenAtom from "../../../recoil/serviceMapModalOpen";
-import TrafficViewer from "@up9/mizu-common"
+import { TrafficViewer } from "@up9/mizu-common"
 import "@up9/mizu-common/dist/index.css"
 import oasModalOpenAtom from "../../../recoil/oasModalOpen/atom";
 import serviceMap from "../../assets/serviceMap.svg";