Develop -> main (version 35.0)

Merge pull request #1182 from up9inc/develop #major

.github/workflows/acceptance_tests.yml

@@ -52,3 +52,15 @@ jobs:
 
       - name: Test
         run: make acceptance-test
+
+      - name: Slack notification on failure
+        uses: ravsamhq/notify-slack-action@v1
+        if: always()
+        with:
+          status: ${{ job.status }}
+          notification_title: 'Mizu {workflow} has {status_message}'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
+          footer: 'Linked Repo <{repo_url}|{repo}>'
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/release.yml

@@ -85,6 +85,94 @@ jobs:
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
 
+  gcp-registry:
+    name: Push Docker image to GCR
+    runs-on: ubuntu-latest
+
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        target:
+          - amd64
+          - arm64v8
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "stable"
+          if_false: "dev"
+
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
+        id: versioning
+        with:
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ${{ steps.base_image_step.outputs.image }}
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
+            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=-${{ matrix.target }},onlatest=true
+
+      - name: Login to GCR
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            TARGETARCH=${{ matrix.target }}
+            VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+
   docker-manifest:
     name: Create and Push a Docker Manifest
     runs-on: ubuntu-latest
@@ -144,7 +232,7 @@ jobs:
   cli:
     name: Build the CLI and publish
     runs-on: ubuntu-latest
-    needs: [docker-manifest]
+    needs: [docker-manifest, gcp-registry]
     steps:
       - name: Set up Go 1.17
         uses: actions/setup-go@v2
@@ -202,3 +290,15 @@ jobs:
           tag: ${{ steps.versioning.outputs.version }}
           prerelease: ${{ github.ref != 'refs/heads/main' }}
           bodyFile: 'cli/bin/README.md'
+      
+      - name: Slack notification on failure
+        uses: ravsamhq/notify-slack-action@v1
+        if: always()
+        with:
+          status: ${{ job.status }}
+          notification_title: 'Mizu enterprise {workflow} has {status_message}'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
+          footer: 'Linked Repo <{repo_url}|{repo}>'
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/static_code_analysis.yml

@@ -32,10 +32,6 @@ jobs:
         id: agent_modified_files
         run: devops/check_modified_files.sh agent/
 
-      - name: Generate eBPF object files and go bindings
-        id: generate_ebpf
-        run: make bpf
-
       - name: Go lint - agent
         uses: golangci/golangci-lint-action@v2
         if: steps.agent_modified_files.outputs.matched == 'true'

.github/workflows/test.yml

@@ -40,10 +40,6 @@ jobs:
         run: |
           ./devops/install-capstone.sh
 
-      - name: Generate eBPF object files and go bindings
-        id: generate_ebpf
-        run: make bpf
-
       - name: Check CLI modified files
         id: cli_modified_files
         run: devops/check_modified_files.sh cli/

.gitignore

@@ -56,6 +56,3 @@ tap/extensions/*/expect
 
 # Ignore *.log files
 *.log
-
-# Object files
-*.o

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent agent-debug cli tap docker bpf clean-bpf
+.PHONY: help ui agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -20,13 +20,6 @@ TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
 export VER?=0.0
-ARCH=$(shell uname -m)
-ifeq ($(ARCH),$(filter $(ARCH),aarch64 arm64))
-	BPF_O_ARCH_LABEL=arm64
-else
-	BPF_O_ARCH_LABEL=x86
-endif
-BPF_O_FILES = tap/tlstapper/tlstapper46_bpfel_$(BPF_O_ARCH_LABEL).o tap/tlstapper/tlstapper_bpfel_$(BPF_O_ARCH_LABEL).o
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -38,17 +31,11 @@ cli: ## Build CLI.
 cli-debug: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build-debug
 
-agent: bpf ## Build agent.
+agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
 	@ls -l agent/build
 
-bpf: $(BPF_O_FILES)
-
-$(BPF_O_FILES): $(wildcard tap/tlstapper/bpf/**/*.[ch])
-	@(echo "building tlstapper bpf")
-	@(./tap/tlstapper/bpf-builder/build.sh)
-
 agent-debug: ## Build agent for debug.
 	@(echo "building mizu agent for debug.." )
 	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
@@ -89,9 +76,6 @@ clean-cli:  ## Clean CLI.
 clean-docker:  ## Run clean docker
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-clean-bpf:
-	@(rm $(BPF_O_FILES) ; echo "bpf cleanup done" )
-
 test-lint:  ## Run lint on all modules
 	cd agent && golangci-lint run
 	cd shared && golangci-lint run

acceptanceTests/cypress.config.js

@@ -11,6 +11,7 @@ module.exports = defineConfig({
     testUrl: 'http://localhost:8899/',
     redactHeaderContent: 'User-Header[REDACTED]',
     redactBodyContent: '{ "User": "[REDACTED]" }',
+    regexMaskingBodyContent: '[REDACTED]',
     greenFilterColor: 'rgb(210, 250, 210)',
     redFilterColor: 'rgb(250, 214, 220)',
     bodyJsonClass: '.hljs',

acceptanceTests/cypress/e2e/tests/RegexMasking.js

@@ -0,0 +1,7 @@
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-logr/logr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -28,6 +29,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/up9inc/mizu/logger v0.0.0 // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect

acceptanceTests/go.sum

@@ -206,6 +206,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

acceptanceTests/setup.sh

@@ -27,7 +27,7 @@ else
 fi
 
 echo "Starting minikube..."
-minikube start --cpus 2 --memory 6000
+minikube start --cpus 2 --memory 6946
 
 echo "Creating mizu tests namespaces"
 kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -

acceptanceTests/tap_test.go

@@ -2,8 +2,10 @@ package acceptanceTests
 
 import (
 	"archive/zip"
+	"bytes"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os/exec"
 	"path"
 	"strings"
@@ -341,7 +343,7 @@ func TestTapRedact(t *testing.T) {
 
 	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-	tapCmdArgs = append(tapCmdArgs, "--redact", "--set", "tap.redact-patterns.request-headers=User-Header", "--set", "tap.redact-patterns.request-body=User")
+	tapCmdArgs = append(tapCmdArgs, "--redact")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
@@ -427,6 +429,60 @@ func TestTapNoRedact(t *testing.T) {
 	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/NoRedact.js\"")
 }
 
+func TestTapRegexMasking(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--redact")
+
+	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	for i := 0; i < DefaultEntriesCount; i++ {
+		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
+		if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"")
+
+}
+
 func TestTapIgnoredUserAgents(t *testing.T) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")

acceptanceTests/testsUtils.go

@@ -215,11 +215,12 @@ func DeleteKubeFile(kubeContext string, namespace string, filename string) error
 func getDefaultCommandArgs() []string {
 	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
 	setFlag := "--set"
+	telemetry := "telemetry=false"
 	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
-	return []string{setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
+	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
 func GetDefaultTapCommandArgs() []string {

agent/go.mod

@@ -48,9 +48,10 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/beevik/etree v1.1.0 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
-	github.com/cilium/ebpf v0.9.0 // indirect
+	github.com/cilium/ebpf v0.8.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
@@ -88,7 +89,6 @@ require (
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
-	github.com/moby/moby v20.10.17+incompatible // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -104,7 +104,6 @@ require (
 	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
 	github.com/segmentio/kafka-go v0.4.27 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/cobra v1.3.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/struCoder/pidusage v0.2.1 // indirect

agent/go.sum

@@ -101,6 +101,7 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -127,8 +128,8 @@ github.com/chanced/openapi v0.0.8/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5Xyq
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cilium/ebpf v0.9.0 h1:ldiV+FscPCQ/p3mNEV4O02EPbUZJFsoEtHvIr9xLTvk=
-github.com/cilium/ebpf v0.9.0/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY=
+github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao=
+github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -516,8 +517,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/moby/moby v20.10.17+incompatible h1:TJJfyk2fLEgK+RzqVpFNkDkm0oEi+MLUfwt9lEYnp5g=
-github.com/moby/moby v20.10.17+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
@@ -630,8 +629,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -1252,9 +1249,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
-gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

agent/pkg/api/entry_streamer_socket_connector.go

@@ -22,7 +22,7 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
 	if params.EnableFullEntries {
 		message, _ = models.CreateFullEntryWebSocketMessage(entry)
 	} else {
-		protocol, ok := protocolsMap[entry.Protocol.ToString()]
+		protocol, ok := protocolsMap[entry.ProtocolId]
 		if !ok {
 			return fmt.Errorf("protocol not found, protocol: %v", protocol)
 		}

agent/pkg/api/main.go

@@ -126,7 +126,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
 
 		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
-		oasGenerator.HandleEntry(mizuEntry)
+		oasGenerator.HandleEntry(mizuEntry, &item.Protocol)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -97,9 +97,7 @@ func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool
 	websocketIdsLock.Unlock()
 
 	defer func() {
-		if socketConnection := connectedWebsockets[socketId]; socketConnection != nil {
-			socketCleanup(socketId, socketConnection)
-		}
+		socketCleanup(socketId, connectedWebsockets[socketId])
 	}()
 
 	eventHandlers.WebSocketConnect(c, socketId, isTapper)

agent/pkg/controllers/service_map_controller_test.go

@@ -36,13 +36,11 @@ var (
 )
 
 var ProtocolHttp = &tapApi.Protocol{
-	ProtocolSummary: tapApi.ProtocolSummary{
-		Name:         "http",
-		Version:      "1.1",
-		Abbreviation: "HTTP",
-	},
+	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+	Abbreviation:    "HTTP",
 	Macro:           "http",
+	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,

agent/pkg/controllers/status_controller.go

@@ -1,10 +1,7 @@
 package controllers
 
 import (
-	"fmt"
 	"net/http"
-	"strconv"
-	"time"
 
 	core "k8s.io/api/core/v1"
 
@@ -83,24 +80,7 @@ func GetGeneralStats(c *gin.Context) {
 }
 
 func GetTrafficStats(c *gin.Context) {
-	startTime, endTime, err := getStartEndTime(c)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
-		return
-	}
-	c.JSON(http.StatusOK, providers.GetTrafficStats(startTime, endTime))
-}
-
-func getStartEndTime(c *gin.Context) (time.Time, time.Time, error) {
-	startTimeValue, err := strconv.Atoi(c.Query("startTimeMs"))
-	if err != nil {
-		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid start time: %v", err)
-	}
-	endTimeValue, err := strconv.Atoi(c.Query("endTimeMs"))
-	if err != nil {
-		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid end time: %v", err)
-	}
-	return time.UnixMilli(int64(startTimeValue)), time.UnixMilli(int64(endTimeValue)), nil
+	c.JSON(http.StatusOK, providers.GetTrafficStats())
 }
 
 func GetCurrentResolvingInformation(c *gin.Context) {

agent/pkg/entries/entries_provider.go

@@ -38,7 +38,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 			return nil, nil, err
 		}
 
-		protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
+		protocol, ok := app.ProtocolsMap[entry.ProtocolId]
 		if !ok {
 			return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
 		}
@@ -77,7 +77,7 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 		return nil, errors.New(string(bytes))
 	}
 
-	protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
+	protocol, ok := app.ProtocolsMap[entry.ProtocolId]
 	if !ok {
 		return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
 	}

agent/pkg/har/utils.go

@@ -11,30 +11,75 @@ import (
 	"github.com/up9inc/mizu/logger"
 )
 
-func BuildHeaders(rawHeaders map[string]interface{}) ([]Header, string, string, string, string, string) {
+// Keep it because we might want cookies in the future
+//func BuildCookies(rawCookies []interface{}) []har.Cookie {
+//	cookies := make([]har.Cookie, 0, len(rawCookies))
+//
+//	for _, cookie := range rawCookies {
+//		c := cookie.(map[string]interface{})
+//		expiresStr := ""
+//		if c["expires"] != nil {
+//			expiresStr = c["expires"].(string)
+//		}
+//		expires, _ := time.Parse(time.RFC3339, expiresStr)
+//		httpOnly := false
+//		if c["httponly"] != nil {
+//			httpOnly, _ = strconv.ParseBool(c["httponly"].(string))
+//		}
+//		secure := false
+//		if c["secure"] != nil {
+//			secure, _ = strconv.ParseBool(c["secure"].(string))
+//		}
+//		path := ""
+//		if c["path"] != nil {
+//			path = c["path"].(string)
+//		}
+//		domain := ""
+//		if c["domain"] != nil {
+//			domain = c["domain"].(string)
+//		}
+//
+//		cookies = append(cookies, har.Cookie{
+//			Name:        c["name"].(string),
+//			Value:       c["value"].(string),
+//			Path:        path,
+//			Domain:      domain,
+//			HTTPOnly:    httpOnly,
+//			Secure:      secure,
+//			Expires:     expires,
+//			Expires8601: expiresStr,
+//		})
+//	}
+//
+//	return cookies
+//}
+
+func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) {
 	var host, scheme, authority, path, status string
 	headers := make([]Header, 0, len(rawHeaders))
 
-	for key, value := range rawHeaders {
+	for _, header := range rawHeaders {
+		h := header.(map[string]interface{})
+
 		headers = append(headers, Header{
-			Name:  key,
-			Value: value.(string),
+			Name:  h["name"].(string),
+			Value: h["value"].(string),
 		})
 
-		if key == "Host" {
-			host = value.(string)
+		if h["name"] == "Host" {
+			host = h["value"].(string)
 		}
-		if key == ":authority" {
-			authority = value.(string)
+		if h["name"] == ":authority" {
+			authority = h["value"].(string)
 		}
-		if key == ":scheme" {
-			scheme = value.(string)
+		if h["name"] == ":scheme" {
+			scheme = h["value"].(string)
 		}
-		if key == ":path" {
-			path = value.(string)
+		if h["name"] == ":path" {
+			path = h["value"].(string)
 		}
-		if key == ":status" {
-			status = value.(string)
+		if h["name"] == ":status" {
+			status = h["value"].(string)
 		}
 	}
 
@@ -74,8 +119,8 @@ func BuildPostParams(rawParams []interface{}) []Param {
 }
 
 func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
-	headers, host, scheme, authority, path, _ := BuildHeaders(request["headers"].(map[string]interface{}))
-	cookies := make([]Cookie, 0)
+	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
@@ -89,20 +134,12 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 	}
 
 	queryString := make([]QueryString, 0)
-	for key, value := range request["queryString"].(map[string]interface{}) {
-		if valuesInterface, ok := value.([]interface{}); ok {
-			for _, valueInterface := range valuesInterface {
-				queryString = append(queryString, QueryString{
-					Name:  key,
-					Value: valueInterface.(string),
-				})
-			}
-		} else {
-			queryString = append(queryString, QueryString{
-				Name:  key,
-				Value: value.(string),
-			})
-		}
+	for _, _qs := range request["_queryString"].([]interface{}) {
+		qs := _qs.(map[string]interface{})
+		queryString = append(queryString, QueryString{
+			Name:  qs["name"].(string),
+			Value: qs["value"].(string),
+		})
 	}
 
 	url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
@@ -135,8 +172,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 }
 
 func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
-	headers, _, _, _, _, _status := BuildHeaders(response["headers"].(map[string]interface{}))
-	cookies := make([]Cookie, 0)
+	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]

agent/pkg/oas/oas_generator.go

@@ -16,7 +16,7 @@ var (
 )
 
 type OasGeneratorSink interface {
-	HandleEntry(mizuEntry *api.Entry)
+	HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol)
 }
 
 type OasGenerator interface {
@@ -58,12 +58,12 @@ func (g *defaultOasGenerator) IsStarted() bool {
 	return g.started
 }
 
-func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
+func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol) {
 	if !g.started {
 		return
 	}
 
-	if mizuEntry.Protocol.Name == "http" {
+	if protocol.Name == "http" {
 		dest := mizuEntry.Destination.Name
 		if dest == "" {
 			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
@@ -85,7 +85,7 @@ func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
 
 		g.handleHARWithSource(entryWSource)
 	} else {
-		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, mizuEntry.Protocol.Name)
+		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, protocol.Name)
 	}
 }
 

agent/pkg/providers/stats_provider.go

@@ -1,10 +1,8 @@
 package providers
 
 import (
-	"crypto/md5"
-	"encoding/hex"
-	"fmt"
 	"reflect"
+	"strings"
 	"sync"
 	"time"
 
@@ -38,13 +36,13 @@ type SizeAndEntriesCount struct {
 
 type AccumulativeStatsCounter struct {
 	Name            string `json:"name"`
-	Color           string `json:"color"`
 	EntriesCount    int    `json:"entriesCount"`
 	VolumeSizeBytes int    `json:"volumeSizeBytes"`
 }
 
 type AccumulativeStatsProtocol struct {
 	AccumulativeStatsCounter
+	Color   string                      `json:"color"`
 	Methods []*AccumulativeStatsCounter `json:"methods"`
 }
 
@@ -54,7 +52,6 @@ type AccumulativeStatsProtocolTime struct {
 }
 
 type TrafficStatsResponse struct {
-	Protocols     []string                         `json:"protocols"`
 	PieStats      []*AccumulativeStatsProtocol     `json:"pie"`
 	TimelineStats []*AccumulativeStatsProtocolTime `json:"timeline"`
 }
@@ -81,35 +78,20 @@ func GetGeneralStats() *GeneralStats {
 
 func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
 	for item, value := range protocolMap {
-		protocolToColor[api.GetProtocolSummary(item).Abbreviation] = value.BackgroundColor
+		protocolToColor[strings.Split(item, "/")[2]] = value.BackgroundColor
 	}
 }
 
-func GetTrafficStats(startTime time.Time, endTime time.Time) *TrafficStatsResponse {
-	bucketsStatsCopy := getFilteredBucketStatsCopy(startTime, endTime)
+func GetTrafficStats() *TrafficStatsResponse {
+	bucketsStatsCopy := getBucketStatsCopy()
+	interval := calculateInterval(bucketsStatsCopy[0].BucketTime.Unix(), bucketsStatsCopy[len(bucketsStatsCopy)-1].BucketTime.Unix()) // in seconds
 
 	return &TrafficStatsResponse{
-		Protocols:     getAvailableProtocols(bucketsStatsCopy),
 		PieStats:      getAccumulativeStats(bucketsStatsCopy),
-		TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy),
+		TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy, interval),
 	}
 }
 
-func EntryAdded(size int, summery *api.BaseEntry) {
-	generalStats.EntriesCount++
-	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
-
-	currentTimestamp := int(time.Now().Unix())
-
-	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
-		generalStats.FirstEntryTimestamp = currentTimestamp
-	}
-
-	addToBucketStats(size, summery)
-
-	generalStats.LastEntryTimestamp = currentTimestamp
-}
-
 func calculateInterval(firstTimestamp int64, lastTimestamp int64) time.Duration {
 	validDurations := []time.Duration{
 		time.Minute,
@@ -158,17 +140,31 @@ func getAccumulativeStats(stats BucketStats) []*AccumulativeStatsProtocol {
 	return convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated)
 }
 
-func getAccumulativeStatsTiming(stats BucketStats) []*AccumulativeStatsProtocolTime {
+func getAccumulativeStatsTiming(stats BucketStats, interval time.Duration) []*AccumulativeStatsProtocolTime {
 	if len(stats) == 0 {
 		return make([]*AccumulativeStatsProtocolTime, 0)
 	}
 
-	interval := calculateInterval(stats[0].BucketTime.Unix(), stats[len(stats)-1].BucketTime.Unix()) // in seconds
-	methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(stats, interval)
+	methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(interval, stats)
 
 	return convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated)
 }
 
+func EntryAdded(size int, summery *api.BaseEntry) {
+	generalStats.EntriesCount++
+	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
+
+	currentTimestamp := int(time.Now().Unix())
+
+	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
+		generalStats.FirstEntryTimestamp = currentTimestamp
+	}
+
+	addToBucketStats(size, summery)
+
+	generalStats.LastEntryTimestamp = currentTimestamp
+}
+
 func addToBucketStats(size int, summery *api.BaseEntry) {
 	entryTimeBucketRounded := getBucketFromTimeStamp(summery.Timestamp)
 
@@ -211,11 +207,11 @@ func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggreg
 	finalResult := make([]*AccumulativeStatsProtocolTime, 0)
 	for timeKey, item := range methodsPerProtocolPerTimeAggregated {
 		protocolsData := make([]*AccumulativeStatsProtocol, 0)
-		for protocolName, value := range item {
+		for protocolName := range item {
 			entriesCount := 0
 			volumeSizeBytes := 0
 			methods := make([]*AccumulativeStatsCounter, 0)
-			for _, methodAccData := range value {
+			for _, methodAccData := range methodsPerProtocolPerTimeAggregated[timeKey][protocolName] {
 				entriesCount += methodAccData.EntriesCount
 				volumeSizeBytes += methodAccData.VolumeSizeBytes
 				methods = append(methods, methodAccData)
@@ -223,10 +219,10 @@ func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggreg
 			protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
 				AccumulativeStatsCounter: AccumulativeStatsCounter{
 					Name:            protocolName,
-					Color:           protocolToColor[protocolName],
 					EntriesCount:    entriesCount,
 					VolumeSizeBytes: volumeSizeBytes,
 				},
+				Color:   protocolToColor[protocolName],
 				Methods: methods,
 			})
 		}
@@ -252,17 +248,17 @@ func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string
 		protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
 			AccumulativeStatsCounter: AccumulativeStatsCounter{
 				Name:            protocolName,
-				Color:           protocolToColor[protocolName],
 				EntriesCount:    entriesCount,
 				VolumeSizeBytes: volumeSizeBytes,
 			},
+			Color:   protocolToColor[protocolName],
 			Methods: methods,
 		})
 	}
 	return protocolsData
 }
 
-func getFilteredBucketStatsCopy(startTime time.Time, endTime time.Time) BucketStats {
+func getBucketStatsCopy() BucketStats {
 	bucketStatsCopy := BucketStats{}
 	bucketStatsLocker.Lock()
 	if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
@@ -270,21 +266,10 @@ func getFilteredBucketStatsCopy(startTime time.Time, endTime time.Time) BucketSt
 		return nil
 	}
 	bucketStatsLocker.Unlock()
-
-	filteredBucketStatsCopy := BucketStats{}
-	interval := InternalBucketThreshold
-
-	for _, bucket := range bucketStatsCopy {
-		if (bucket.BucketTime.After(startTime.Add(-1*interval/2).Round(interval)) && bucket.BucketTime.Before(endTime.Add(-1*interval/2).Round(interval))) ||
-			bucket.BucketTime.Equal(startTime.Add(-1*interval/2).Round(interval)) ||
-			bucket.BucketTime.Equal(endTime.Add(-1*interval/2).Round(interval)) {
-			filteredBucketStatsCopy = append(filteredBucketStatsCopy, bucket)
-		}
-	}
-	return filteredBucketStatsCopy
+	return bucketStatsCopy
 }
 
-func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
+func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
 	methodsPerProtocolPerTimeAggregated := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{}
 
 	bucketStatsIndex := len(stats) - 1
@@ -304,7 +289,6 @@ func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[ti
 				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName]; !ok {
 					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName] = &AccumulativeStatsCounter{
 						Name:            methodName,
-						Color:           getColorForMethod(protocolName, methodName),
 						EntriesCount:    0,
 						VolumeSizeBytes: 0,
 					}
@@ -319,9 +303,9 @@ func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[ti
 	return methodsPerProtocolPerTimeAggregated
 }
 
-func getAggregatedStats(stats BucketStats) map[string]map[string]*AccumulativeStatsCounter {
+func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*AccumulativeStatsCounter {
 	methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
-	for _, countersOfTimeFrame := range stats {
+	for _, countersOfTimeFrame := range bucketStatsCopy {
 		for protocolName, value := range countersOfTimeFrame.ProtocolStats {
 			for method, countersValue := range value.MethodsStats {
 				if _, found := methodsPerProtocolAggregated[protocolName]; !found {
@@ -330,7 +314,6 @@ func getAggregatedStats(stats BucketStats) map[string]map[string]*AccumulativeSt
 				if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
 					methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
 						Name:            method,
-						Color:           getColorForMethod(protocolName, method),
 						EntriesCount:    0,
 						VolumeSizeBytes: 0,
 					}
@@ -342,25 +325,3 @@ func getAggregatedStats(stats BucketStats) map[string]map[string]*AccumulativeSt
 	}
 	return methodsPerProtocolAggregated
 }
-
-func getColorForMethod(protocolName string, methodName string) string {
-	hash := md5.Sum([]byte(fmt.Sprintf("%v_%v", protocolName, methodName)))
-	input := hex.EncodeToString(hash[:])
-	return fmt.Sprintf("#%v", input[:6])
-}
-
-func getAvailableProtocols(stats BucketStats) []string {
-	protocols := map[string]bool{}
-	for _, countersOfTimeFrame := range stats {
-		for protocolName := range countersOfTimeFrame.ProtocolStats {
-			protocols[protocolName] = true
-		}
-	}
-
-	result := make([]string, 0)
-	for protocol := range protocols {
-		result = append(result, protocol)
-	}
-	result = append(result, "ALL")
-	return result
-}

agent/pkg/providers/stats_provider_internal_test.go

@@ -2,6 +2,7 @@ package providers
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
 )
@@ -109,8 +110,8 @@ func TestGetAggregatedStatsAllTime(t *testing.T) {
 	}
 	actual := getAggregatedStats(bucketStatsForTest)
 
-	if len(actual) != len(expected) {
-		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
 	}
 }
 
@@ -194,10 +195,10 @@ func TestGetAggregatedStatsFromSpecificTime(t *testing.T) {
 			},
 		},
 	}
-	actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute*5)
+	actual := getAggregatedResultTiming(time.Minute*5, bucketStatsForTest)
 
-	if len(actual) != len(expected) {
-		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
 	}
 }
 
@@ -290,9 +291,9 @@ func TestGetAggregatedStatsFromSpecificTimeMultipleBuckets(t *testing.T) {
 			},
 		},
 	}
-	actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute)
+	actual := getAggregatedResultTiming(time.Minute, bucketStatsForTest)
 
-	if len(actual) != len(expected) {
-		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
 	}
 }

agent/pkg/providers/stats_provider_test.go

@@ -26,7 +26,7 @@ func TestEntryAddedCount(t *testing.T) {
 
 	entryBucketKey := time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC)
 	valueLessThanBucketThreshold := time.Second * 130
-	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{Name: "mock"}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
 			for i := 0; i < entriesCount; i++ {
@@ -61,7 +61,7 @@ func TestEntryAddedVolume(t *testing.T) {
 	var expectedEntriesCount int
 	var expectedVolumeInGB float64
 
-	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{Name: "mock"}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
 
 	for _, data := range tests {
 		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {

agent/pkg/servicemap/servicemap_test.go

@@ -50,13 +50,11 @@ var (
 		IP:   fmt.Sprintf("%s.%s", Ip, UnresolvedNodeName),
 	}
 	ProtocolHttp = &tapApi.Protocol{
-		ProtocolSummary: tapApi.ProtocolSummary{
-			Name:         "http",
-			Version:      "1.1",
-			Abbreviation: "HTTP",
-		},
+		Name:            "http",
 		LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+		Abbreviation:    "HTTP",
 		Macro:           "http",
+		Version:         "1.1",
 		BackgroundColor: "#205cf5",
 		ForegroundColor: "#ffffff",
 		FontSize:        12,
@@ -65,13 +63,11 @@ var (
 		Priority:        0,
 	}
 	ProtocolRedis = &tapApi.Protocol{
-		ProtocolSummary: tapApi.ProtocolSummary{
-			Name:         "redis",
-			Version:      "3.x",
-			Abbreviation: "REDIS",
-		},
+		Name:            "redis",
 		LongName:        "Redis Serialization Protocol",
+		Abbreviation:    "REDIS",
 		Macro:           "redis",
+		Version:         "3.x",
 		BackgroundColor: "#a41e11",
 		ForegroundColor: "#ffffff",
 		FontSize:        11,

cli/apiserver/provider.go

@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/up9inc/mizu/cli/utils"
@@ -91,3 +93,45 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 		}
 	}
 }
+
+func (provider *Provider) GetGeneralStats() (map[string]interface{}, error) {
+	generalStatsUrl := fmt.Sprintf("%s/status/general", provider.url)
+
+	response, requestErr := utils.Get(generalStatsUrl, provider.client)
+	if requestErr != nil {
+		return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr)
+	}
+
+	defer response.Body.Close()
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		return nil, fmt.Errorf("failed to read general stats for telemetry, err: %v", readErr)
+	}
+
+	var generalStats map[string]interface{}
+	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
+		return nil, fmt.Errorf("failed to parse general stats for telemetry, err: %v", parseErr)
+	}
+	return generalStats, nil
+}
+
+func (provider *Provider) GetVersion() (string, error) {
+	versionUrl, _ := url.Parse(fmt.Sprintf("%s/metadata/version", provider.url))
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    versionUrl,
+	}
+	statusResp, err := utils.Do(req, provider.client)
+	if err != nil {
+		return "", err
+	}
+	defer statusResp.Body.Close()
+
+	versionResponse := &shared.VersionResponse{}
+	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
+		return "", err
+	}
+
+	return versionResponse.Ver, nil
+}

cli/cmd/check.go

@@ -4,6 +4,7 @@ import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -11,6 +12,7 @@ var checkCmd = &cobra.Command{
 	Use:   "check",
 	Short: "Check the Mizu installation for potential problems",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("check", nil)
 		runMizuCheck()
 		return nil
 	},

cli/cmd/clean.go

@@ -2,12 +2,14 @@ package cmd
 
 import (
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/telemetry"
 )
 
 var cleanCmd = &cobra.Command{
 	Use:   "clean",
 	Short: "Removes all mizu resources",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("clean", nil)
 		performCleanCommand()
 		return nil
 	},

cli/cmd/config.go

@@ -7,6 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/logger"
 )
@@ -15,6 +16,8 @@ var configCmd = &cobra.Command{
 	Use:   "config",
 	Short: "Generate config with default values",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("config", config.Config.Config)
+
 		configWithDefaults, err := config.GetConfigWithDefaults()
 		if err != nil {
 			logger.Log.Errorf("Failed generating config with defaults, err: %v", err)

cli/cmd/install.go

@@ -4,6 +4,7 @@ import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -11,6 +12,7 @@ var installCmd = &cobra.Command{
 	Use:   "install",
 	Short: "Installs mizu components",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("install", nil)
 		runMizuInstall()
 		return nil
 	},

cli/cmd/logs.go

@@ -9,6 +9,7 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -16,6 +17,8 @@ var logsCmd = &cobra.Command{
 	Use:   "logs",
 	Short: "Create a zip file with logs for Github issue or troubleshoot",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("logs", config.Config.Logs)
+
 		kubernetesProvider, err := getKubernetesProviderForCli()
 		if err != nil {
 			return nil

cli/cmd/tap.go

@@ -48,6 +48,7 @@ func init() {
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
+	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.EnableRedactionTapName, defaultTapConfig.EnableRedaction, "Enables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.")

cli/cmd/tapRunner.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/up9inc/mizu/cli/resources"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/utils"
 
 	core "k8s.io/api/core/v1"
@@ -108,6 +109,8 @@ func RunMizuTap() {
 }
 
 func finishTapExecution(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportTapTelemetry(apiProvider, config.Config.Tap, state.startTime)
+
 	finishMizuExecution(kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
 }
 
@@ -123,6 +126,7 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		AgentDatabasePath:      shared.DataDirPath,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Telemetry:              config.Config.Telemetry,
 	}
 
 	return &mizuAgentConfig
@@ -147,18 +151,17 @@ func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.
 	}
 }
 
-func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, startTime time.Time) error {
+func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error {
 	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
-		TargetNamespaces:       targetNamespaces,
-		PodFilterRegex:         *config.Config.Tap.PodRegex(),
-		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
-		AgentImage:             config.Config.AgentImage,
-		TapperResources:        config.Config.Tap.TapperResources,
-		ImagePullPolicy:        config.Config.ImagePullPolicy(),
-		LogLevel:               config.Config.LogLevel(),
-		MizuApiFilteringOptions: api.TrafficFilteringOptions{
-			IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
-		},
+		TargetNamespaces:         targetNamespaces,
+		PodFilterRegex:           *config.Config.Tap.PodRegex(),
+		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
+		AgentImage:               config.Config.AgentImage,
+		TapperResources:          config.Config.Tap.TapperResources,
+		ImagePullPolicy:          config.Config.ImagePullPolicy(),
+		LogLevel:                 config.Config.LogLevel(),
+		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
+		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
 		ServiceMesh:              config.Config.Tap.ServiceMesh,
 		Tls:                      config.Config.Tap.Tls,
@@ -226,6 +229,27 @@ func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError)
 	}
 }
 
+func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
+	var compiledRegexSlice []*api.SerializableRegexp
+
+	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
+		compiledRegexSlice = make([]*api.SerializableRegexp, 0)
+		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
+			compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr)
+			if err != nil {
+				return nil, err
+			}
+			compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
+		}
+	}
+
+	return &api.TrafficFilteringOptions{
+		PlainTextMaskingRegexes: compiledRegexSlice,
+		IgnoredUserAgents:       config.Config.Tap.IgnoredUserAgents,
+		EnableRedaction:        config.Config.Tap.EnableRedaction,
+	}, nil
+}
+
 func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName))
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
@@ -343,7 +367,8 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel, config.Config.Tap.GuiPort)
 
-	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, state.startTime); err != nil {
+	options, _ := getMizuApiFilteringOptions()
+	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", errormessage.FormatError(err)))
 		cancel()
 	}

cli/cmd/version.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 
 	"github.com/creasty/defaults"
@@ -17,6 +18,8 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("version", config.Config.Version)
+
 		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
 			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.Ver, mizu.Branch, mizu.GitCommitHash)

cli/cmd/view.go

@@ -3,7 +3,9 @@ package cmd
 import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/logger"
 )
 
@@ -11,6 +13,7 @@ var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("view", config.Config.View)
 		runMizuView()
 		return nil
 	},

cli/config/configStruct.go

@@ -31,6 +31,7 @@ type ConfigStruct struct {
 	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
+	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
 	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
 	KubeContext            string                      `yaml:"kube-context"`

cli/config/configStructs/tapConfig.go

@@ -6,7 +6,6 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
-	"strings"
 
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
@@ -16,43 +15,38 @@ import (
 )
 
 const (
-	GuiPortTapName               = "gui-port"
-	NamespacesTapName            = "namespaces"
-	AllNamespacesTapName         = "all-namespaces"
-	EnableRedactionTapName       = "redact"
-	HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
-	InsertionFilterName          = "insertion-filter"
-	DryRunTapName                = "dry-run"
-	ServiceMeshName              = "service-mesh"
-	TlsName                      = "tls"
-	ProfilerName                 = "profiler"
-	MaxLiveStreamsName           = "max-live-streams"
+	GuiPortTapName                = "gui-port"
+	NamespacesTapName             = "namespaces"
+	AllNamespacesTapName          = "all-namespaces"
+	PlainTextFilterRegexesTapName = "regex-masking"
+	EnableRedactionTapName        = "redact"
+	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	InsertionFilterName           = "insertion-filter"
+	DryRunTapName                 = "dry-run"
+	ServiceMeshName               = "service-mesh"
+	TlsName                       = "tls"
+	ProfilerName                  = "profiler"
+	MaxLiveStreamsName            = "max-live-streams"
 )
 
 type TapConfig struct {
-	PodRegexStr       string   `yaml:"regex" default:".*"`
-	GuiPort           uint16   `yaml:"gui-port" default:"8899"`
-	ProxyHost         string   `yaml:"proxy-host" default:"127.0.0.1"`
-	Namespaces        []string `yaml:"namespaces"`
-	AllNamespaces     bool     `yaml:"all-namespaces" default:"false"`
-	IgnoredUserAgents []string `yaml:"ignored-user-agents"`
-	EnableRedaction   bool     `yaml:"redact" default:"false"`
-	RedactPatterns    struct {
-		RequestHeaders     []string `yaml:"request-headers"`
-		ResponseHeaders    []string `yaml:"response-headers"`
-		RequestBody        []string `yaml:"request-body"`
-		ResponseBody       []string `yaml:"response-body"`
-		RequestQueryParams []string `yaml:"request-query-params"`
-	} `yaml:"redact-patterns"`
-	HumanMaxEntriesDBSize string           `yaml:"max-entries-db-size" default:"200MB"`
-	InsertionFilter       string           `yaml:"insertion-filter" default:""`
-	DryRun                bool             `yaml:"dry-run" default:"false"`
-	ApiServerResources    shared.Resources `yaml:"api-server-resources"`
-	TapperResources       shared.Resources `yaml:"tapper-resources"`
-	ServiceMesh           bool             `yaml:"service-mesh" default:"false"`
-	Tls                   bool             `yaml:"tls" default:"false"`
-	Profiler              bool             `yaml:"profiler" default:"false"`
-	MaxLiveStreams        int              `yaml:"max-live-streams" default:"500"`
+	PodRegexStr            string           `yaml:"regex" default:".*"`
+	GuiPort                uint16           `yaml:"gui-port" default:"8899"`
+	ProxyHost              string           `yaml:"proxy-host" default:"127.0.0.1"`
+	Namespaces             []string         `yaml:"namespaces"`
+	AllNamespaces          bool             `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes []string         `yaml:"regex-masking"`
+	IgnoredUserAgents      []string         `yaml:"ignored-user-agents"`
+	EnableRedaction        bool             `yaml:"redact" default:"false"`
+	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
+	InsertionFilter        string           `yaml:"insertion-filter" default:""`
+	DryRun                 bool             `yaml:"dry-run" default:"false"`
+	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
+	TapperResources        shared.Resources `yaml:"tapper-resources"`
+	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
+	Tls                    bool             `yaml:"tls" default:"false"`
+	Profiler               bool             `yaml:"profiler" default:"false"`
+	MaxLiveStreams         int              `yaml:"max-live-streams" default:"500"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -77,48 +71,9 @@ func (config *TapConfig) GetInsertionFilter() string {
 			}
 		}
 	}
-
-	redactFilter := getRedactFilter(config)
-	if insertionFilter != "" && redactFilter != "" {
-		return fmt.Sprintf("(%s) and (%s)", insertionFilter, redactFilter)
-	} else if insertionFilter == "" && redactFilter != "" {
-		return redactFilter
-	}
-
 	return insertionFilter
 }
 
-func getRedactFilter(config *TapConfig) string {
-	if !config.EnableRedaction {
-		return ""
-	}
-
-	var redactValues []string
-	for _, requestHeader := range config.RedactPatterns.RequestHeaders {
-		redactValues = append(redactValues, fmt.Sprintf("request.headers['%s']", requestHeader))
-	}
-	for _, responseHeader := range config.RedactPatterns.ResponseHeaders {
-		redactValues = append(redactValues, fmt.Sprintf("response.headers['%s']", responseHeader))
-	}
-
-	for _, requestBody := range config.RedactPatterns.RequestBody {
-		redactValues = append(redactValues, fmt.Sprintf("request.postData.text.json()...%s", requestBody))
-	}
-	for _, responseBody := range config.RedactPatterns.ResponseBody {
-		redactValues = append(redactValues, fmt.Sprintf("response.content.text.json()...%s", responseBody))
-	}
-
-	for _, requestQueryParams := range config.RedactPatterns.RequestQueryParams {
-		redactValues = append(redactValues, fmt.Sprintf("request.queryString['%s']", requestQueryParams))
-	}
-
-	if len(redactValues) == 0 {
-		return ""
-	}
-
-	return fmt.Sprintf("redact(\"%s\")", strings.Join(redactValues, "\",\""))
-}
-
 func (config *TapConfig) Validate() error {
 	_, compileErr := regexp.Compile(config.PodRegexStr)
 	if compileErr != nil {

cli/go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/creasty/defaults v1.5.2
+	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/google/go-github/v37 v37.0.0
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0

cli/go.sum

@@ -145,6 +145,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
+github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
+github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=

cli/mizu/consts.go

@@ -14,6 +14,8 @@ var (
 	Platform                                   = ""
 )
 
+const DEVENVVAR = "MIZU_DISABLE_TELEMTRY"
+
 func GetMizuFolderPath() string {
 	home, homeDirErr := os.UserHomeDir()
 	if homeDirErr != nil {

cli/mizu/version/versionCheck.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"runtime"
 	"strings"
 	"time"
@@ -17,6 +18,10 @@ import (
 )
 
 func CheckNewerVersion(versionChan chan string) {
+	if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
+		versionChan <- ""
+		return
+	}
 	logger.Log.Debugf("Checking for newer version...")
 	start := time.Now()
 	client := github.NewClient(nil)

cli/telemetry/telemetry.go

@@ -0,0 +1,97 @@
+package telemetry
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/denisbrodbeck/machineid"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/logger"
+)
+
+const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
+
+func ReportRun(cmd string, args interface{}) {
+	if !shouldRunTelemetry() {
+		logger.Log.Debug("not reporting telemetry")
+		return
+	}
+
+	argsBytes, _ := json.Marshal(args)
+	argsMap := map[string]interface{}{
+		"cmd":  cmd,
+		"args": string(argsBytes),
+	}
+
+	if err := sendTelemetry(argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
+}
+
+func ReportTapTelemetry(apiProvider *apiserver.Provider, args interface{}, startTime time.Time) {
+	if !shouldRunTelemetry() {
+		logger.Log.Debug("not reporting telemetry")
+		return
+	}
+
+	generalStats, err := apiProvider.GetGeneralStats()
+	if err != nil {
+		logger.Log.Debugf("[ERROR] failed to get general stats from api server %v", err)
+		return
+	}
+	argsBytes, _ := json.Marshal(args)
+	argsMap := map[string]interface{}{
+		"cmd":                    "tap",
+		"args":                   string(argsBytes),
+		"executionTimeInSeconds": int(time.Since(startTime).Seconds()),
+		"apiCallsCount":          generalStats["EntriesCount"],
+		"trafficVolumeInGB":      generalStats["EntriesVolumeInGB"],
+	}
+
+	if err := sendTelemetry(argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debug("successfully reported telemetry of tap command")
+}
+
+func shouldRunTelemetry() bool {
+	if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
+		return false
+	}
+	if !config.Config.Telemetry {
+		return false
+	}
+
+	return mizu.Branch == "main" || mizu.Branch == "develop"
+}
+
+func sendTelemetry(argsMap map[string]interface{}) error {
+	argsMap["component"] = "mizu_cli"
+	argsMap["buildTimestamp"] = mizu.BuildTimestamp
+	argsMap["branch"] = mizu.Branch
+	argsMap["version"] = mizu.Ver
+	argsMap["platform"] = mizu.Platform
+
+	if machineId, err := machineid.ProtectedID("mizu"); err == nil {
+		argsMap["machineId"] = machineId
+	}
+
+	jsonValue, _ := json.Marshal(argsMap)
+
+	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp)
+	}
+
+	return nil
+}

performance_analysis/run_tapper_benchmark.sh

@@ -57,7 +57,7 @@ log "Writing output to $MIZU_BENCHMARK_OUTPUT_DIR"
 cd $MIZU_HOME || exit 1
 
 export HOST_MODE=0
-export SENSITIVE_DATA_FILTERING_OPTIONS='{}'
+export SENSITIVE_DATA_FILTERING_OPTIONS='{"EnableRedaction": false}'
 export MIZU_DEBUG_DISABLE_PCAP=false
 export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false
 export MIZU_DEBUG_DISABLE_TCP_STREAM=false

shared/kubernetes/mizuTapperSyncer.go

@@ -43,6 +43,7 @@ type TapperSyncerConfig struct {
 	TapperResources          shared.Resources
 	ImagePullPolicy          core.PullPolicy
 	LogLevel                 logging.Level
+	IgnoredUserAgents        []string
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
 	ServiceMesh              bool

shared/models.go

@@ -43,6 +43,7 @@ type MizuAgentConfig struct {
 	AgentDatabasePath      string        `json:"agentDatabasePath"`
 	ServiceMap             bool          `json:"serviceMap"`
 	OAS                    OASConfig     `json:"oas"`
+	Telemetry              bool          `json:"telemetry"`
 }
 
 type WebSocketMessageMetadata struct {

tap/api/api.go

@@ -2,9 +2,7 @@ package api
 
 import (
 	"bufio"
-	"fmt"
 	"net"
-	"strings"
 	"sync"
 	"time"
 
@@ -16,29 +14,12 @@ const UnknownNamespace = ""
 var UnknownIp = net.IP{0, 0, 0, 0}
 var UnknownPort uint16 = 0
 
-type ProtocolSummary struct {
-	Name         string `json:"name"`
-	Version      string `json:"version"`
-	Abbreviation string `json:"abbr"`
-}
-
-func (protocol *ProtocolSummary) ToString() string {
-	return fmt.Sprintf("%s?%s?%s", protocol.Name, protocol.Version, protocol.Abbreviation)
-}
-
-func GetProtocolSummary(inputString string) *ProtocolSummary {
-	splitted := strings.SplitN(inputString, "?", 3)
-	return &ProtocolSummary{
-		Name:         splitted[0],
-		Version:      splitted[1],
-		Abbreviation: splitted[2],
-	}
-}
-
 type Protocol struct {
-	ProtocolSummary
+	Name            string   `json:"name"`
 	LongName        string   `json:"longName"`
+	Abbreviation    string   `json:"abbr"`
 	Macro           string   `json:"macro"`
+	Version         string   `json:"version"`
 	BackgroundColor string   `json:"backgroundColor"`
 	ForegroundColor string   `json:"foregroundColor"`
 	FontSize        int8     `json:"fontSize"`
@@ -170,7 +151,7 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 
 type Entry struct {
 	Id           string                 `json:"id"`
-	Protocol     ProtocolSummary        `json:"protocol"`
+	ProtocolId   string                 `json:"protocol"`
 	Capture      Capture                `json:"capture"`
 	Source       *TCP                   `json:"src"`
 	Destination  *TCP                   `json:"dst"`

tap/api/options.go

@@ -1,5 +1,7 @@
 package api
 
 type TrafficFilteringOptions struct {
-	IgnoredUserAgents []string
+	IgnoredUserAgents       []string
+	PlainTextMaskingRegexes []*SerializableRegexp
+	EnableRedaction         bool
 }

tap/extensions/amqp/Makefile

@@ -8,7 +8,9 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/amqp bin
+	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/amqp/\*.bin bin
 
 test-pull-expect:
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/amqp expect
+	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/amqp/\* expect

tap/extensions/amqp/go.mod

@@ -4,20 +4,16 @@ go 1.17
 
 require (
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
 
-replace github.com/up9inc/mizu/logger v0.0.0 => ../../../logger
-
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api
 
 replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../../dbgctl

tap/extensions/amqp/go.sum

@@ -1,7 +1,5 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

tap/extensions/amqp/helpers.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
+	"time"
 
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
 
@@ -25,14 +25,14 @@ var connectionMethodMap = map[int]string{
 	61: "connection unblocked",
 }
 
-var channelMethodMap = map[int]string{
-	10: "channel open",
-	11: "channel open-ok",
-	20: "channel flow",
-	21: "channel flow-ok",
-	40: "channel close",
-	41: "channel close-ok",
-}
+// var channelMethodMap = map[int]string{
+// 	10: "channel open",
+// 	11: "channel open-ok",
+// 	20: "channel flow",
+// 	21: "channel flow-ok",
+// 	40: "channel close",
+// 	41: "channel close-ok",
+// }
 
 var exchangeMethodMap = map[int]string{
 	10: "exchange declare",
@@ -94,41 +94,29 @@ type AMQPWrapper struct {
 	Details interface{} `json:"details"`
 }
 
-type emptyResponse struct {
-}
-
-const emptyMethod = "empty"
-
-func getIdent(reader api.TcpReader, methodFrame *MethodFrame) (ident string) {
-	tcpID := reader.GetTcpID()
-	// To match methods to their Ok(s)
-	methodId := methodFrame.MethodId - methodFrame.MethodId%10
-
-	if reader.GetIsClient() {
-		ident = fmt.Sprintf(
-			"%s_%s_%s_%s_%d_%d_%d",
-			tcpID.SrcIP,
-			tcpID.DstIP,
-			tcpID.SrcPort,
-			tcpID.DstPort,
-			methodFrame.ChannelId,
-			methodFrame.ClassId,
-			methodId,
-		)
-	} else {
-		ident = fmt.Sprintf(
-			"%s_%s_%s_%s_%d_%d_%d",
-			tcpID.DstIP,
-			tcpID.SrcIP,
-			tcpID.DstPort,
-			tcpID.SrcPort,
-			methodFrame.ChannelId,
-			methodFrame.ClassId,
-			methodId,
-		)
+func emitAMQP(event interface{}, _type string, method string, connectionInfo *api.ConnectionInfo, captureTime time.Time, captureSize int, emitter api.Emitter, capture api.Capture) {
+	request := &api.GenericMessage{
+		IsRequest:   true,
+		CaptureTime: captureTime,
+		Payload: AMQPPayload{
+			Data: &AMQPWrapper{
+				Method:  method,
+				Url:     "",
+				Details: event,
+			},
+		},
 	}
-
-	return
+	item := &api.OutputChannelItem{
+		Protocol:       protocol,
+		Capture:        capture,
+		Timestamp:      captureTime.UnixNano() / int64(time.Millisecond),
+		ConnectionInfo: connectionInfo,
+		Pair: &api.RequestResponsePair{
+			Request:  *request,
+			Response: api.GenericMessage{},
+		},
+	}
+	emitter.Emit(item)
 }
 
 func representProperties(properties map[string]interface{}, rep []interface{}) ([]interface{}, string, string) {
@@ -472,36 +460,6 @@ func representQueueDeclare(event map[string]interface{}) []interface{} {
 	return rep
 }
 
-func representQueueDeclareOk(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	details, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Queue",
-			Value:    event["queue"].(string),
-			Selector: `response.queue`,
-		},
-		{
-			Name:     "Message Count",
-			Value:    fmt.Sprintf("%g", event["messageCount"].(float64)),
-			Selector: `response.messageCount`,
-		},
-		{
-			Name:     "Consumer Count",
-			Value:    fmt.Sprintf("%g", event["consumerCount"].(float64)),
-			Selector: `response.consumerCount`,
-		},
-	})
-
-	rep = append(rep, api.SectionData{
-		Type:  api.TABLE,
-		Title: "Details",
-		Data:  string(details),
-	})
-
-	return rep
-}
-
 func representExchangeDeclare(event map[string]interface{}) []interface{} {
 	rep := make([]interface{}, 0)
 
@@ -613,7 +571,7 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 				x, _ := json.Marshal(value)
 				outcome = string(x)
 			default:
-				logger.Log.Info("Unknown data type for the server property!")
+				panic("Unknown data type for the server property!")
 			}
 			headers = append(headers, api.TableData{
 				Name:     name,
@@ -635,65 +593,6 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 	return rep
 }
 
-func representConnectionStartOk(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	details, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Mechanism",
-			Value:    event["mechanism"].(string),
-			Selector: `response.mechanism`,
-		},
-		{
-			Name:     "Mechanism",
-			Value:    event["mechanism"].(string),
-			Selector: `response.response`,
-		},
-		{
-			Name:     "Locale",
-			Value:    event["locale"].(string),
-			Selector: `response.locale`,
-		},
-	})
-	rep = append(rep, api.SectionData{
-		Type:  api.TABLE,
-		Title: "Details",
-		Data:  string(details),
-	})
-
-	if event["clientProperties"] != nil {
-		headers := make([]api.TableData, 0)
-		for name, value := range event["clientProperties"].(map[string]interface{}) {
-			var outcome string
-			switch v := value.(type) {
-			case string:
-				outcome = v
-			case map[string]interface{}:
-				x, _ := json.Marshal(value)
-				outcome = string(x)
-			default:
-				logger.Log.Info("Unknown data type for the client property!")
-			}
-			headers = append(headers, api.TableData{
-				Name:     name,
-				Value:    outcome,
-				Selector: fmt.Sprintf(`response.clientProperties["%s"]`, name),
-			})
-		}
-		sort.Slice(headers, func(i, j int) bool {
-			return headers[i].Name < headers[j].Name
-		})
-		headersMarshaled, _ := json.Marshal(headers)
-		rep = append(rep, api.SectionData{
-			Type:  api.TABLE,
-			Title: "Client Properties",
-			Data:  string(headersMarshaled),
-		})
-	}
-
-	return rep
-}
-
 func representConnectionClose(event map[string]interface{}) []interface{} {
 	replyCode := ""
 
@@ -851,122 +750,3 @@ func representBasicConsume(event map[string]interface{}) []interface{} {
 
 	return rep
 }
-
-func representBasicConsumeOk(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	details, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Consumer Tag",
-			Value:    event["consumerTag"].(string),
-			Selector: `response.consumerTag`,
-		},
-	})
-
-	rep = append(rep, api.SectionData{
-		Type:  api.TABLE,
-		Title: "Details",
-		Data:  string(details),
-	})
-
-	return rep
-}
-
-func representConnectionOpen(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	details, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Virtual Host",
-			Value:    event["virtualHost"].(string),
-			Selector: `request.virtualHost`,
-		},
-	})
-	rep = append(rep, api.SectionData{
-		Type:  api.TABLE,
-		Title: "Details",
-		Data:  string(details),
-	})
-
-	return rep
-}
-
-func representConnectionTune(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	details, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Channel Max",
-			Value:    fmt.Sprintf("%g", event["channelMax"].(float64)),
-			Selector: `request.channelMax`,
-		},
-		{
-			Name:     "Frame Max",
-			Value:    fmt.Sprintf("%g", event["frameMax"].(float64)),
-			Selector: `request.frameMax`,
-		},
-		{
-			Name:     "Heartbeat",
-			Value:    fmt.Sprintf("%g", event["heartbeat"].(float64)),
-			Selector: `request.heartbeat`,
-		},
-	})
-	rep = append(rep, api.SectionData{
-		Type:  api.TABLE,
-		Title: "Details",
-		Data:  string(details),
-	})
-
-	return rep
-}
-
-func representBasicCancel(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	details, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Consumer Tag",
-			Value:    event["consumerTag"].(string),
-			Selector: `response.consumerTag`,
-		},
-		{
-			Name:     "NoWait",
-			Value:    strconv.FormatBool(event["noWait"].(bool)),
-			Selector: `request.noWait`,
-		},
-	})
-
-	rep = append(rep, api.SectionData{
-		Type:  api.TABLE,
-		Title: "Details",
-		Data:  string(details),
-	})
-
-	return rep
-}
-
-func representBasicCancelOk(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	details, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Consumer Tag",
-			Value:    event["consumerTag"].(string),
-			Selector: `response.consumerTag`,
-		},
-	})
-
-	rep = append(rep, api.SectionData{
-		Type:  api.TABLE,
-		Title: "Details",
-		Data:  string(details),
-	})
-
-	return rep
-}
-
-func representEmpty(event map[string]interface{}) []interface{} {
-	rep := make([]interface{}, 0)
-
-	return rep
-}

tap/extensions/amqp/main.go

@@ -13,13 +13,11 @@ import (
 )
 
 var protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "amqp",
-		Version:      "0-9-1",
-		Abbreviation: "AMQP",
-	},
+	Name:            "amqp",
 	LongName:        "Advanced Message Queuing Protocol 0-9-1",
+	Abbreviation:    "AMQP",
 	Macro:           "amqp",
+	Version:         "0-9-1",
 	BackgroundColor: "#ff6600",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -29,7 +27,7 @@ var protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	protocol.ToString(): &protocol,
+	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
 }
 
 type dissecting string
@@ -46,12 +44,22 @@ func (d dissecting) Ping() {
 	log.Printf("pong %s", protocol.Name)
 }
 
+const amqpRequest string = "amqp_request"
+
 func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.TrafficFilteringOptions) error {
 	r := AmqpReader{b}
 
 	var remaining int
 	var header *HeaderFrame
 
+	connectionInfo := &api.ConnectionInfo{
+		ClientIP:   reader.GetTcpID().SrcIP,
+		ClientPort: reader.GetTcpID().SrcPort,
+		ServerIP:   reader.GetTcpID().DstIP,
+		ServerPort: reader.GetTcpID().DstPort,
+		IsOutgoing: true,
+	}
+
 	eventBasicPublish := &BasicPublish{
 		Exchange:   "",
 		RoutingKey: "",
@@ -73,10 +81,6 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 
 	var lastMethodFrameMessage Message
 
-	var ident string
-	isClient := reader.GetIsClient()
-	reqResMatcher := reader.GetReqResMatcher().(*requestResponseMatcher)
-
 	for {
 		frameVal, err := r.readFrame()
 		if err == io.EOF {
@@ -115,22 +119,16 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 			switch lastMethodFrameMessage.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Body = f.Body
-				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[40], *eventBasicPublish, reader)
-				reqResMatcher.emitEvent(!isClient, ident, emptyMethod, &emptyResponse{}, reader)
-
+				emitAMQP(*eventBasicPublish, amqpRequest, basicMethodMap[40], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 			case *BasicDeliver:
 				eventBasicDeliver.Body = f.Body
-				reqResMatcher.emitEvent(!isClient, ident, basicMethodMap[60], *eventBasicDeliver, reader)
-				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
+				emitAMQP(*eventBasicDeliver, amqpRequest, basicMethodMap[60], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 			}
 
 		case *MethodFrame:
 			reader.GetParent().SetProtocol(&protocol)
 
 			lastMethodFrameMessage = f.Method
-
-			ident = getIdent(reader, f)
-
 			switch m := f.Method.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Exchange = m.Exchange
@@ -146,10 +144,7 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[20], *eventQueueBind, reader)
-
-			case *QueueBindOk:
-				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[21], m, reader)
+				emitAMQP(*eventQueueBind, amqpRequest, queueMethodMap[20], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 
 			case *BasicConsume:
 				eventBasicConsume := &BasicConsume{
@@ -161,10 +156,7 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:      m.NoWait,
 					Arguments:   m.Arguments,
 				}
-				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[20], *eventBasicConsume, reader)
-
-			case *BasicConsumeOk:
-				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[21], m, reader)
+				emitAMQP(*eventBasicConsume, amqpRequest, basicMethodMap[20], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 
 			case *BasicDeliver:
 				eventBasicDeliver.ConsumerTag = m.ConsumerTag
@@ -183,10 +175,7 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[10], *eventQueueDeclare, reader)
-
-			case *QueueDeclareOk:
-				reqResMatcher.emitEvent(isClient, ident, queueMethodMap[11], m, reader)
+				emitAMQP(*eventQueueDeclare, amqpRequest, queueMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 
 			case *ExchangeDeclare:
 				eventExchangeDeclare := &ExchangeDeclare{
@@ -199,19 +188,17 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					NoWait:     m.NoWait,
 					Arguments:  m.Arguments,
 				}
-				reqResMatcher.emitEvent(isClient, ident, exchangeMethodMap[10], *eventExchangeDeclare, reader)
-
-			case *ExchangeDeclareOk:
-				reqResMatcher.emitEvent(isClient, ident, exchangeMethodMap[11], m, reader)
+				emitAMQP(*eventExchangeDeclare, amqpRequest, exchangeMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 
 			case *ConnectionStart:
-				// In our tests, *ConnectionStart does not result in *ConnectionStartOk
-				reqResMatcher.emitEvent(!isClient, ident, connectionMethodMap[10], m, reader)
-				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
-
-			case *ConnectionStartOk:
-				// In our tests, *ConnectionStart does not result in *ConnectionStartOk
-				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[11], m, reader)
+				eventConnectionStart := &ConnectionStart{
+					VersionMajor:     m.VersionMajor,
+					VersionMinor:     m.VersionMinor,
+					ServerProperties: m.ServerProperties,
+					Mechanisms:       m.Mechanisms,
+					Locales:          m.Locales,
+				}
+				emitAMQP(*eventConnectionStart, amqpRequest, connectionMethodMap[10], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 
 			case *ConnectionClose:
 				eventConnectionClose := &ConnectionClose{
@@ -220,40 +207,7 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 					ClassId:   m.ClassId,
 					MethodId:  m.MethodId,
 				}
-				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[50], *eventConnectionClose, reader)
-
-			case *ConnectionCloseOk:
-				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[51], m, reader)
-
-			case *connectionOpen:
-				eventConnectionOpen := &connectionOpen{
-					VirtualHost: m.VirtualHost,
-				}
-				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[40], *eventConnectionOpen, reader)
-
-			case *connectionOpenOk:
-				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[41], m, reader)
-
-			case *channelOpen:
-				reqResMatcher.emitEvent(isClient, ident, channelMethodMap[10], m, reader)
-
-			case *channelOpenOk:
-				reqResMatcher.emitEvent(isClient, ident, channelMethodMap[11], m, reader)
-
-			case *connectionTune:
-				// In our tests, *connectionTune does not result in *connectionTuneOk
-				reqResMatcher.emitEvent(!isClient, ident, connectionMethodMap[30], m, reader)
-				reqResMatcher.emitEvent(isClient, ident, emptyMethod, &emptyResponse{}, reader)
-
-			case *connectionTuneOk:
-				// In our tests, *connectionTune does not result in *connectionTuneOk
-				reqResMatcher.emitEvent(isClient, ident, connectionMethodMap[31], m, reader)
-
-			case *basicCancel:
-				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[30], m, reader)
-
-			case *basicCancelOk:
-				reqResMatcher.emitEvent(isClient, ident, basicMethodMap[31], m, reader)
+				emitAMQP(*eventConnectionClose, amqpRequest, connectionMethodMap[50], connectionInfo, reader.GetCaptureTime(), reader.GetReadProgress().Current(), reader.GetEmitter(), reader.GetParent().GetOrigin())
 			}
 
 		default:
@@ -264,20 +218,12 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 
 func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
-	response := item.Pair.Response.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
-	resDetails := response["details"].(map[string]interface{})
-
-	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	if elapsedTime < 0 {
-		elapsedTime = 0
-	}
 
 	reqDetails["method"] = request["method"]
-	resDetails["method"] = response["method"]
 	return &api.Entry{
-		Protocol: protocol.ProtocolSummary,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -288,15 +234,13 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 			IP:   item.ConnectionInfo.ServerIP,
 			Port: item.ConnectionInfo.ServerPort,
 		},
-		Namespace:    namespace,
-		Outgoing:     item.ConnectionInfo.IsOutgoing,
-		Request:      reqDetails,
-		Response:     resDetails,
-		RequestSize:  item.Pair.Request.CaptureSize,
-		ResponseSize: item.Pair.Response.CaptureSize,
-		Timestamp:    item.Timestamp,
-		StartTime:    item.Pair.Request.CaptureTime,
-		ElapsedTime:  elapsedTime,
+		Namespace:   namespace,
+		Outgoing:    item.ConnectionInfo.IsOutgoing,
+		Request:     reqDetails,
+		RequestSize: item.Pair.Request.CaptureSize,
+		Timestamp:   item.Timestamp,
+		StartTime:   item.Pair.Request.CaptureTime,
+		ElapsedTime: 0,
 	}
 
 }
@@ -337,26 +281,11 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	case basicMethodMap[20]:
 		summary = entry.Request["queue"].(string)
 		summaryQuery = fmt.Sprintf(`request.queue == "%s"`, summary)
-	case connectionMethodMap[40]:
-		summary = entry.Request["virtualHost"].(string)
-		summaryQuery = fmt.Sprintf(`request.virtualHost == "%s"`, summary)
-	case connectionMethodMap[30]:
-		summary = fmt.Sprintf("%g", entry.Request["channelMax"].(float64))
-		summaryQuery = fmt.Sprintf(`request.channelMax == "%s"`, summary)
-	case connectionMethodMap[31]:
-		summary = fmt.Sprintf("%g", entry.Request["channelMax"].(float64))
-		summaryQuery = fmt.Sprintf(`request.channelMax == "%s"`, summary)
-	case basicMethodMap[30]:
-		summary = entry.Request["consumerTag"].(string)
-		summaryQuery = fmt.Sprintf(`request.consumerTag == "%s"`, summary)
-	case basicMethodMap[31]:
-		summary = entry.Request["consumerTag"].(string)
-		summaryQuery = fmt.Sprintf(`request.consumerTag == "%s"`, summary)
 	}
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.Protocol.ToString()],
+		Protocol:     *protocolsMap[entry.ProtocolId],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -375,8 +304,6 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, err error) {
 	representation := make(map[string]interface{})
 	var repRequest []interface{}
-	var repResponse []interface{}
-
 	switch request["method"].(string) {
 	case basicMethodMap[40]:
 		repRequest = representBasicPublish(request)
@@ -394,56 +321,20 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 		repRequest = representQueueBind(request)
 	case basicMethodMap[20]:
 		repRequest = representBasicConsume(request)
-	case connectionMethodMap[40]:
-		repRequest = representConnectionOpen(request)
-	case channelMethodMap[10]:
-		repRequest = representEmpty(request)
-	case connectionMethodMap[30]:
-		repRequest = representConnectionTune(request)
-	case basicMethodMap[30]:
-		repRequest = representBasicCancel(request)
 	}
-
-	switch response["method"].(string) {
-	case queueMethodMap[11]:
-		repResponse = representQueueDeclareOk(response)
-	case exchangeMethodMap[11]:
-		repResponse = representEmpty(response)
-	case connectionMethodMap[11]:
-		repResponse = representConnectionStartOk(response)
-	case connectionMethodMap[51]:
-		repResponse = representEmpty(response)
-	case basicMethodMap[21]:
-		repResponse = representBasicConsumeOk(response)
-	case queueMethodMap[21]:
-		repResponse = representEmpty(response)
-	case connectionMethodMap[41]:
-		repResponse = representEmpty(response)
-	case channelMethodMap[11]:
-		repResponse = representEmpty(request)
-	case connectionMethodMap[31]:
-		repResponse = representConnectionTune(request)
-	case basicMethodMap[31]:
-		repResponse = representBasicCancelOk(request)
-	case emptyMethod:
-		repResponse = representEmpty(response)
-	}
-
 	representation["request"] = repRequest
-	representation["response"] = repResponse
 	object, err = json.Marshal(representation)
-
 	return
 }
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`amqp`: fmt.Sprintf(`protocol.name == "%s"`, protocol.Name),
+		`amqp`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
 	}
 }
 
 func (d dissecting) NewResponseRequestMatcher() api.RequestResponseMatcher {
-	return createResponseRequestMatcher()
+	return nil
 }
 
 var Dissector dissecting

tap/extensions/amqp/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"amqp": `protocol.name == "amqp"`,
+		"amqp": `protocol == "amqp/0-9-1/AMQP"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/amqp/matcher.go

@@ -1,113 +0,0 @@
-package amqp
-
-import (
-	"sync"
-	"time"
-
-	"github.com/up9inc/mizu/tap/api"
-)
-
-// Key is {client_addr}_{client_port}_{dest_addr}_{dest_port}_{channel_id}_{class_id}_{method_id}
-type requestResponseMatcher struct {
-	openMessagesMap *sync.Map
-}
-
-func createResponseRequestMatcher() api.RequestResponseMatcher {
-	return &requestResponseMatcher{openMessagesMap: &sync.Map{}}
-}
-
-func (matcher *requestResponseMatcher) GetMap() *sync.Map {
-	return matcher.openMessagesMap
-}
-
-func (matcher *requestResponseMatcher) SetMaxTry(value int) {
-}
-
-func (matcher *requestResponseMatcher) emitEvent(isRequest bool, ident string, method string, event interface{}, reader api.TcpReader) {
-	reader.GetParent().SetProtocol(&protocol)
-
-	var item *api.OutputChannelItem
-	if isRequest {
-		item = matcher.registerRequest(ident, method, event, reader.GetCaptureTime(), reader.GetReadProgress().Current())
-	} else {
-		item = matcher.registerResponse(ident, method, event, reader.GetCaptureTime(), reader.GetReadProgress().Current())
-	}
-
-	if item != nil {
-		item.ConnectionInfo = &api.ConnectionInfo{
-			ClientIP:   reader.GetTcpID().SrcIP,
-			ClientPort: reader.GetTcpID().SrcPort,
-			ServerIP:   reader.GetTcpID().DstIP,
-			ServerPort: reader.GetTcpID().DstPort,
-			IsOutgoing: true,
-		}
-		item.Capture = reader.GetParent().GetOrigin()
-		reader.GetEmitter().Emit(item)
-	}
-}
-
-func (matcher *requestResponseMatcher) registerRequest(ident string, method string, request interface{}, captureTime time.Time, captureSize int) *api.OutputChannelItem {
-	requestAMQPMessage := api.GenericMessage{
-		IsRequest:   true,
-		CaptureTime: captureTime,
-		CaptureSize: captureSize,
-		Payload: AMQPPayload{
-			Data: &AMQPWrapper{
-				Method:  method,
-				Url:     "",
-				Details: request,
-			},
-		},
-	}
-
-	if response, found := matcher.openMessagesMap.LoadAndDelete(ident); found {
-		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
-		responseAMQPMessage := response.(*api.GenericMessage)
-		if responseAMQPMessage.IsRequest {
-			return nil
-		}
-		return matcher.preparePair(&requestAMQPMessage, responseAMQPMessage)
-	}
-
-	matcher.openMessagesMap.Store(ident, &requestAMQPMessage)
-	return nil
-}
-
-func (matcher *requestResponseMatcher) registerResponse(ident string, method string, response interface{}, captureTime time.Time, captureSize int) *api.OutputChannelItem {
-	responseAMQPMessage := api.GenericMessage{
-		IsRequest:   false,
-		CaptureTime: captureTime,
-		CaptureSize: captureSize,
-		Payload: AMQPPayload{
-			Data: &AMQPWrapper{
-				Method:  method,
-				Url:     "",
-				Details: response,
-			},
-		},
-	}
-
-	if request, found := matcher.openMessagesMap.LoadAndDelete(ident); found {
-		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
-		requestAMQPMessage := request.(*api.GenericMessage)
-		if !requestAMQPMessage.IsRequest {
-			return nil
-		}
-		return matcher.preparePair(requestAMQPMessage, &responseAMQPMessage)
-	}
-
-	matcher.openMessagesMap.Store(ident, &responseAMQPMessage)
-	return nil
-}
-
-func (matcher *requestResponseMatcher) preparePair(requestAMQPMessage *api.GenericMessage, responseAMQPMessage *api.GenericMessage) *api.OutputChannelItem {
-	return &api.OutputChannelItem{
-		Protocol:       protocol,
-		Timestamp:      requestAMQPMessage.CaptureTime.UnixNano() / int64(time.Millisecond),
-		ConnectionInfo: nil,
-		Pair: &api.RequestResponsePair{
-			Request:  *requestAMQPMessage,
-			Response: *responseAMQPMessage,
-		},
-	}
-}

tap/extensions/amqp/spec091.go

@@ -81,10 +81,10 @@ func (msg *ConnectionStart) read(r io.Reader) (err error) {
 }
 
 type ConnectionStartOk struct {
-	ClientProperties Table  `json:"clientProperties"`
-	Mechanism        string `json:"mechanism"`
-	Response         string `json:"response"`
-	Locale           string `json:"locale"`
+	ClientProperties Table
+	Mechanism        string
+	Response         string
+	Locale           string
 }
 
 func (msg *ConnectionStartOk) read(r io.Reader) (err error) {
@@ -135,9 +135,9 @@ func (msg *connectionSecureOk) read(r io.Reader) (err error) {
 }
 
 type connectionTune struct {
-	ChannelMax uint16 `json:"channelMax"`
-	FrameMax   uint32 `json:"frameMax"`
-	Heartbeat  uint16 `json:"heartbeat"`
+	ChannelMax uint16
+	FrameMax   uint32
+	Heartbeat  uint16
 }
 
 func (msg *connectionTune) read(r io.Reader) (err error) {
@@ -181,7 +181,7 @@ func (msg *connectionTuneOk) read(r io.Reader) (err error) {
 }
 
 type connectionOpen struct {
-	VirtualHost string `json:"virtualHost"`
+	VirtualHost string
 	reserved1   string
 	reserved2   bool
 }
@@ -580,9 +580,9 @@ func (msg *QueueDeclare) read(r io.Reader) (err error) {
 }
 
 type QueueDeclareOk struct {
-	Queue         string `json:"queue"`
-	MessageCount  uint32 `json:"messageCount"`
-	ConsumerCount uint32 `json:"consumerCount"`
+	Queue         string
+	MessageCount  uint32
+	ConsumerCount uint32
 }
 
 func (msg *QueueDeclareOk) read(r io.Reader) (err error) {
@@ -840,7 +840,7 @@ func (msg *BasicConsume) read(r io.Reader) (err error) {
 }
 
 type BasicConsumeOk struct {
-	ConsumerTag string `json:"consumerTag"`
+	ConsumerTag string
 }
 
 func (msg *BasicConsumeOk) read(r io.Reader) (err error) {
@@ -853,8 +853,8 @@ func (msg *BasicConsumeOk) read(r io.Reader) (err error) {
 }
 
 type basicCancel struct {
-	ConsumerTag string `json:"consumerTag"`
-	NoWait      bool   `json:"noWait"`
+	ConsumerTag string
+	NoWait      bool
 }
 
 func (msg *basicCancel) read(r io.Reader) (err error) {
@@ -873,7 +873,7 @@ func (msg *basicCancel) read(r io.Reader) (err error) {
 }
 
 type basicCancelOk struct {
-	ConsumerTag string `json:"consumerTag"`
+	ConsumerTag string
 }
 
 func (msg *basicCancelOk) read(r io.Reader) (err error) {

tap/extensions/get-folder-of-tests.sh

@@ -1,24 +0,0 @@
-mizu_test_files_repo=https://github.com/up9inc/mizu-tests-data
-mizu_test_files_tmp_folder="mizu-test-files-tmp"
-
-requested_folder=$1
-destination_folder_name=$2
-echo "Going to download folder (${requested_folder}) from repo (${mizu_test_files_repo}) and save it to local folder (${destination_folder_name})"
-
-echo "Cloning repo to tmp folder (${mizu_test_files_tmp_folder})"
-git clone ${mizu_test_files_repo} ${mizu_test_files_tmp_folder} --no-checkout --depth 1 --filter=blob:none --sparse
-cd ${mizu_test_files_tmp_folder}
-
-echo "Adding sparse checkout folder"
-git sparse-checkout add ${requested_folder}
-
-
-echo "Checkout"
-git checkout 
-
-echo "Moving folder to the destination location"
-mv ${requested_folder} ../${destination_folder_name}
-
-cd ..
-echo "Removing the tmp folder"
-rm -rf ${mizu_test_files_tmp_folder}

tap/extensions/http/Makefile

@@ -8,7 +8,9 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/http bin
+	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/http/\*.bin bin
 
 test-pull-expect:
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/http expect
+	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/http/\* expect

tap/extensions/http/handlers.go

@@ -18,6 +18,10 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
 		return
 	}
 
+	if options.EnableRedaction {
+		FilterSensitiveData(item, options)
+	}
+
 	replaceForwardedFor(item)
 
 	emitter.Emit(item)

tap/extensions/http/helpers.go

@@ -6,16 +6,13 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
-	"strings"
 
 	"github.com/up9inc/mizu/tap/api"
 )
 
 func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) {
 	newMap = make(map[string]interface{})
-
-	mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice)
-	for _, item := range mergedMapSlice {
+	for _, item := range mapSlice {
 		h := item.(map[string]interface{})
 		newMap[h["name"].(string)] = h["value"]
 	}
@@ -23,28 +20,6 @@ func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}
 	return
 }
 
-func mapSliceRebuildAsMergedMap(mapSlice []interface{}) (newMap map[string]interface{}) {
-	newMap = make(map[string]interface{})
-
-	mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice)
-	for _, item := range mergedMapSlice {
-		h := item.(map[string]interface{})
-
-		if valuesInterface, ok := h["value"].([]interface{}); ok {
-			var values []string
-			for _, valueInterface := range valuesInterface {
-				values = append(values, valueInterface.(string))
-			}
-
-			newMap[h["name"].(string)] = strings.Join(values, ",")
-		} else {
-			newMap[h["name"].(string)] = h["value"]
-		}
-	}
-
-	return
-}
-
 func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{}) {
 	newMapSlice = make([]interface{}, 0)
 	valuesMap := make(map[string][]interface{})
@@ -72,24 +47,6 @@ func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{
 	return
 }
 
-func representMapAsTable(mapToTable map[string]interface{}, selectorPrefix string) (representation string) {
-	var table []api.TableData
-
-	keys := make([]string, 0, len(mapToTable))
-	for k := range mapToTable {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	for _, key := range keys {
-		table = append(table, createTableForKey(key, mapToTable[key], selectorPrefix)...)
-	}
-
-	obj, _ := json.Marshal(table)
-	representation = string(obj)
-	return
-}
-
 func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
 	for _, item := range mapSlice {
@@ -97,7 +54,34 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re
 		key := h["name"].(string)
 		value := h["value"]
 
-		table = append(table, createTableForKey(key, value, selectorPrefix)...)
+		var reflectKind reflect.Kind
+		reflectType := reflect.TypeOf(value)
+		if reflectType == nil {
+			reflectKind = reflect.Interface
+		} else {
+			reflectKind = reflect.TypeOf(value).Kind()
+		}
+
+		switch reflectKind {
+		case reflect.Slice:
+			fallthrough
+		case reflect.Array:
+			for i, el := range value.([]interface{}) {
+				selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i)
+				table = append(table, api.TableData{
+					Name:     fmt.Sprintf("%s [%d]", key, i),
+					Value:    el,
+					Selector: selector,
+				})
+			}
+		default:
+			selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
+			table = append(table, api.TableData{
+				Name:     key,
+				Value:    value,
+				Selector: selector,
+			})
+		}
 	}
 
 	obj, _ := json.Marshal(table)
@@ -105,41 +89,6 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re
 	return
 }
 
-func createTableForKey(key string, value interface{}, selectorPrefix string) []api.TableData {
-	var table []api.TableData
-
-	var reflectKind reflect.Kind
-	reflectType := reflect.TypeOf(value)
-	if reflectType == nil {
-		reflectKind = reflect.Interface
-	} else {
-		reflectKind = reflect.TypeOf(value).Kind()
-	}
-
-	switch reflectKind {
-	case reflect.Slice:
-		fallthrough
-	case reflect.Array:
-		for i, el := range value.([]interface{}) {
-			selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i)
-			table = append(table, api.TableData{
-				Name:     fmt.Sprintf("%s [%d]", key, i),
-				Value:    el,
-				Selector: selector,
-			})
-		}
-	default:
-		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
-		table = append(table, api.TableData{
-			Name:     key,
-			Value:    value,
-			Selector: selector,
-		})
-	}
-
-	return table
-}
-
 func representSliceAsTable(slice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
 	for i, item := range slice {

tap/extensions/http/main.go

@@ -15,13 +15,11 @@ import (
 )
 
 var http10protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "http",
-		Version:      "1.0",
-		Abbreviation: "HTTP",
-	},
+	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.0",
+	Abbreviation:    "HTTP",
 	Macro:           "http",
+	Version:         "1.0",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -31,13 +29,11 @@ var http10protocol = api.Protocol{
 }
 
 var http11protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "http",
-		Version:      "1.1",
-		Abbreviation: "HTTP",
-	},
+	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+	Abbreviation:    "HTTP",
 	Macro:           "http",
+	Version:         "1.1",
 	BackgroundColor: "#205cf5",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -47,13 +43,11 @@ var http11protocol = api.Protocol{
 }
 
 var http2Protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "http",
-		Version:      "2.0",
-		Abbreviation: "HTTP/2",
-	},
+	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2)",
+	Abbreviation:    "HTTP/2",
 	Macro:           "http2",
+	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -63,13 +57,11 @@ var http2Protocol = api.Protocol{
 }
 
 var grpcProtocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "http",
-		Version:      "2.0",
-		Abbreviation: "gRPC",
-	},
+	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ gRPC over HTTP/2 ]",
+	Abbreviation:    "gRPC",
 	Macro:           "grpc",
+	Version:         "2.0",
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -79,13 +71,11 @@ var grpcProtocol = api.Protocol{
 }
 
 var graphQL1Protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "http",
-		Version:      "1.1",
-		Abbreviation: "GQL",
-	},
+	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1 [ GraphQL over HTTP/1.1 ]",
+	Abbreviation:    "GQL",
 	Macro:           "gql",
+	Version:         "1.1",
 	BackgroundColor: "#e10098",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -95,13 +85,11 @@ var graphQL1Protocol = api.Protocol{
 }
 
 var graphQL2Protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "http",
-		Version:      "2.0",
-		Abbreviation: "GQL",
-	},
+	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ GraphQL over HTTP/2 ]",
+	Abbreviation:    "GQL",
 	Macro:           "gql",
+	Version:         "2.0",
 	BackgroundColor: "#e10098",
 	ForegroundColor: "#ffffff",
 	FontSize:        12,
@@ -111,12 +99,12 @@ var graphQL2Protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	http10protocol.ToString():   &http10protocol,
-	http11protocol.ToString():   &http11protocol,
-	http2Protocol.ToString():    &http2Protocol,
-	grpcProtocol.ToString():     &grpcProtocol,
-	graphQL1Protocol.ToString(): &graphQL1Protocol,
-	graphQL2Protocol.ToString(): &graphQL2Protocol,
+	fmt.Sprintf("%s/%s/%s", http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation):       &http10protocol,
+	fmt.Sprintf("%s/%s/%s", http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation):       &http11protocol,
+	fmt.Sprintf("%s/%s/%s", http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation):          &http2Protocol,
+	fmt.Sprintf("%s/%s/%s", grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation):             &grpcProtocol,
+	fmt.Sprintf("%s/%s/%s", graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation): &graphQL1Protocol,
+	fmt.Sprintf("%s/%s/%s", graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation): &graphQL2Protocol,
 }
 
 const (
@@ -286,13 +274,19 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	reqDetails["pathSegments"] = strings.Split(path, "/")[1:]
 
 	// Rearrange the maps for the querying
-	reqDetails["headers"] = mapSliceRebuildAsMergedMap(reqDetails["headers"].([]interface{}))
-	resDetails["headers"] = mapSliceRebuildAsMergedMap(resDetails["headers"].([]interface{}))
+	reqDetails["_headers"] = reqDetails["headers"]
+	reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{}))
+	resDetails["_headers"] = resDetails["headers"]
+	resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{}))
 
-	reqDetails["cookies"] = mapSliceRebuildAsMergedMap(reqDetails["cookies"].([]interface{}))
-	resDetails["cookies"] = mapSliceRebuildAsMergedMap(resDetails["cookies"].([]interface{}))
+	reqDetails["_cookies"] = reqDetails["cookies"]
+	reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{}))
+	resDetails["_cookies"] = resDetails["cookies"]
+	resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{}))
 
-	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["queryString"].([]interface{}))
+	reqDetails["_queryString"] = reqDetails["queryString"]
+	reqDetails["_queryStringMerged"] = mapSliceMergeRepeatedKeys(reqDetails["_queryString"].([]interface{}))
+	reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryStringMerged"].([]interface{}))
 
 	elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
 	if elapsedTime < 0 {
@@ -300,8 +294,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 
 	return &api.Entry{
-		Protocol: item.Protocol.ProtocolSummary,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", item.Protocol.Name, item.Protocol.Version, item.Protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -334,7 +328,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.Protocol.ToString()],
+		Protocol:     *protocolsMap[entry.ProtocolId],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -391,19 +385,19 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Headers",
-		Data:  representMapAsTable(request["headers"].(map[string]interface{}), `request.headers`),
+		Data:  representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`),
 	})
 
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Cookies",
-		Data:  representMapAsTable(request["cookies"].(map[string]interface{}), `request.cookies`),
+		Data:  representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`),
 	})
 
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Query String",
-		Data:  representMapAsTable(request["queryString"].(map[string]interface{}), `request.queryString`),
+		Data:  representMapSliceAsTable(request["_queryStringMerged"].([]interface{}), `request.queryString`),
 	})
 
 	postData, _ := request["postData"].(map[string]interface{})
@@ -479,13 +473,13 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 	repResponse = append(repResponse, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Headers",
-		Data:  representMapAsTable(response["headers"].(map[string]interface{}), `response.headers`),
+		Data:  representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`),
 	})
 
 	repResponse = append(repResponse, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Cookies",
-		Data:  representMapAsTable(response["cookies"].(map[string]interface{}), `response.cookies`),
+		Data:  representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`),
 	})
 
 	content, _ := response["content"].(map[string]interface{})
@@ -521,10 +515,10 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`http`:  fmt.Sprintf(`protocol.abbr == "%s"`, http11protocol.Abbreviation),
-		`http2`: fmt.Sprintf(`protocol.abbr == "%s"`, http2Protocol.Abbreviation),
-		`grpc`:  fmt.Sprintf(`protocol.abbr == "%s"`, grpcProtocol.Abbreviation),
-		`gql`:   fmt.Sprintf(`protocol.abbr == "%s"`, graphQL1Protocol.Abbreviation),
+		`http`:  fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation, http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation),
+		`http2`: fmt.Sprintf(`protocol == "%s/%s/%s"`, http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation),
+		`grpc`:  fmt.Sprintf(`protocol == "%s/%s/%s"`, grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation),
+		`gql`:   fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation, graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation),
 	}
 }
 

tap/extensions/http/main_test.go

@@ -44,10 +44,10 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"http":  `protocol.abbr == "HTTP"`,
-		"http2": `protocol.abbr == "HTTP/2"`,
-		"grpc":  `protocol.abbr == "gRPC"`,
-		"gql":   `protocol.abbr == "GQL"`,
+		"http":  `protocol == "http/1.0/HTTP" or protocol == "http/1.1/HTTP"`,
+		"http2": `protocol == "http/2.0/HTTP/2"`,
+		"grpc":  `protocol == "http/2.0/gRPC"`,
+		"gql":   `protocol == "http/1.1/GQL" or protocol == "http/2.0/GQL"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/http/sensitive_data_cleaner.go

@@ -1,14 +1,30 @@
 package http
 
 import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 
+	"github.com/beevik/etree"
 	"github.com/up9inc/mizu/tap/api"
 )
 
+const maskedFieldPlaceholderValue = "[REDACTED]"
 const userAgent = "user-agent"
 
+//these values MUST be all lower case and contain no `-` or `_` characters
+var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
+	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
+	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
+	"zip", "zipcode", "address", "country", "firstname", "lastname",
+	"middlename", "fname", "lname", "birthdate"}
+
 func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool {
 	if item.Protocol.Name != "http" {
 		return false
@@ -32,3 +48,192 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 
 	return false
 }
+
+func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
+	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
+	response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)
+
+	filterHeaders(&request.Header)
+	filterHeaders(&response.Header)
+	filterUrl(request.URL)
+	filterRequestBody(request, options)
+	filterResponseBody(response, options)
+}
+
+func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) {
+	contenType := getContentTypeHeaderValue(request.Header)
+	body, err := ioutil.ReadAll(request.Body)
+	if err != nil {
+		return
+	}
+	filteredBody, err := filterHttpBody(body, contenType, options)
+	if err == nil {
+		request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
+	} else {
+		request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
+	}
+}
+
+func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) {
+	contentType := getContentTypeHeaderValue(response.Header)
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return
+	}
+	filteredBody, err := filterHttpBody(body, contentType, options)
+	if err == nil {
+		response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
+	} else {
+		response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
+	}
+}
+
+func filterHeaders(headers *http.Header) {
+	for key := range *headers {
+		if strings.ToLower(key) == userAgent {
+			continue
+		}
+
+		if strings.ToLower(key) == "cookie" {
+			headers.Del(key)
+		} else if isFieldNameSensitive(key) {
+			headers.Set(key, maskedFieldPlaceholderValue)
+		}
+	}
+}
+
+func getContentTypeHeaderValue(headers http.Header) string {
+	for key := range headers {
+		if strings.ToLower(key) == "content-type" {
+			return headers.Get(key)
+		}
+	}
+	return ""
+}
+
+func isFieldNameSensitive(fieldName string) bool {
+	if fieldName == ":authority" {
+		return false
+	}
+
+	name := strings.ToLower(fieldName)
+	name = strings.ReplaceAll(name, "_", "")
+	name = strings.ReplaceAll(name, "-", "")
+	name = strings.ReplaceAll(name, " ", "")
+
+	for _, sensitiveField := range personallyIdentifiableDataFields {
+		if strings.Contains(name, sensitiveField) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) {
+	mimeType := strings.Split(contentType, ";")[0]
+	switch strings.ToLower(mimeType) {
+	case "application/json":
+		return filterJsonBody(bytes)
+	case "text/html":
+		fallthrough
+	case "application/xhtml+xml":
+		fallthrough
+	case "text/xml":
+		fallthrough
+	case "application/xml":
+		return filterXmlEtree(bytes)
+	case "text/plain":
+		if options != nil && options.PlainTextMaskingRegexes != nil {
+			return filterPlainText(bytes, options), nil
+		}
+	}
+	return bytes, nil
+}
+
+func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte {
+	for _, regex := range options.PlainTextMaskingRegexes {
+		bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
+	}
+	return bytes
+}
+
+func filterXmlEtree(bytes []byte) ([]byte, error) {
+	if !IsValidXML(bytes) {
+		return nil, errors.New("Invalid XML")
+	}
+	xmlDoc := etree.NewDocument()
+	err := xmlDoc.ReadFromBytes(bytes)
+	if err != nil {
+		return nil, err
+	} else {
+		filterXmlElement(xmlDoc.Root())
+	}
+	return xmlDoc.WriteToBytes()
+}
+
+func IsValidXML(data []byte) bool {
+	return xml.Unmarshal(data, new(interface{})) == nil
+}
+
+func filterXmlElement(element *etree.Element) {
+	for i, attribute := range element.Attr {
+		if isFieldNameSensitive(attribute.Key) {
+			element.Attr[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+	if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
+		if isFieldNameSensitive(element.Tag) {
+			element.SetText(maskedFieldPlaceholderValue)
+		}
+	} else {
+		for _, element := range element.ChildElements() {
+			filterXmlElement(element)
+		}
+	}
+}
+
+func filterJsonBody(bytes []byte) ([]byte, error) {
+	var bodyJsonMap map[string]interface{}
+	err := json.Unmarshal(bytes, &bodyJsonMap)
+	if err != nil {
+		return nil, err
+	}
+	filterJsonMap(bodyJsonMap)
+	return json.Marshal(bodyJsonMap)
+}
+
+func filterJsonMap(jsonMap map[string]interface{}) {
+	for key, value := range jsonMap {
+		// Do not replace nil values with maskedFieldPlaceholderValue
+		if value == nil {
+			continue
+		}
+
+		nestedMap, isNested := value.(map[string]interface{})
+		if isNested {
+			filterJsonMap(nestedMap)
+		} else {
+			if isFieldNameSensitive(key) {
+				jsonMap[key] = maskedFieldPlaceholderValue
+			}
+		}
+	}
+}
+
+func filterUrl(url *url.URL) {
+	if len(url.RawQuery) > 0 {
+		newQueryArgs := make([]string, 0)
+		for urlQueryParamName, urlQueryParamValues := range url.Query() {
+			newValues := urlQueryParamValues
+			if isFieldNameSensitive(urlQueryParamName) {
+				newValues = []string{maskedFieldPlaceholderValue}
+			}
+			for _, paramValue := range newValues {
+				newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
+			}
+		}
+
+		url.RawQuery = strings.Join(newQueryArgs, "&")
+	}
+}

tap/extensions/kafka/Makefile

@@ -8,7 +8,9 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/kafka bin
+	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/kafka/\*.bin bin
 
 test-pull-expect:
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/kafka expect
+	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/kafka/\* expect

tap/extensions/kafka/main.go

@@ -11,13 +11,11 @@ import (
 )
 
 var _protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "kafka",
-		Version:      "12",
-		Abbreviation: "KAFKA",
-	},
+	Name:            "kafka",
 	LongName:        "Apache Kafka Protocol",
+	Abbreviation:    "KAFKA",
 	Macro:           "kafka",
+	Version:         "12",
 	BackgroundColor: "#000000",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -27,7 +25,7 @@ var _protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	_protocol.ToString(): &_protocol,
+	fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation): &_protocol,
 }
 
 type dissecting string
@@ -72,8 +70,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		Protocol: _protocol.ProtocolSummary,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -197,7 +195,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.Protocol.ToString()],
+		Protocol:     *protocolsMap[entry.ProtocolId],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -252,7 +250,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`kafka`: fmt.Sprintf(`protocol.name == "%s"`, _protocol.Name),
+		`kafka`: fmt.Sprintf(`protocol == "%s/%s/%s"`, _protocol.Name, _protocol.Version, _protocol.Abbreviation),
 	}
 }
 

tap/extensions/kafka/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"kafka": `protocol.name == "kafka"`,
+		"kafka": `protocol == "kafka/12/KAFKA"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/redis/Makefile

@@ -8,7 +8,9 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/redis bin
+	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/redis/\*.bin bin
 
 test-pull-expect:
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/redis expect
+	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/redis/\* expect

tap/extensions/redis/main.go

@@ -11,13 +11,11 @@ import (
 )
 
 var protocol = api.Protocol{
-	ProtocolSummary: api.ProtocolSummary{
-		Name:         "redis",
-		Version:      "3.x",
-		Abbreviation: "REDIS",
-	},
+	Name:            "redis",
 	LongName:        "Redis Serialization Protocol",
+	Abbreviation:    "REDIS",
 	Macro:           "redis",
+	Version:         "3.x",
 	BackgroundColor: "#a41e11",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
@@ -27,7 +25,7 @@ var protocol = api.Protocol{
 }
 
 var protocolsMap = map[string]*api.Protocol{
-	protocol.ToString(): &protocol,
+	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
 }
 
 type dissecting string
@@ -80,8 +78,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		Protocol: protocol.ProtocolSummary,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -125,7 +123,7 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 
 	return &api.BaseEntry{
 		Id:           entry.Id,
-		Protocol:     *protocolsMap[entry.Protocol.ToString()],
+		Protocol:     *protocolsMap[entry.ProtocolId],
 		Capture:      entry.Capture,
 		Summary:      summary,
 		SummaryQuery: summaryQuery,
@@ -153,7 +151,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`redis`: fmt.Sprintf(`protocol.name == "%s"`, protocol.Name),
+		`redis`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
 	}
 }
 

tap/extensions/redis/main_test.go

@@ -45,7 +45,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"redis": `protocol.name == "redis"`,
+		"redis": `protocol == "redis/3.x/REDIS"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/go.mod

@@ -4,12 +4,11 @@ go 1.17
 
 require (
 	github.com/Masterminds/semver v1.5.0
-	github.com/cilium/ebpf v0.9.0
+	github.com/cilium/ebpf v0.8.1
 	github.com/go-errors/errors v1.4.2
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9
-	github.com/moby/moby v20.10.17+incompatible
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/struCoder/pidusage v0.2.1
 	github.com/up9inc/mizu/logger v0.0.0
@@ -29,7 +28,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
@@ -38,7 +36,6 @@ require (
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gotest.tools/v3 v3.3.0 // indirect
 	k8s.io/apimachinery v0.23.3 // indirect
 	k8s.io/klog/v2 v2.40.1 // indirect
 	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect

tap/go.sum

@@ -7,8 +7,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cilium/ebpf v0.9.0 h1:ldiV+FscPCQ/p3mNEV4O02EPbUZJFsoEtHvIr9xLTvk=
-github.com/cilium/ebpf v0.9.0/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY=
+github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao=
+github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -94,8 +94,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/moby v20.10.17+incompatible h1:TJJfyk2fLEgK+RzqVpFNkDkm0oEi+MLUfwt9lEYnp5g=
-github.com/moby/moby v20.10.17+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -125,15 +123,11 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
 github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
@@ -192,14 +186,12 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -229,7 +221,6 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -278,8 +269,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
-gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/api v0.23.3 h1:KNrME8KHGr12Ozjf8ytOewKzZh6hl/hHUZeHddT3a38=

tap/tlstapper/bpf-builder/build.sh

@@ -9,7 +9,7 @@ docker build -t mizu-ebpf-builder . || exit 1
 BPF_TARGET=amd64
 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
 ARCH=$(uname -m)
-if [[ $ARCH == "aarch64" || $ARCH == "arm64" ]]; then
+if [[ $ARCH == "aarch64" ]]; then
     BPF_TARGET=arm64
     BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
 fi
@@ -18,10 +18,10 @@ docker run --rm \
 	--name mizu-ebpf-builder \
 	-v $MIZU_HOME:/mizu \
 	-v $(go env GOPATH):/root/go \
-	mizu-ebpf-builder \
+	-it mizu-ebpf-builder \
 	sh -c "
 		BPF_TARGET=\"$BPF_TARGET\" BPF_CFLAGS=\"$BPF_CFLAGS\" go generate tap/tlstapper/tls_tapper.go
-        chown $(id -u):$(id -g) tap/tlstapper/tlstapper*_bpf*
+		chown $(id -u):$(id -g) tap/tlstapper/tlstapper_bpf*
 	" || exit 1
 
 popd

tap/tlstapper/bpf/common.c

@@ -12,20 +12,23 @@ Copyright (C) UP9 Inc.
 #include "include/common.h"
 
 
-static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd, struct ssl_info* info) {
+static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd) {
     __u32 pid = id >> 32;
     __u64 key = (__u64) pid << 32 | fd;
 
-    conn_flags *flags = bpf_map_lookup_elem(&connection_context, &key);
+    struct fd_info *fdinfo = bpf_map_lookup_elem(&file_descriptor_to_ipv4, &key);
 
-    // Happens when we don't catch the connect / accept (if the connection is created before tapping is started)
-    if (flags == NULL) {
+    if (fdinfo == NULL) {
         return 0;
     }
 
-    chunk->flags |= (*flags & FLAGS_IS_CLIENT_BIT);
+    int err = bpf_probe_read(chunk->address, sizeof(chunk->address), fdinfo->ipv4_addr);
+    chunk->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT);
 
-    bpf_probe_read(&chunk->address_info, sizeof(chunk->address_info), &info->address_info);
+    if (err != 0) {
+        log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
+        return 0;
+    }
 
     return 1;
 }
@@ -101,7 +104,7 @@ static __always_inline void output_ssl_chunk(struct pt_regs *ctx, struct ssl_inf
     chunk->len = count_bytes;
     chunk->fd = info->fd;
 
-    if (!add_address_to_chunk(ctx, chunk, id, chunk->fd, info)) {
+    if (!add_address_to_chunk(ctx, chunk, id, chunk->fd)) {
         // Without an address, we drop the chunk because there is not much to do with it in Go
         //
         return;

tap/tlstapper/bpf/fd_to_address_tracepoints.c

@@ -14,6 +14,7 @@ Copyright (C) UP9 Inc.
 #define IPV4_ADDR_LEN (16)
 
 struct accept_info {
+	__u64* sockaddr;
 	__u32* addrlen;
 };
 
@@ -38,6 +39,7 @@ void sys_enter_accept4(struct sys_enter_accept4_ctx *ctx) {
 	
 	struct accept_info info = {};
 	
+	info.sockaddr = ctx->sockaddr;
 	info.addrlen = ctx->addrlen;
 	
 	long err = bpf_map_update_elem(&accept_syscall_context, &id, &info, BPF_ANY);
@@ -92,21 +94,26 @@ void sys_exit_accept4(struct sys_exit_accept4_ctx *ctx) {
 		return;
 	}
 	
-	conn_flags flags = 0;
+	struct fd_info fdinfo = {
+		.flags = 0
+	};
+	
+	bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr);
 	
 	__u32 pid = id >> 32;
 	__u32 fd = (__u32) ctx->ret;
 	
 	__u64 key = (__u64) pid << 32 | fd;
-	err = bpf_map_update_elem(&connection_context, &key, &flags, BPF_ANY);
+	err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY);
 	
 	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_CONNECTION_CONTEXT, id, err, ORIGIN_SYS_EXIT_ACCEPT4_CODE);
+		log_error(ctx, LOG_ERROR_PUTTING_FD_MAPPING, id, err, ORIGIN_SYS_EXIT_ACCEPT4_CODE);
 	}
 }
 
 struct connect_info {
 	__u64 fd;
+	__u64* sockaddr;
 	__u32 addrlen;
 };
 
@@ -131,6 +138,7 @@ void sys_enter_connect(struct sys_enter_connect_ctx *ctx) {
 	
 	struct connect_info info = {};
 	
+	info.sockaddr = ctx->sockaddr;
 	info.addrlen = ctx->addrlen;
 	info.fd = ctx->fd;
 	
@@ -185,15 +193,19 @@ void sys_exit_connect(struct sys_exit_connect_ctx *ctx) {
 		return;
 	}
 	
-	conn_flags flags = FLAGS_IS_CLIENT_BIT;
+	struct fd_info fdinfo = {
+		.flags = FLAGS_IS_CLIENT_BIT
+	};
+	
+	bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr);
 	
 	__u32 pid = id >> 32;
 	__u32 fd = (__u32) info.fd;
 	
 	__u64 key = (__u64) pid << 32 | fd;
-	err = bpf_map_update_elem(&connection_context, &key, &flags, BPF_ANY);
+	err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY);
 	
 	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_CONNECTION_CONTEXT, id, err, ORIGIN_SYS_EXIT_CONNECT_CODE);
+		log_error(ctx, LOG_ERROR_PUTTING_FD_MAPPING, id, err, ORIGIN_SYS_EXIT_CONNECT_CODE);
 	}
 }

tap/tlstapper/bpf/fd_tracepoints.c

@@ -10,9 +10,8 @@ Copyright (C) UP9 Inc.
 #include "include/log.h"
 #include "include/logger_messages.h"
 #include "include/pids.h"
-#include "include/common.h"
 
-struct sys_enter_read_write_ctx {
+struct sys_enter_read_ctx {
 	__u64 __unused_syscall_header;
 	__u32 __unused_syscall_nr;
 	
@@ -21,46 +20,8 @@ struct sys_enter_read_write_ctx {
 	__u64 count;
 };
 
-struct sys_exit_read_write_ctx {
-	__u64 __unused_syscall_header;
-	__u32 __unused_syscall_nr;
-
-	__u64 ret;
-};
-
-
-static __always_inline void fd_tracepoints_handle_openssl(struct sys_enter_read_write_ctx *ctx, __u64 id, struct ssl_info *infoPtr, struct bpf_map_def *map_fd, __u64 origin_code) {
-	struct ssl_info info;
-	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, origin_code);
-		return;
-	}
-	
-	info.fd = ctx->fd;
-	
-	err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, origin_code);
-		return;
-	}
-}
-
-static __always_inline void fd_tracepoints_handle_go(struct sys_enter_read_write_ctx *ctx, __u64 id, struct bpf_map_def *map_fd, __u64 origin_code) {
-	__u32 fd = ctx->fd;
-
-	long err = bpf_map_update_elem(map_fd, &id, &fd, BPF_ANY);
-	
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, origin_code);
-		return;
-	}
-}
-
 SEC("tracepoint/syscalls/sys_enter_read")
-void sys_enter_read(struct sys_enter_read_write_ctx *ctx) {
+void sys_enter_read(struct sys_enter_read_ctx *ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -69,15 +30,38 @@ void sys_enter_read(struct sys_enter_read_write_ctx *ctx) {
 	
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_read_context, &id);
 	
-	if (infoPtr != NULL) {
-		fd_tracepoints_handle_openssl(ctx, id, infoPtr, &openssl_read_context, ORIGIN_SYS_ENTER_READ_CODE);
+	if (infoPtr == NULL) {
+		return;
 	}
-
-	fd_tracepoints_handle_go(ctx, id, &go_kernel_read_context, ORIGIN_SYS_ENTER_READ_CODE);
-}
 	
+	struct ssl_info info;
+	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, ORIGIN_SYS_ENTER_READ_CODE);
+		return;
+	}
+	
+	info.fd = ctx->fd;
+	
+	err = bpf_map_update_elem(&openssl_read_context, &id, &info, BPF_ANY);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_READ_CODE);
+	}
+}
+
+struct sys_enter_write_ctx {
+	__u64 __unused_syscall_header;
+	__u32 __unused_syscall_nr;
+	
+	__u64 fd;
+	__u64* buf;
+	__u64 count;
+};
+
 SEC("tracepoint/syscalls/sys_enter_write")
-void sys_enter_write(struct sys_enter_read_write_ctx *ctx) {
+void sys_enter_write(struct sys_enter_write_ctx *ctx) {
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -86,25 +70,23 @@ void sys_enter_write(struct sys_enter_read_write_ctx *ctx) {
 	
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(&openssl_write_context, &id);
 	
-	if (infoPtr != NULL) {
-		fd_tracepoints_handle_openssl(ctx, id, infoPtr, &openssl_write_context, ORIGIN_SYS_ENTER_WRITE_CODE);
+	if (infoPtr == NULL) {
+		return;
+	}
+	
+	struct ssl_info info;
+	long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_READING_SSL_CONTEXT, id, err, ORIGIN_SYS_ENTER_WRITE_CODE);
+		return;
+	}
+	
+	info.fd = ctx->fd;
+	
+	err = bpf_map_update_elem(&openssl_write_context, &id, &info, BPF_ANY);
+	
+	if (err != 0) {
+		log_error(ctx, LOG_ERROR_PUTTING_FILE_DESCRIPTOR, id, err, ORIGIN_SYS_ENTER_WRITE_CODE);
 	}
-
-	fd_tracepoints_handle_go(ctx, id, &go_kernel_write_context, ORIGIN_SYS_ENTER_WRITE_CODE);
-}
-
-SEC("tracepoint/syscalls/sys_exit_read")
-void sys_exit_read(struct sys_exit_read_write_ctx *ctx) {
-	__u64 id = bpf_get_current_pid_tgid();
-	// Delete from go map. The value is not used after exiting this syscall.
-	// Keep value in openssl map.
-	bpf_map_delete_elem(&go_kernel_read_context, &id);
-}
-
-SEC("tracepoint/syscalls/sys_exit_write")
-void sys_exit_write(struct sys_exit_read_write_ctx *ctx) {
-	__u64 id = bpf_get_current_pid_tgid();
-	// Delete from go map. The value is not used after exiting this syscall.
-	// Keep value in openssl map.
-	bpf_map_delete_elem(&go_kernel_write_context, &id);
 }

tap/tlstapper/bpf/go_uprobes.c

@@ -35,12 +35,11 @@ using `bpf_probe_read` calls in `go_crypto_tls_get_fd_from_tcp_conn` function.
 
 SOURCES:
 
-Tracing Go Functions with eBPF (<=1.16): https://www.grant.pizza/blog/tracing-go-functions-with-ebpf-part-2/
+Tracing Go Functions with eBPF (before 1.17): https://www.grant.pizza/blog/tracing-go-functions-with-ebpf-part-2/
 Challenges of BPF Tracing Go: https://blog.0x74696d.com/posts/challenges-of-bpf-tracing-go/
 x86 calling conventions: https://en.wikipedia.org/wiki/X86_calling_conventions
 Plan 9 from Bell Labs: https://en.wikipedia.org/wiki/Plan_9_from_Bell_Labs
 The issue for calling convention change in Go: https://github.com/golang/go/issues/40724
-Go ABI0 (<=1.16) specification: https://go.dev/doc/asm
 Proposal of Register-based Go calling convention: https://go.googlesource.com/proposal/+/master/design/40724-register-calling.md
 Go internal ABI (1.17) specification: https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-abi.md
 Go internal ABI (current) specification: https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/abi-internal.md
@@ -56,60 +55,10 @@ Capstone Engine: https://www.capstone-engine.org/
 #include "include/logger_messages.h"
 #include "include/pids.h"
 #include "include/common.h"
-#include "include/go_abi_0.h"
 #include "include/go_abi_internal.h"
 #include "include/go_types.h"
 
-
-// TODO: cilium/ebpf does not support .kconfig Therefore; for now, we build object files per kernel version.
-// Error: reference to .kconfig: not supported
-// See: https://github.com/cilium/ebpf/issues/698
-// extern int LINUX_KERNEL_VERSION __kconfig;
-
-enum ABI {
-    ABI0=0,
-    ABIInternal=1,
-};
-
-#if defined(bpf_target_x86)
-// get_goid_from_thread_local_storage function is x86 specific
-static __always_inline __u32 get_goid_from_thread_local_storage(__u64 *goroutine_id) {
-    int zero = 0;
-    int one = 1;
-    struct goid_offsets* offsets = bpf_map_lookup_elem(&goid_offsets_map, &zero);
-    if (offsets == NULL) {
-        return 0;
-    }
-
-    // Get the task that currently assigned to this thread.
-    struct task_struct *task = (struct task_struct*) bpf_get_current_task();
-    if (task == NULL) {
-        return 0;
-    }
-
-    // Read task->thread
-    struct thread_struct *thr;
-    bpf_probe_read(&thr, sizeof(thr), &task->thread);
-
-    // Read task->thread.fsbase
-    u64 fsbase;
-#ifdef KERNEL_BEFORE_4_6
-    // TODO: if (LINUX_KERNEL_VERSION <= KERNEL_VERSION(4, 6, 0)) {
-    fsbase = BPF_CORE_READ((struct thread_struct___v46 *)thr, fs);
-#else
-    fsbase = BPF_CORE_READ(thr, fsbase);
-#endif
-
-    // Get the Goroutine ID (goid) which is stored in thread-local storage.
-    size_t g_addr;
-    bpf_probe_read_user(&g_addr, sizeof(void *), (void*)(fsbase + offsets->g_addr_offset));
-    bpf_probe_read_user(goroutine_id, sizeof(void *), (void*)(g_addr + offsets->goid_offset));
-
-    return 1;
-}
-#endif
-
-static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *ctx, enum ABI abi) {
+static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *ctx) {
     struct go_interface conn;
     long err;
     __u64 addr;
@@ -118,15 +67,8 @@ static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *
     if (err != 0) {
         return invalid_fd;
     }
-#elif defined(bpf_target_x86)
-    if (abi == ABI0) {
-        err = bpf_probe_read(&addr, sizeof(addr), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx)+0x8);
-        if (err != 0) {
-            return invalid_fd;
-        }
-    } else {
-        addr = GO_ABI_INTERNAL_PT_REGS_R1(ctx);
-    }
+#else
+    addr = GO_ABI_INTERNAL_PT_REGS_R1(ctx);
 #endif
 
     err = bpf_probe_read(&conn, sizeof(conn), (void*)addr);
@@ -149,7 +91,7 @@ static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *
     return fd;
 }
 
-static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, enum ABI abi) {
+static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context) {
     __u64 pid_tgid = bpf_get_current_pid_tgid();
     __u64 pid = pid_tgid >> 32;
     if (!should_tap(pid)) {
@@ -165,52 +107,14 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf
         log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
         return;
     }
-#elif defined(bpf_target_x86)
-    if (abi == ABI0) {
-        err = bpf_probe_read(&info.buffer_len, sizeof(__u32), (void*)GO_ABI_0_PT_REGS_SP(ctx)+0x18);
-        if (err != 0) {
-            log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
-            return;
-        }
-    } else {
-        info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx);
-    }
+#else
+    info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx);
 #endif
+    info.buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx);
+    info.fd = go_crypto_tls_get_fd_from_tcp_conn(ctx);
 
-#if defined(bpf_target_x86)
-    if (abi == ABI0) {
-        err = bpf_probe_read(&info.buffer, sizeof(__u32), (void*)GO_ABI_0_PT_REGS_SP(ctx)+0x11);
-        if (err != 0) {
-            log_error(ctx, LOG_ERROR_READING_FROM_SSL_BUFFER, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
-            return;
-        }
-        // We basically add 00 suffix to the hex address.
-        info.buffer = (void*)((long)info.buffer << 8);
-    } else {
-#endif
-        info.buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx);
-#if defined(bpf_target_x86)
-    }
-#endif
-    info.fd = go_crypto_tls_get_fd_from_tcp_conn(ctx, abi);
-
-    __u64 goroutine_id;
-    if (abi == ABI0) {
-#if defined(bpf_target_arm64)
-        // In case of ABI0 and arm64, it's stored in the Goroutine register
-        goroutine_id = GO_ABI_0_PT_REGS_GP(ctx);
-#elif defined(bpf_target_x86)
-        // In case of ABI0 and amd64, it's stored in the thread-local storage
-        int status = get_goid_from_thread_local_storage(&goroutine_id);
-        if (!status) {
-            return;
-        }
-#endif
-    } else {
-        // GO_ABI_INTERNAL_PT_REGS_GP is the Goroutine address in ABIInternal
-        goroutine_id = GO_ABI_INTERNAL_PT_REGS_GP(ctx);
-    }
-    __u64 pid_fp = pid << 32 | goroutine_id;
+    // GO_ABI_INTERNAL_PT_REGS_GP is Goroutine address
+    __u64 pid_fp = pid << 32 | GO_ABI_INTERNAL_PT_REGS_GP(ctx);
     err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY);
 
     if (err != 0) {
@@ -220,30 +124,15 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf
     return;
 }
 
-static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, struct bpf_map_def* go_user_kernel_context, __u32 flags, enum ABI abi) {
+static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct bpf_map_def* go_context, __u32 flags) {
     __u64 pid_tgid = bpf_get_current_pid_tgid();
     __u64 pid = pid_tgid >> 32;
     if (!should_tap(pid)) {
         return;
     }
 
-    __u64 goroutine_id;
-    if (abi == ABI0) {
-#if defined(bpf_target_arm64)
-        // In case of ABI0 and arm64, it's stored in the Goroutine register
-        goroutine_id = GO_ABI_0_PT_REGS_GP(ctx);
-#elif defined(bpf_target_x86)
-        // In case of ABI0 and amd64, it's stored in the thread-local storage
-        int status = get_goid_from_thread_local_storage(&goroutine_id);
-        if (!status) {
-            return;
-        }
-#endif
-    } else {
-        // GO_ABI_INTERNAL_PT_REGS_GP is the Goroutine address in ABIInternal
-        goroutine_id = GO_ABI_INTERNAL_PT_REGS_GP(ctx);
-    }
-    __u64 pid_fp = pid << 32 | goroutine_id;
+    // GO_ABI_INTERNAL_PT_REGS_GP is Goroutine address
+    __u64 pid_fp = pid << 32 | GO_ABI_INTERNAL_PT_REGS_GP(ctx);
     struct ssl_info *info_ptr = bpf_map_lookup_elem(go_context, &pid_fp);
 
     if (info_ptr == NULL) {
@@ -267,17 +156,8 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct
             return;
         }
         info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R7(ctx); // n in return n, nil
-#elif defined(bpf_target_x86)
-        if (abi == ABI0) {
-            // n in return n, nil
-            err = bpf_probe_read(&info.buffer_len, sizeof(__u32), (void*)GO_ABI_0_PT_REGS_SP(ctx)+0x28);
-            if (err != 0) {
-                log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
-                return;
-            }
-        } else {
-            info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R1(ctx); // n in return n, nil
-        }
+#else
+        info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R1(ctx); // n in return n, nil
 #endif
         // This check achieves ignoring 0 length reads (the reads result with an error)
         if (info.buffer_len <= 0) {
@@ -285,70 +165,27 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct
         }
     }
 
-    __u64 key = (__u64) pid << 32 | info_ptr->fd;
-    struct address_info *address_info = bpf_map_lookup_elem(go_user_kernel_context, &key);
-    // Ideally we would delete the entry from the map after reading it,
-    // but sometimes the uprobe is called twice in a row without the tcp kprobes in between to fill in
-    // the entry again. Keeping it in the map and rely on LRU logic.
-    if (address_info == NULL) {
-        log_error(ctx, LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT, pid_tgid, info_ptr->fd, err);
-        return;
-    }
-
-    info.address_info.daddr = address_info->daddr;
-    info.address_info.dport = address_info->dport;
-    info.address_info.saddr = address_info->saddr;
-    info.address_info.sport = address_info->sport;
-
     output_ssl_chunk(ctx, &info, info.buffer_len, pid_tgid, flags);
 
     return;
 }
 
-SEC("uprobe/go_crypto_tls_abi0_write")
-int BPF_KPROBE(go_crypto_tls_abi0_write) {
-    go_crypto_tls_uprobe(ctx, &go_write_context, ABI0);
-    return 1;
+SEC("uprobe/go_crypto_tls_write")
+void BPF_KPROBE(go_crypto_tls_write) {
+    go_crypto_tls_uprobe(ctx, &go_write_context);
 }
 
-SEC("uprobe/go_crypto_tls_abi0_write_ex")
-int BPF_KPROBE(go_crypto_tls_abi0_write_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_write_context, &go_user_kernel_write_context, 0, ABI0);
-    return 1;
+SEC("uprobe/go_crypto_tls_write_ex")
+void BPF_KPROBE(go_crypto_tls_write_ex) {
+    go_crypto_tls_ex_uprobe(ctx, &go_write_context, 0);
 }
 
-SEC("uprobe/go_crypto_tls_abi0_read")
-int BPF_KPROBE(go_crypto_tls_abi0_read) {
-    go_crypto_tls_uprobe(ctx, &go_read_context, ABI0);
-    return 1;
+SEC("uprobe/go_crypto_tls_read")
+void BPF_KPROBE(go_crypto_tls_read) {
+    go_crypto_tls_uprobe(ctx, &go_read_context);
 }
 
-SEC("uprobe/go_crypto_tls_abi0_read_ex")
-int BPF_KPROBE(go_crypto_tls_abi0_read_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_read_context, &go_user_kernel_read_context, FLAGS_IS_READ_BIT, ABI0);
-    return 1;
-}
-
-SEC("uprobe/go_crypto_tls_abi_internal_write")
-int BPF_KPROBE(go_crypto_tls_abi_internal_write) {
-    go_crypto_tls_uprobe(ctx, &go_write_context, ABIInternal);
-    return 1;
-}
-
-SEC("uprobe/go_crypto_tls_abi_internal_write_ex")
-int BPF_KPROBE(go_crypto_tls_abi_internal_write_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_write_context, &go_user_kernel_write_context, 0, ABIInternal);
-    return 1;
-}
-
-SEC("uprobe/go_crypto_tls_abi_internal_read")
-int BPF_KPROBE(go_crypto_tls_abi_internal_read) {
-    go_crypto_tls_uprobe(ctx, &go_read_context, ABIInternal);
-    return 1;
-}
-
-SEC("uprobe/go_crypto_tls_abi_internal_read_ex")
-int BPF_KPROBE(go_crypto_tls_abi_internal_read_ex) {
-    go_crypto_tls_ex_uprobe(ctx, &go_read_context, &go_user_kernel_read_context, FLAGS_IS_READ_BIT, ABIInternal);
-    return 1;
+SEC("uprobe/go_crypto_tls_read_ex")
+void BPF_KPROBE(go_crypto_tls_read_ex) {
+    go_crypto_tls_ex_uprobe(ctx, &go_read_context, FLAGS_IS_READ_BIT);
 }

tap/tlstapper/bpf/include/common.h

@@ -7,11 +7,9 @@ Copyright (C) UP9 Inc.
 #ifndef __COMMON__
 #define __COMMON__
 
-#define AF_INET	2	/* Internet IP Protocol */
-
 const __s32 invalid_fd = -1;
 
-static int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd, struct ssl_info* info);
+static int add_address_to_chunk(struct pt_regs *ctx, struct tls_chunk* chunk, __u64 id, __u32 fd);
 static void send_chunk_part(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk, int start, int end);
 static void send_chunk(struct pt_regs *ctx, __u8* buffer, __u64 id, struct tls_chunk* chunk);
 static void output_ssl_chunk(struct pt_regs *ctx, struct ssl_info* info, int count_bytes, __u64 id, __u32 flags);

tap/tlstapper/bpf/include/go_abi_0.h

@@ -1,52 +0,0 @@
-/*
-Note: This file is licenced differently from the rest of the project
-SPDX-License-Identifier: GPL-2.0
-Copyright (C) UP9 Inc.
-*/
-
-#ifndef __GO_ABI_0__
-#define __GO_ABI_0__
-
-/*
-Go ABI0 (<=1.16) specification
-https://go.dev/doc/asm
-
-Since ABI0 is a stack-based calling convention we only need the stack pointer and
-if it's applicable the Goroutine pointer
-*/
-
-#include "target_arch.h"
-
-#if defined(bpf_target_x86)
-
-#ifdef __i386__
-
-#define GO_ABI_0_PT_REGS_SP(x) ((x)->esp)
-
-#else
-
-#define GO_ABI_0_PT_REGS_SP(x) ((x)->sp)
-
-#endif
-
-#elif defined(bpf_target_arm)
-
-#define GO_ABI_0_PT_REGS_SP(x) ((x)->uregs[13])
-#define GO_ABI_0_PT_REGS_GP(x) ((x)->uregs[10])
-
-#elif defined(bpf_target_arm64)
-
-/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */
-struct pt_regs;
-#define PT_REGS_ARM64 const volatile struct user_pt_regs
-#define GO_ABI_0_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp)
-#define GO_ABI_0_PT_REGS_GP(x) (((PT_REGS_ARM64 *)(x))->regs[18])
-
-#elif defined(bpf_target_powerpc)
-
-#define GO_ABI_0_PT_REGS_SP(x) ((x)->sp)
-#define GO_ABI_0_PT_REGS_GP(x) ((x)->gpr[30])
-
-#endif
-
-#endif /* __GO_ABI_0__ */

tap/tlstapper/bpf/include/go_abi_internal.h

@@ -8,11 +8,54 @@ Copyright (C) UP9 Inc.
 #define __GO_ABI_INTERNAL__
 
 /*
-Go internal ABI (1.17/current) specification
+Go internal ABI specification
 https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/abi-internal.md
 */
 
-#include "target_arch.h"
+/* Scan the ARCH passed in from ARCH env variable */
+#if defined(__TARGET_ARCH_x86)
+    #define bpf_target_x86
+    #define bpf_target_defined
+#elif defined(__TARGET_ARCH_s390)
+    #define bpf_target_s390
+    #define bpf_target_defined
+#elif defined(__TARGET_ARCH_arm)
+    #define bpf_target_arm
+    #define bpf_target_defined
+#elif defined(__TARGET_ARCH_arm64)
+    #define bpf_target_arm64
+    #define bpf_target_defined
+#elif defined(__TARGET_ARCH_mips)
+    #define bpf_target_mips
+    #define bpf_target_defined
+#elif defined(__TARGET_ARCH_powerpc)
+    #define bpf_target_powerpc
+    #define bpf_target_defined
+#elif defined(__TARGET_ARCH_sparc)
+    #define bpf_target_sparc
+    #define bpf_target_defined
+#else
+    #undef bpf_target_defined
+#endif
+
+/* Fall back to what the compiler says */
+#ifndef bpf_target_defined
+#if defined(__x86_64__)
+    #define bpf_target_x86
+#elif defined(__s390__)
+    #define bpf_target_s390
+#elif defined(__arm__)
+    #define bpf_target_arm
+#elif defined(__aarch64__)
+    #define bpf_target_arm64
+#elif defined(__mips__)
+    #define bpf_target_mips
+#elif defined(__powerpc__)
+    #define bpf_target_powerpc
+#elif defined(__sparc__)
+    #define bpf_target_sparc
+#endif
+#endif
 
 #if defined(bpf_target_x86)
 
@@ -35,15 +78,15 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD6
 
 #else
 
-#define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->ax)
-#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->cx)
-#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->dx)
-#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->bx)
-#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->bp)
-#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->si)
-#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->di)
-#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->sp)
-#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->bp)
+#define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->rax)
+#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->rcx)
+#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->rdx)
+#define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->rbx)
+#define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->rbp)
+#define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->rsi)
+#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->rdi)
+#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->rsp)
+#define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->rbp)
 #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->r14)
 
 #endif

tap/tlstapper/bpf/include/headers.h

@@ -8,16 +8,9 @@ Copyright (C) UP9 Inc.
 #define __HEADERS__
 
 #include <stddef.h>
-
-#include "target_arch.h"
-#include "vmlinux_x86.h"
-#include "vmlinux_arm64.h"
-
-#include "legacy_kernel.h"
-
-#include <bpf/bpf_endian.h>
+#include <linux/bpf.h>
+#include <linux/ptrace.h>
 #include <bpf/bpf_helpers.h>
-#include <bpf/bpf_tracing.h>
-#include <bpf/bpf_core_read.h>
+#include "bpf/bpf_tracing.h"
 
 #endif /* __HEADERS__ */

tap/tlstapper/bpf/include/legacy_kernel.h

@@ -1,50 +0,0 @@
-#ifndef __LEGACY_KERNEL_H__
-#define __LEGACY_KERNEL_H__
-
-#if defined(bpf_target_x86)
-
-struct thread_struct___v46 {
-	struct desc_struct tls_array[3];
-	unsigned long sp0;
-	unsigned long sp;
-	unsigned short es;
-	unsigned short ds;
-	unsigned short fsindex;
-	unsigned short gsindex;
-	unsigned long fs;
-	unsigned long gs;
-	struct perf_event ptrace_bps[4];
-	unsigned long debugreg6;
-	unsigned long ptrace_dr7;
-	unsigned long cr2;
-	unsigned long trap_nr;
-	unsigned long error_code;
-	unsigned long io_bitmap_ptr;
-	unsigned long iopl;
-	unsigned io_bitmap_max;
-    long: 63;
-	long: 64;
-	long: 64;
-	long: 64;
-	long: 64;
-	long: 64;
-	struct fpu fpu;
-};
-
-#elif defined(bpf_target_arm)
-
-// Commented out since thread_struct is not used in ARM64 yet.
-
-// struct thread_struct___v46 {
-//     struct cpu_context cpu_context;
-//     long: 64;
-// 	unsigned long tp_value;
-// 	struct fpsimd_state fpsimd_state;
-// 	unsigned long fault_address;
-// 	unsigned long fault_code;
-// 	struct debug_info debug;
-// };
-
-#endif
-
-#endif /* __LEGACY_KERNEL_H__ */

tap/tlstapper/bpf/include/logger_messages.h

@@ -10,28 +10,22 @@ Copyright (C) UP9 Inc.
 // Must be synced with bpf_logger_messages.go
 //
 #define LOG_ERROR_READING_BYTES_COUNT (0)
-#define LOG_ERROR_READING_FROM_SSL_BUFFER (1)
-#define LOG_ERROR_BUFFER_TOO_BIG (2)
-#define LOG_ERROR_ALLOCATING_CHUNK (3)
-#define LOG_ERROR_READING_SSL_CONTEXT (4)
-#define LOG_ERROR_PUTTING_SSL_CONTEXT (5)
-#define LOG_ERROR_GETTING_SSL_CONTEXT (6)
-#define LOG_ERROR_MISSING_FILE_DESCRIPTOR (7)
-#define LOG_ERROR_PUTTING_FILE_DESCRIPTOR (8)
-#define LOG_ERROR_PUTTING_ACCEPT_INFO (9)
-#define LOG_ERROR_GETTING_ACCEPT_INFO (10)
-#define LOG_ERROR_READING_ACCEPT_INFO (11)
-#define LOG_ERROR_PUTTING_CONNECTION_CONTEXT (12)
-#define LOG_ERROR_PUTTING_CONNECT_INFO (13)
-#define LOG_ERROR_GETTING_CONNECT_INFO (14)
-#define LOG_ERROR_READING_CONNECT_INFO (15)
-#define LOG_ERROR_READING_SOCKET_FAMILY (16)
-#define LOG_ERROR_READING_SOCKET_DADDR (17)
-#define LOG_ERROR_READING_SOCKET_SADDR (18)
-#define LOG_ERROR_READING_SOCKET_DPORT (19)
-#define LOG_ERROR_READING_SOCKET_SPORT (20)
-#define LOG_ERROR_PUTTING_GO_USER_KERNEL_CONTEXT (21)
-#define LOG_ERROR_GETTING_GO_USER_KERNEL_CONTEXT (22)
+#define LOG_ERROR_READING_FD_ADDRESS (1)
+#define LOG_ERROR_READING_FROM_SSL_BUFFER (2)
+#define LOG_ERROR_BUFFER_TOO_BIG (3)
+#define LOG_ERROR_ALLOCATING_CHUNK (4)
+#define LOG_ERROR_READING_SSL_CONTEXT (5)
+#define LOG_ERROR_PUTTING_SSL_CONTEXT (6)
+#define LOG_ERROR_GETTING_SSL_CONTEXT (7)
+#define LOG_ERROR_MISSING_FILE_DESCRIPTOR (8)
+#define LOG_ERROR_PUTTING_FILE_DESCRIPTOR (9)
+#define LOG_ERROR_PUTTING_ACCEPT_INFO (10)
+#define LOG_ERROR_GETTING_ACCEPT_INFO (11)
+#define LOG_ERROR_READING_ACCEPT_INFO (12)
+#define LOG_ERROR_PUTTING_FD_MAPPING (13)
+#define LOG_ERROR_PUTTING_CONNECT_INFO (14)
+#define LOG_ERROR_GETTING_CONNECT_INFO (15)
+#define LOG_ERROR_READING_CONNECT_INFO (16)
 
 // Sometimes we have the same error, happening from different locations.
 // 	in order to be able to distinct between them in the log, we add an 

tap/tlstapper/bpf/include/maps.h

@@ -24,14 +24,6 @@ Copyright (C) UP9 Inc.
 //  
 //  Be careful when editing, alignment and padding should be exactly the same in go/c.
 //
-
-struct address_info {
-    __be32 saddr;
-    __be32 daddr;
-    __be16 sport;
-    __be16 dport;
-};
-
 struct tls_chunk {
     __u32 pid;
     __u32 tgid;
@@ -40,7 +32,7 @@ struct tls_chunk {
     __u32 recorded;
     __u32 fd;
     __u32 flags;
-    struct address_info address_info;
+    __u8 address[16];
     __u8 data[CHUNK_SIZE]; // Must be N^2
 };
 
@@ -49,7 +41,6 @@ struct ssl_info {
     __u32 buffer_len;
     __u32 fd;
     __u64 created_at_nano;
-    struct address_info address_info;
     
     // for ssl_write and ssl_read must be zero
     // for ssl_write_ex and ssl_read_ex save the *written/*readbytes pointer. 
@@ -57,15 +48,11 @@ struct ssl_info {
     size_t *count_ptr;
 };
 
-typedef __u8 conn_flags;
-
-struct goid_offsets {
-    __u64 g_addr_offset;
-    __u64 goid_offset;
+struct fd_info {
+    __u8 ipv4_addr[16]; // struct sockaddr (linux-src/include/linux/socket.h)
+    __u8 flags;
 };
 
-const struct goid_offsets *unused __attribute__((unused));
-
 // Heap-like area for eBPF programs - stack size limited to 512 bytes, we must use maps for bigger (chunk) objects.
 //
 struct {
@@ -95,7 +82,7 @@ struct {
 
 // Generic
 BPF_HASH(pids_map, __u32, __u32);
-BPF_LRU_HASH(connection_context, __u64, conn_flags);
+BPF_LRU_HASH(file_descriptor_to_ipv4, __u64, struct fd_info);
 BPF_PERF_OUTPUT(chunks_buffer);
 BPF_PERF_OUTPUT(log_buffer);
 
@@ -104,12 +91,7 @@ BPF_LRU_HASH(openssl_write_context, __u64, struct ssl_info);
 BPF_LRU_HASH(openssl_read_context, __u64, struct ssl_info);
 
 // Go specific
-BPF_HASH(goid_offsets_map, __u32, struct goid_offsets);
 BPF_LRU_HASH(go_write_context, __u64, struct ssl_info);
 BPF_LRU_HASH(go_read_context, __u64, struct ssl_info);
-BPF_LRU_HASH(go_kernel_write_context, __u64, __u32);
-BPF_LRU_HASH(go_kernel_read_context, __u64, __u32);
-BPF_LRU_HASH(go_user_kernel_write_context, __u64, struct address_info);
-BPF_LRU_HASH(go_user_kernel_read_context, __u64, struct address_info);
 
 #endif /* __MAPS__ */

tap/tlstapper/bpf/include/target_arch.h

@@ -1,55 +0,0 @@
-/*
-Note: This file is licenced differently from the rest of the project
-SPDX-License-Identifier: GPL-2.0
-Copyright (C) UP9 Inc.
-*/
-
-#ifndef __TARGET_ARCH__
-#define __TARGET_ARCH__
-
-/* Scan the ARCH passed in from ARCH env variable */
-#if defined(__TARGET_ARCH_x86)
-    #define bpf_target_x86
-    #define bpf_target_defined
-#elif defined(__TARGET_ARCH_s390)
-    #define bpf_target_s390
-    #define bpf_target_defined
-#elif defined(__TARGET_ARCH_arm)
-    #define bpf_target_arm
-    #define bpf_target_defined
-#elif defined(__TARGET_ARCH_arm64)
-    #define bpf_target_arm64
-    #define bpf_target_defined
-#elif defined(__TARGET_ARCH_mips)
-    #define bpf_target_mips
-    #define bpf_target_defined
-#elif defined(__TARGET_ARCH_powerpc)
-    #define bpf_target_powerpc
-    #define bpf_target_defined
-#elif defined(__TARGET_ARCH_sparc)
-    #define bpf_target_sparc
-    #define bpf_target_defined
-#else
-    #undef bpf_target_defined
-#endif
-
-/* Fall back to what the compiler says */
-#ifndef bpf_target_defined
-#if defined(__x86_64__)
-    #define bpf_target_x86
-#elif defined(__s390__)
-    #define bpf_target_s390
-#elif defined(__arm__)
-    #define bpf_target_arm
-#elif defined(__aarch64__)
-    #define bpf_target_arm64
-#elif defined(__mips__)
-    #define bpf_target_mips
-#elif defined(__powerpc__)
-    #define bpf_target_powerpc
-#elif defined(__sparc__)
-    #define bpf_target_sparc
-#endif
-#endif
-
-#endif /* __TARGET_ARCH__ */

tap/tlstapper/bpf/openssl_uprobes.c

@@ -42,8 +42,6 @@ static __always_inline int get_count_bytes(struct pt_regs *ctx, struct ssl_info*
 }
 
 static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buffer, int num, struct bpf_map_def* map_fd, size_t *count_ptr) {
-	long err;
-
 	__u64 id = bpf_get_current_pid_tgid();
 	
 	if (!should_tap(id >> 32)) {
@@ -55,7 +53,7 @@ static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buf
 	info.count_ptr = count_ptr;
 	info.buffer = buffer;
 	
-	err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
+	long err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY);
 	
 	if (err != 0) {
 		log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, id, err, 0l);
@@ -68,7 +66,7 @@ static __always_inline void ssl_uretprobe(struct pt_regs *ctx, struct bpf_map_de
 	if (!should_tap(id >> 32)) {
 		return;
 	}
-
+	
 	struct ssl_info *infoPtr = bpf_map_lookup_elem(map_fd, &id);
 	
 	if (infoPtr == NULL) {
@@ -101,10 +99,10 @@ static __always_inline void ssl_uretprobe(struct pt_regs *ctx, struct bpf_map_de
 		return;
 	}
 
-	int count_bytes = get_count_bytes(ctx, &info, id);
-	if (count_bytes <= 0) {
-		return;
-	}
+    int count_bytes = get_count_bytes(ctx, &info, id);
+    if (count_bytes <= 0) {
+        return;
+    }
 
 	output_ssl_chunk(ctx, &info, count_bytes, id, flags);
 }

tap/tlstapper/bpf/tcp_kprobes.c

@@ -1,118 +0,0 @@
-#include "include/headers.h"
-#include "include/maps.h"
-#include "include/log.h"
-#include "include/logger_messages.h"
-#include "include/pids.h"
-#include "include/common.h"
-
-
-static __always_inline int tcp_kprobes_get_address_pair_from_ctx(struct pt_regs *ctx, __u64 id, struct address_info *address_info_ptr) {
-	long err;
-	struct sock *sk = (struct sock *) PT_REGS_PARM1(ctx);
-
-	short unsigned int family;
-	err = bpf_probe_read(&family, sizeof(family), (void *)&sk->__sk_common.skc_family);
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SOCKET_FAMILY, id, err, 0l);
-		return -1;
-	}
-	if (family != AF_INET) {
-		return -1;
-	}
-
-	// daddr, saddr and dport are in network byte order (big endian)
-	// sport is in host byte order
-	__be32 saddr;
-	__be32 daddr;
-	__be16 dport;
-	__u16 sport;
-
-	err = bpf_probe_read(&saddr, sizeof(saddr), (void *)&sk->__sk_common.skc_rcv_saddr);
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SOCKET_SADDR, id, err, 0l);
-		return -1;
-	}
-	err = bpf_probe_read(&daddr, sizeof(daddr), (void *)&sk->__sk_common.skc_daddr);
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SOCKET_DADDR, id, err, 0l);
-		return -1;
-	}
-	err = bpf_probe_read(&dport, sizeof(dport), (void *)&sk->__sk_common.skc_dport);
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SOCKET_DPORT, id, err, 0l);
-		return -1;
-	}
-	err = bpf_probe_read(&sport, sizeof(sport), (void *)&sk->__sk_common.skc_num);
-	if (err != 0) {
-		log_error(ctx, LOG_ERROR_READING_SOCKET_SPORT, id, err, 0l);
-		return -1;
-	}
-
-	address_info_ptr->daddr = daddr;
-	address_info_ptr->saddr = saddr;
-	address_info_ptr->dport = dport;
-	address_info_ptr->sport = bpf_htons(sport);
-
-	return 0;
-}
-
-static __always_inline void tcp_kprobes_forward_go(struct pt_regs *ctx, __u64 id, __u32 fd, struct address_info address_info, struct bpf_map_def *map_fd_go_user_kernel) {
-		__u32 pid = id >> 32;
-		__u64 key = (__u64) pid << 32 | fd;
-
-		long err = bpf_map_update_elem(map_fd_go_user_kernel, &key, &address_info, BPF_ANY);
-    if (err != 0) {
-        log_error(ctx, LOG_ERROR_PUTTING_GO_USER_KERNEL_CONTEXT, id, fd, err);
-				return;
-    }
-}
-
-static void __always_inline tcp_kprobes_forward_openssl(struct ssl_info *info_ptr, struct address_info address_info) {
-		info_ptr->address_info.daddr = address_info.daddr;
-		info_ptr->address_info.saddr = address_info.saddr;
-		info_ptr->address_info.dport = address_info.dport;
-		info_ptr->address_info.sport = address_info.sport;
-}
-
-static __always_inline void tcp_kprobe(struct pt_regs *ctx, struct bpf_map_def *map_fd_openssl, struct bpf_map_def *map_fd_go_kernel, struct bpf_map_def *map_fd_go_user_kernel) {
-	long err;
-
-	__u64 id = bpf_get_current_pid_tgid();
-
-	if (!should_tap(id >> 32)) {
-		return;
-	}
-
-	struct address_info address_info;
-	if (0 != tcp_kprobes_get_address_pair_from_ctx(ctx, id, &address_info)) {
-		return;
-	}
-
-	struct ssl_info *info_ptr = bpf_map_lookup_elem(map_fd_openssl, &id);
-	__u32 *fd_ptr;
-	if (info_ptr == NULL) {
-		fd_ptr = bpf_map_lookup_elem(map_fd_go_kernel, &id);
-		// Connection is used by a Go program
-		if (fd_ptr == NULL) {
-			// Connection was not created by a Go program or by openssl lib
-			return;
-		}
-		tcp_kprobes_forward_go(ctx, id, *fd_ptr, address_info, map_fd_go_user_kernel);
-	} else {
-		// Connection is used by openssl lib
-		tcp_kprobes_forward_openssl(info_ptr, address_info);
-	}
-
-}
-
-SEC("kprobe/tcp_sendmsg")
-void BPF_KPROBE(tcp_sendmsg) {
-	__u64 id = bpf_get_current_pid_tgid();
-	tcp_kprobe(ctx, &openssl_write_context, &go_kernel_write_context, &go_user_kernel_write_context);
-}
-
-SEC("kprobe/tcp_recvmsg")
-void BPF_KPROBE(tcp_recvmsg) {
-	__u64 id = bpf_get_current_pid_tgid();
-	tcp_kprobe(ctx, &openssl_read_context, &go_kernel_read_context, &go_user_kernel_read_context);
-}

tap/tlstapper/bpf/tls_tapper.c

@@ -15,7 +15,6 @@ Copyright (C) UP9 Inc.
 //
 #include "common.c"
 #include "openssl_uprobes.c"
-#include "tcp_kprobes.c"
 #include "go_uprobes.c"
 #include "fd_tracepoints.c"
 #include "fd_to_address_tracepoints.c"

tap/tlstapper/bpf_logger_messages.go

@@ -4,26 +4,20 @@ package tlstapper
 //
 var bpfLogMessages = []string{
 	/*0000*/ "[%d] Unable to read bytes count from _ex methods [err: %d]",
-	/*0001*/ "[%d] Unable to read ssl buffer [err: %d] [origin: %d]",
-	/*0002*/ "[%d] Buffer is too big [size: %d]",
-	/*0003*/ "[%d] Unable to allocate chunk in bpf heap",
-	/*0004*/ "[%d] Unable to read ssl context [err: %d] [origin: %d]",
-	/*0005*/ "[%d] Unable to put ssl context [err: %d]",
-	/*0006*/ "[%d] Unable to get ssl context",
-	/*0007*/ "[%d] File descriptor is missing for tls chunk",
-	/*0008*/ "[%d] Unable to put file descriptor [err: %d] [origin: %d]",
-	/*0009*/ "[%d] Unable to put accept info [err: %d]",
-	/*0010*/ "[%d] Unable to get accept info",
-	/*0011*/ "[%d] Unable to read accept info [err: %d]",
-	/*0012*/ "[%d] Unable to put connection info to connection context [err: %d] [origin: %d]",
-	/*0013*/ "[%d] Unable to put connect info [err: %d]",
-	/*0014*/ "[%d] Unable to get connect info",
-	/*0015*/ "[%d] Unable to read connect info [err: %d]",
-	/*0016*/ "[%d] Unable to read socket family [err: %d]",
-	/*0017*/ "[%d] Unable to read socket daddr [err: %d]",
-	/*0018*/ "[%d] Unable to read socket saddr [err: %d]",
-	/*0019*/ "[%d] Unable to read socket dport [err: %d]",
-	/*0020*/ "[%d] Unable to read socket sport [err: %d]",
-	/*0021*/ "[%d] Unable to put go user-kernel context [fd: %d] [err: %d]",
-	/*0022*/ "[%d] Unable to get go user-kernel context [fd: %d]]",
+	/*0001*/ "[%d] Unable to read ipv4 address [err: %d]",
+	/*0002*/ "[%d] Unable to read ssl buffer [err: %d]",
+	/*0003*/ "[%d] Buffer is too big [size: %d]",
+	/*0004*/ "[%d] Unable to allocate chunk in bpf heap",
+	/*0005*/ "[%d] Unable to read ssl context [err: %d] [origin: %d]",
+	/*0006*/ "[%d] Unable to put ssl context [err: %d]",
+	/*0007*/ "[%d] Unable to get ssl context",
+	/*0008*/ "[%d] File descriptor is missing for tls chunk",
+	/*0009*/ "[%d] Unable to put file descriptor [err: %d] [origin: %d]",
+	/*0010*/ "[%d] Unable to put accept info [err: %d]",
+	/*0011*/ "[%d] Unable to get accept info",
+	/*0012*/ "[%d] Unable to read accept info [err: %d]",
+	/*0013*/ "[%d] Unable to put file descriptor to address mapping [err: %d] [origin: %d]",
+	/*0014*/ "[%d] Unable to put connect info [err: %d]",
+	/*0015*/ "[%d] Unable to get connect info",
+	/*0016*/ "[%d] Unable to read connect info [err: %d]",
 }

tap/tlstapper/chunk.go

@@ -1,33 +1,38 @@
 package tlstapper
 
 import (
+	"bytes"
 	"encoding/binary"
 	"net"
-	"unsafe"
+
+	"github.com/go-errors/errors"
+	"github.com/up9inc/mizu/tap/api"
 )
 
 const FlagsIsClientBit uint32 = 1 << 0
 const FlagsIsReadBit uint32 = 1 << 1
 
-type addressPair struct {
-	srcIp   net.IP
-	srcPort uint16
-	dstIp   net.IP
-	dstPort uint16
-}
+func (c *tlsTapperTlsChunk) getAddress() (net.IP, uint16, error) {
+	address := bytes.NewReader(c.Address[:])
+	var family uint16
+	var port uint16
+	var ip32 uint32
 
-func (c *tlsTapperTlsChunk) getSrcAddress() (net.IP, uint16) {
-	ip := intToIP(c.AddressInfo.Saddr)
-	port := ntohs(c.AddressInfo.Sport)
+	if err := binary.Read(address, binary.BigEndian, &family); err != nil {
+		return nil, 0, errors.Wrap(err, 0)
+	}
 
-	return ip, port
-}
+	if err := binary.Read(address, binary.BigEndian, &port); err != nil {
+		return nil, 0, errors.Wrap(err, 0)
+	}
 
-func (c *tlsTapperTlsChunk) getDstAddress() (net.IP, uint16) {
-	ip := intToIP(c.AddressInfo.Daddr)
-	port := ntohs(c.AddressInfo.Dport)
+	if err := binary.Read(address, binary.BigEndian, &ip32); err != nil {
+		return nil, 0, errors.Wrap(err, 0)
+	}
 
-	return ip, port
+	ip := net.IP{uint8(ip32 >> 24), uint8(ip32 >> 16), uint8(ip32 >> 8), uint8(ip32)}
+
+	return ip, port, nil
 }
 
 func (c *tlsTapperTlsChunk) isClient() bool {
@@ -54,36 +59,26 @@ func (c *tlsTapperTlsChunk) isRequest() bool {
 	return (c.isClient() && c.isWrite()) || (c.isServer() && c.isRead())
 }
 
-func (c *tlsTapperTlsChunk) getAddressPair() addressPair {
-	var (
-		srcIp, dstIp     net.IP
-		srcPort, dstPort uint16
-	)
+func (c *tlsTapperTlsChunk) getAddressPair() (addressPair, error) {
+	ip, port, err := c.getAddress()
+
+	if err != nil {
+		return addressPair{}, err
+	}
 
 	if c.isRequest() {
-		srcIp, srcPort = c.getSrcAddress()
-		dstIp, dstPort = c.getDstAddress()
+		return addressPair{
+			srcIp:   api.UnknownIp,
+			srcPort: api.UnknownPort,
+			dstIp:   ip,
+			dstPort: port,
+		}, nil
 	} else {
-		srcIp, srcPort = c.getDstAddress()
-		dstIp, dstPort = c.getSrcAddress()
-	}
-
-	return addressPair{
-		srcIp:   srcIp,
-		srcPort: srcPort,
-		dstIp:   dstIp,
-		dstPort: dstPort,
+		return addressPair{
+			srcIp:   ip,
+			srcPort: port,
+			dstIp:   api.UnknownIp,
+			dstPort: api.UnknownPort,
+		}, nil
 	}
 }
-
-// intToIP converts IPv4 number to net.IP
-func intToIP(ip32be uint32) net.IP {
-	return net.IPv4(uint8(ip32be), uint8(ip32be>>8), uint8(ip32be>>16), uint8(ip32be>>24))
-}
-
-// ntohs converts big endian (network byte order) to little endian (assuming that's the host byte order)
-func ntohs(i16be uint16) uint16 {
-	b := make([]byte, 2)
-	binary.BigEndian.PutUint16(b, i16be)
-	return *(*uint16)(unsafe.Pointer(&b[0]))
-}

tap/tlstapper/go_hooks.go

@@ -31,32 +31,9 @@ func (s *goHooks) installUprobes(bpfObjects *tlsTapperObjects, filePath string)
 func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable, offsets goOffsets) error {
 	var err error
 
-	goCryptoTlsWrite := bpfObjects.GoCryptoTlsAbiInternalWrite
-	goCryptoTlsWriteEx := bpfObjects.GoCryptoTlsAbiInternalWriteEx
-	goCryptoTlsRead := bpfObjects.GoCryptoTlsAbiInternalRead
-	goCryptoTlsReadEx := bpfObjects.GoCryptoTlsAbiInternalReadEx
-
-	if offsets.Abi == ABI0 {
-		goCryptoTlsWrite = bpfObjects.GoCryptoTlsAbi0Write
-		goCryptoTlsWriteEx = bpfObjects.GoCryptoTlsAbi0WriteEx
-		goCryptoTlsRead = bpfObjects.GoCryptoTlsAbi0Read
-		goCryptoTlsReadEx = bpfObjects.GoCryptoTlsAbi0ReadEx
-
-		// Pass goid and g struct offsets to an eBPF map to retrieve it in eBPF context
-		if err := bpfObjects.tlsTapperMaps.GoidOffsetsMap.Put(
-			uint32(0),
-			tlsTapperGoidOffsets{
-				G_addrOffset: offsets.GStructOffset,
-				GoidOffset:   offsets.GoidOffset,
-			},
-		); err != nil {
-			return errors.Wrap(err, 0)
-		}
-	}
-
 	// Symbol points to
 	// [`crypto/tls.(*Conn).Write`](https://github.com/golang/go/blob/go1.17.6/src/crypto/tls/conn.go#L1099)
-	s.goWriteProbe, err = ex.Uprobe(goWriteSymbol, goCryptoTlsWrite, &link.UprobeOptions{
+	s.goWriteProbe, err = ex.Uprobe(goWriteSymbol, bpfObjects.GoCryptoTlsWrite, &link.UprobeOptions{
 		Offset: offsets.GoWriteOffset.enter,
 	})
 
@@ -65,7 +42,7 @@ func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable
 	}
 
 	for _, offset := range offsets.GoWriteOffset.exits {
-		probe, err := ex.Uprobe(goWriteSymbol, goCryptoTlsWriteEx, &link.UprobeOptions{
+		probe, err := ex.Uprobe(goWriteSymbol, bpfObjects.GoCryptoTlsWriteEx, &link.UprobeOptions{
 			Offset: offset,
 		})
 
@@ -78,7 +55,7 @@ func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable
 
 	// Symbol points to
 	// [`crypto/tls.(*Conn).Read`](https://github.com/golang/go/blob/go1.17.6/src/crypto/tls/conn.go#L1263)
-	s.goReadProbe, err = ex.Uprobe(goReadSymbol, goCryptoTlsRead, &link.UprobeOptions{
+	s.goReadProbe, err = ex.Uprobe(goReadSymbol, bpfObjects.GoCryptoTlsRead, &link.UprobeOptions{
 		Offset: offsets.GoReadOffset.enter,
 	})
 
@@ -87,7 +64,7 @@ func (s *goHooks) installHooks(bpfObjects *tlsTapperObjects, ex *link.Executable
 	}
 
 	for _, offset := range offsets.GoReadOffset.exits {
-		probe, err := ex.Uprobe(goReadSymbol, goCryptoTlsReadEx, &link.UprobeOptions{
+		probe, err := ex.Uprobe(goReadSymbol, bpfObjects.GoCryptoTlsReadEx, &link.UprobeOptions{
 			Offset: offset,
 		})
 

tap/tlstapper/go_offsets.go

@@ -2,10 +2,8 @@ package tlstapper
 
 import (
 	"bufio"
-	"debug/dwarf"
 	"debug/elf"
 	"fmt"
-	"io"
 	"os"
 	"runtime"
 
@@ -15,22 +13,9 @@ import (
 	"github.com/up9inc/mizu/logger"
 )
 
-type goAbi int
-
-const (
-	ABI0 goAbi = iota
-	ABIInternal
-)
-
-const PtrSize int = 8
-
 type goOffsets struct {
 	GoWriteOffset *goExtendedOffset
 	GoReadOffset  *goExtendedOffset
-	GoVersion     string
-	Abi           goAbi
-	GoidOffset    uint64
-	GStructOffset uint64
 }
 
 type goExtendedOffset struct {
@@ -39,33 +24,30 @@ type goExtendedOffset struct {
 }
 
 const (
-	minimumABIInternalGoVersion = "1.17.0"
-	goVersionSymbol             = "runtime.buildVersion.str" // symbol does not exist in Go (<=1.16)
-	goWriteSymbol               = "crypto/tls.(*Conn).Write"
-	goReadSymbol                = "crypto/tls.(*Conn).Read"
+	minimumSupportedGoVersion = "1.17.0"
+	goVersionSymbol           = "runtime.buildVersion.str"
+	goWriteSymbol             = "crypto/tls.(*Conn).Write"
+	goReadSymbol              = "crypto/tls.(*Conn).Read"
 )
 
 func findGoOffsets(filePath string) (goOffsets, error) {
-	offsets, goidOffset, gStructOffset, err := getOffsets(filePath)
+	offsets, err := getOffsets(filePath)
 	if err != nil {
 		return goOffsets{}, err
 	}
 
-	abi := ABI0
-	var passed bool
-	var goVersion string
-
 	goVersionOffset, err := getOffset(offsets, goVersionSymbol)
-	if err == nil {
-		// TODO: Replace this logic with https://pkg.go.dev/debug/buildinfo#ReadFile once we upgrade to 1.18
-		passed, goVersion, err = checkGoVersion(filePath, goVersionOffset)
-		if err != nil {
-			return goOffsets{}, fmt.Errorf("Checking Go version: %s", err)
-		}
+	if err != nil {
+		return goOffsets{}, err
 	}
 
-	if passed {
-		abi = ABIInternal
+	passed, goVersion, err := checkGoVersion(filePath, goVersionOffset)
+	if err != nil {
+		return goOffsets{}, fmt.Errorf("Checking Go version: %s", err)
+	}
+
+	if !passed {
+		return goOffsets{}, fmt.Errorf("Unsupported Go version: %s", goVersion)
 	}
 
 	writeOffset, err := getOffset(offsets, goWriteSymbol)
@@ -81,139 +63,10 @@ func findGoOffsets(filePath string) (goOffsets, error) {
 	return goOffsets{
 		GoWriteOffset: writeOffset,
 		GoReadOffset:  readOffset,
-		GoVersion:     goVersion,
-		Abi:           abi,
-		GoidOffset:    goidOffset,
-		GStructOffset: gStructOffset,
 	}, nil
 }
 
-func getSymbol(exe *elf.File, name string) *elf.Symbol {
-	symbols, err := exe.Symbols()
-	if err != nil {
-		return nil
-	}
-
-	for _, symbol := range symbols {
-		if symbol.Name == name {
-			s := symbol
-			return &s
-		}
-	}
-	return nil
-}
-
-func getGStructOffset(exe *elf.File) (gStructOffset uint64, err error) {
-	// This is a bit arcane. Essentially:
-	// - If the program is pure Go, it can do whatever it wants, and puts the G
-	//   pointer at %fs-8 on 64 bit.
-	// - %Gs is the index of private storage in GDT on 32 bit, and puts the G
-	//   pointer at -4(tls).
-	// - Otherwise, Go asks the external linker to place the G pointer by
-	//   emitting runtime.tlsg, a TLS symbol, which is relocated to the chosen
-	//   offset in libc's TLS block.
-	// - On ARM64 (but really, any architecture other than i386 and 86x64) the
-	//   offset is calculate using runtime.tls_g and the formula is different.
-
-	var tls *elf.Prog
-	for _, prog := range exe.Progs {
-		if prog.Type == elf.PT_TLS {
-			tls = prog
-			break
-		}
-	}
-
-	switch exe.Machine {
-	case elf.EM_X86_64, elf.EM_386:
-		tlsg := getSymbol(exe, "runtime.tlsg")
-		if tlsg == nil || tls == nil {
-			gStructOffset = ^uint64(PtrSize) + 1 //-ptrSize
-			return
-		}
-
-		// According to https://reviews.llvm.org/D61824, linkers must pad the actual
-		// size of the TLS segment to ensure that (tlsoffset%align) == (vaddr%align).
-		// This formula, copied from the lld code, matches that.
-		// https://github.com/llvm-mirror/lld/blob/9aef969544981d76bea8e4d1961d3a6980980ef9/ELF/InputSection.cpp#L643
-		memsz := tls.Memsz + (-tls.Vaddr-tls.Memsz)&(tls.Align-1)
-
-		// The TLS register points to the end of the TLS block, which is
-		// tls.Memsz long. runtime.tlsg is an offset from the beginning of that block.
-		gStructOffset = ^(memsz) + 1 + tlsg.Value // -tls.Memsz + tlsg.Value
-
-	case elf.EM_AARCH64:
-		tlsg := getSymbol(exe, "runtime.tls_g")
-		if tlsg == nil || tls == nil {
-			gStructOffset = 2 * uint64(PtrSize)
-			return
-		}
-
-		gStructOffset = tlsg.Value + uint64(PtrSize*2) + ((tls.Vaddr - uint64(PtrSize*2)) & (tls.Align - 1))
-
-	default:
-		// we should never get here
-		err = fmt.Errorf("architecture not supported")
-	}
-
-	return
-}
-
-func getGoidOffset(elfFile *elf.File) (goidOffset uint64, gStructOffset uint64, err error) {
-	var dwarfData *dwarf.Data
-	dwarfData, err = elfFile.DWARF()
-	if err != nil {
-		return
-	}
-
-	entryReader := dwarfData.Reader()
-
-	var runtimeGOffset uint64
-	var seenRuntimeG bool
-
-	for {
-		// Read all entries in sequence
-		var entry *dwarf.Entry
-		entry, err = entryReader.Next()
-		if err == io.EOF || entry == nil {
-			// We've reached the end of DWARF entries
-			break
-		}
-
-		// Check if this entry is a struct
-		if entry.Tag == dwarf.TagStructType {
-			// Go through fields
-			for _, field := range entry.Field {
-				if field.Attr == dwarf.AttrName {
-					val := field.Val.(string)
-					if val == "runtime.g" {
-						runtimeGOffset = uint64(entry.Offset)
-						seenRuntimeG = true
-					}
-				}
-			}
-		}
-
-		// Check if this entry is a struct member
-		if seenRuntimeG && entry.Tag == dwarf.TagMember {
-			// Go through fields
-			for _, field := range entry.Field {
-				if field.Attr == dwarf.AttrName {
-					val := field.Val.(string)
-					if val == "goid" {
-						goidOffset = uint64(entry.Offset) - runtimeGOffset - 0x4b
-						gStructOffset, err = getGStructOffset(elfFile)
-						return
-					}
-				}
-			}
-		}
-	}
-
-	err = fmt.Errorf("goid not found in DWARF")
-	return
-}
-
-func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, goidOffset uint64, gStructOffset uint64, err error) {
+func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err error) {
 	var engine gapstone.Engine
 	switch runtime.GOARCH {
 	case "amd64":
@@ -251,13 +104,13 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, goidOffs
 	}
 	defer fd.Close()
 
-	var elfFile *elf.File
-	elfFile, err = elf.NewFile(fd)
+	var se *elf.File
+	se, err = elf.NewFile(fd)
 	if err != nil {
 		return
 	}
 
-	textSection := elfFile.Section(".text")
+	textSection := se.Section(".text")
 	if textSection == nil {
 		err = fmt.Errorf("No text section")
 		return
@@ -271,7 +124,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, goidOffs
 	}
 
 	var syms []elf.Symbol
-	syms, err = elfFile.Symbols()
+	syms, err = se.Symbols()
 	if err != nil {
 		return
 	}
@@ -279,7 +132,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, goidOffs
 		offset := sym.Value
 
 		var lastProg *elf.Prog
-		for _, prog := range elfFile.Progs {
+		for _, prog := range se.Progs {
 			if prog.Vaddr <= sym.Value && sym.Value < (prog.Vaddr+prog.Memsz) {
 				offset = sym.Value - prog.Vaddr + prog.Off
 				lastProg = prog
@@ -336,8 +189,6 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, goidOffs
 		offsets[sym.Name] = extendedOffset
 	}
 
-	goidOffset, gStructOffset, err = getGoidOffset(elfFile)
-
 	return
 }
 
@@ -378,7 +229,7 @@ func checkGoVersion(filePath string, offset *goExtendedOffset) (bool, string, er
 		return false, goVersionStr, err
 	}
 
-	goVersionConstraint, err := semver.NewConstraint(fmt.Sprintf(">= %s", minimumABIInternalGoVersion))
+	goVersionConstraint, err := semver.NewConstraint(fmt.Sprintf(">= %s", minimumSupportedGoVersion))
 	if err != nil {
 		return false, goVersionStr, err
 	}

tap/tlstapper/sockfd_info.go

@@ -0,0 +1,122 @@
+package tlstapper
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/go-errors/errors"
+)
+
+var socketInodeRegex = regexp.MustCompile(`socket:\[(\d+)\]`)
+
+const (
+	SRC_ADDRESS_FILED_INDEX = 1
+	DST_ADDRESS_FILED_INDEX = 2
+	INODE_FILED_INDEX       = 9
+)
+
+type addressPair struct {
+	srcIp   net.IP
+	srcPort uint16
+	dstIp   net.IP
+	dstPort uint16
+}
+
+// This file helps to extract Ip and Port out of a Socket file descriptor.
+//
+// The equivalent bash commands are:
+//
+//  > ls -l /proc/<pid>/fd/<fd>
+// 	    Output something like "socket:[1234]" for sockets - 1234 is the inode of the socket
+//  > cat /proc/<pid>/net/tcp | grep <inode>
+//      Output a line per ipv4 socket, the 9th field is the inode of the socket
+//      The 1st and 2nd fields are the source and dest ip and ports in a Hex format
+//      0100007F:50 is 127.0.0.1:80
+
+func getAddressBySockfd(procfs string, pid uint32, fd uint32) (addressPair, error) {
+	inode, err := getSocketInode(procfs, pid, fd)
+
+	if err != nil {
+		return addressPair{}, err
+	}
+
+	tcppath := fmt.Sprintf("%s/%d/net/tcp", procfs, pid)
+	tcp, err := ioutil.ReadFile(tcppath)
+
+	if err != nil {
+		return addressPair{}, errors.Wrap(err, 0)
+	}
+
+	for _, line := range strings.Split(string(tcp), "\n") {
+		parts := strings.Fields(line)
+
+		if len(parts) < 10 {
+			continue
+		}
+
+		if inode == parts[INODE_FILED_INDEX] {
+			srcIp, srcPort, srcErr := parseHexAddress(parts[SRC_ADDRESS_FILED_INDEX])
+
+			if srcErr != nil {
+				return addressPair{}, srcErr
+			}
+
+			dstIp, dstPort, dstErr := parseHexAddress(parts[DST_ADDRESS_FILED_INDEX])
+
+			if dstErr != nil {
+				return addressPair{}, dstErr
+			}
+
+			return addressPair{
+				srcIp:   srcIp,
+				srcPort: srcPort,
+				dstIp:   dstIp,
+				dstPort: dstPort,
+			}, nil
+		}
+	}
+
+	return addressPair{}, errors.Errorf("address not found [pid: %d] [sockfd: %d] [inode: %s]", pid, fd, inode)
+}
+
+func getSocketInode(procfs string, pid uint32, fd uint32) (string, error) {
+	fdlinkPath := fmt.Sprintf("%s/%d/fd/%d", procfs, pid, fd)
+	fdlink, err := os.Readlink(fdlinkPath)
+
+	if err != nil {
+		return "", errors.Wrap(err, 0)
+	}
+
+	tokens := socketInodeRegex.FindStringSubmatch(fdlink)
+
+	if tokens == nil || len(tokens) < 1 {
+		return "", errors.Errorf("socket inode not found [pid: %d] [sockfd: %d] [link: %s]", pid, fd, fdlink)
+	}
+
+	return tokens[1], nil
+}
+
+// Format looks like 0100007F:50 for 127.0.0.1:80
+//
+func parseHexAddress(addr string) (net.IP, uint16, error) {
+	addrParts := strings.Split(addr, ":")
+
+	port, err := strconv.ParseUint(addrParts[1], 16, 16)
+
+	if err != nil {
+		return nil, 0, errors.Wrap(err, 0)
+	}
+
+	ip, err := strconv.ParseUint(addrParts[0], 16, 32)
+
+	if err != nil {
+		return nil, 0, errors.Wrap(err, 0)
+	}
+
+	return net.IP{uint8(ip), uint8(ip >> 8), uint8(ip >> 16), uint8(ip >> 24)}, uint16(port), nil
+}

tap/tlstapper/syscall_hooks.go

@@ -8,8 +8,6 @@ import (
 type syscallHooks struct {
 	sysEnterRead    link.Link
 	sysEnterWrite   link.Link
-	sysExitRead     link.Link
-	sysExitWrite    link.Link
 	sysEnterAccept4 link.Link
 	sysExitAccept4  link.Link
 	sysEnterConnect link.Link
@@ -19,49 +17,37 @@ type syscallHooks struct {
 func (s *syscallHooks) installSyscallHooks(bpfObjects *tlsTapperObjects) error {
 	var err error
 
-	s.sysEnterRead, err = link.Tracepoint("syscalls", "sys_enter_read", bpfObjects.SysEnterRead, nil)
+	s.sysEnterRead, err = link.Tracepoint("syscalls", "sys_enter_read", bpfObjects.SysEnterRead)
 
 	if err != nil {
 		return errors.Wrap(err, 0)
 	}
 
-	s.sysEnterWrite, err = link.Tracepoint("syscalls", "sys_enter_write", bpfObjects.SysEnterWrite, nil)
+	s.sysEnterWrite, err = link.Tracepoint("syscalls", "sys_enter_write", bpfObjects.SysEnterWrite)
 
 	if err != nil {
 		return errors.Wrap(err, 0)
 	}
 
-	s.sysExitRead, err = link.Tracepoint("syscalls", "sys_exit_read", bpfObjects.SysExitRead, nil)
+	s.sysEnterAccept4, err = link.Tracepoint("syscalls", "sys_enter_accept4", bpfObjects.SysEnterAccept4)
 
 	if err != nil {
 		return errors.Wrap(err, 0)
 	}
 
-	s.sysExitWrite, err = link.Tracepoint("syscalls", "sys_exit_write", bpfObjects.SysExitWrite, nil)
+	s.sysExitAccept4, err = link.Tracepoint("syscalls", "sys_exit_accept4", bpfObjects.SysExitAccept4)
 
 	if err != nil {
 		return errors.Wrap(err, 0)
 	}
 
-	s.sysEnterAccept4, err = link.Tracepoint("syscalls", "sys_enter_accept4", bpfObjects.SysEnterAccept4, nil)
+	s.sysEnterConnect, err = link.Tracepoint("syscalls", "sys_enter_connect", bpfObjects.SysEnterConnect)
 
 	if err != nil {
 		return errors.Wrap(err, 0)
 	}
 
-	s.sysExitAccept4, err = link.Tracepoint("syscalls", "sys_exit_accept4", bpfObjects.SysExitAccept4, nil)
-
-	if err != nil {
-		return errors.Wrap(err, 0)
-	}
-
-	s.sysEnterConnect, err = link.Tracepoint("syscalls", "sys_enter_connect", bpfObjects.SysEnterConnect, nil)
-
-	if err != nil {
-		return errors.Wrap(err, 0)
-	}
-
-	s.sysExitConnect, err = link.Tracepoint("syscalls", "sys_exit_connect", bpfObjects.SysExitConnect, nil)
+	s.sysExitConnect, err = link.Tracepoint("syscalls", "sys_exit_connect", bpfObjects.SysExitConnect)
 
 	if err != nil {
 		return errors.Wrap(err, 0)
@@ -81,14 +67,6 @@ func (s *syscallHooks) close() []error {
 		returnValue = append(returnValue, err)
 	}
 
-	if err := s.sysExitRead.Close(); err != nil {
-		returnValue = append(returnValue, err)
-	}
-
-	if err := s.sysExitWrite.Close(); err != nil {
-		returnValue = append(returnValue, err)
-	}
-
 	if err := s.sysEnterAccept4.Close(); err != nil {
 		returnValue = append(returnValue, err)
 	}

tap/tlstapper/tcp_kprobe_hooks.go

@@ -1,45 +0,0 @@
-package tlstapper
-
-import (
-	"github.com/cilium/ebpf/link"
-	"github.com/go-errors/errors"
-)
-
-type tcpKprobeHooks struct {
-	tcpSendmsg         link.Link
-	tcpRecvmsg         link.Link
-}
-
-func (s *tcpKprobeHooks) installTcpKprobeHooks(bpfObjects *tlsTapperObjects) error {
-	var err error
-
-	s.tcpSendmsg, err = link.Kprobe("tcp_sendmsg", bpfObjects.TcpSendmsg, nil)
-	if err != nil {
-		return errors.Wrap(err, 0)
-	}
-
-	s.tcpRecvmsg, err = link.Kprobe("tcp_recvmsg", bpfObjects.TcpRecvmsg, nil)
-	if err != nil {
-		return errors.Wrap(err, 0)
-	}
-
-	return nil
-}
-
-func (s *tcpKprobeHooks) close() []error {
-	returnValue := make([]error, 0)
-
-	if s.tcpSendmsg != nil {
-		if err := s.tcpSendmsg.Close(); err != nil {
-			returnValue = append(returnValue, err)
-		}
-	}
-
-	if s.tcpRecvmsg != nil {
-		if err := s.tcpRecvmsg.Close(); err != nil {
-			returnValue = append(returnValue, err)
-		}
-	}
-
-	return returnValue
-}

tap/tlstapper/tls_poller.go

@@ -134,7 +134,15 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsTapperTlsChunk) {
 
 func (p *tlsPoller) handleTlsChunk(chunk *tlsTapperTlsChunk, extension *api.Extension, emitter api.Emitter,
 	options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) error {
-	address := chunk.getAddressPair()
+	address, err := p.getSockfdAddressPair(chunk)
+
+	if err != nil {
+		address, err = chunk.getAddressPair()
+
+		if err != nil {
+			return err
+		}
+	}
 
 	key := buildTlsKey(address)
 	reader, exists := p.readers[key]
@@ -208,6 +216,41 @@ func (p *tlsPoller) closeReader(key string, r *tlsReader) {
 	p.closedReaders <- key
 }
 
+func (p *tlsPoller) getSockfdAddressPair(chunk *tlsTapperTlsChunk) (addressPair, error) {
+	address, err := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd)
+	fdCacheKey := fmt.Sprintf("%d:%d", chunk.Pid, chunk.Fd)
+
+	if err == nil {
+		if !chunk.isRequest() {
+			switchedAddress := addressPair{
+				srcIp:   address.dstIp,
+				srcPort: address.dstPort,
+				dstIp:   address.srcIp,
+				dstPort: address.srcPort,
+			}
+			p.fdCache.Add(fdCacheKey, switchedAddress)
+			return switchedAddress, nil
+		} else {
+			p.fdCache.Add(fdCacheKey, address)
+			return address, nil
+		}
+	}
+
+	fromCacheIfc, ok := p.fdCache.Get(fdCacheKey)
+
+	if !ok {
+		return addressPair{}, err
+	}
+
+	fromCache, ok := fromCacheIfc.(addressPair)
+
+	if !ok {
+		return address, errors.Errorf("Unable to cast %T to addressPair", fromCacheIfc)
+	}
+
+	return fromCache, nil
+}
+
 func buildTlsKey(address addressPair) string {
 	return fmt.Sprintf("%s:%d>%s:%d", address.srcIp, address.srcPort, address.dstIp, address.dstPort)
 }

tap/tlstapper/tls_tapper.go

@@ -6,23 +6,17 @@ import (
 
 	"github.com/cilium/ebpf/rlimit"
 	"github.com/go-errors/errors"
-	"github.com/moby/moby/pkg/parsers/kernel"
 	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
 
 const GlobalTapPid = 0
 
-// TODO: cilium/ebpf does not support .kconfig Therefore; for now, we build object files per kernel version.
-
-//go:generate go run github.com/cilium/ebpf/cmd/bpf2go@v0.9.0 -target $BPF_TARGET -cflags $BPF_CFLAGS -type tls_chunk -type goid_offsets tlsTapper bpf/tls_tapper.c
-
-//go:generate go run github.com/cilium/ebpf/cmd/bpf2go@v0.9.0 -target $BPF_TARGET -cflags "${BPF_CFLAGS} -DKERNEL_BEFORE_4_6" -type tls_chunk -type goid_offsets tlsTapper46 bpf/tls_tapper.c
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go@0d0727ef53e2f53b1731c73f4c61e0f58693083a -target $BPF_TARGET -cflags $BPF_CFLAGS -type tls_chunk tlsTapper bpf/tls_tapper.c
 
 type TlsTapper struct {
 	bpfObjects      tlsTapperObjects
 	syscallHooks    syscallHooks
-	tcpKprobeHooks  tcpKprobeHooks
 	sslHooksStructs []sslHooks
 	goHooksStructs  []goHooks
 	poller          *tlsPoller
@@ -33,30 +27,13 @@ type TlsTapper struct {
 func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string, extension *api.Extension) error {
 	logger.Log.Infof("Initializing tls tapper (chunksSize: %d) (logSize: %d)", chunksBufferSize, logBufferSize)
 
-	var err error
-	err = setupRLimit()
-	if err != nil {
+	if err := setupRLimit(); err != nil {
 		return err
 	}
 
-	var kernelVersion *kernel.VersionInfo
-	kernelVersion, err = kernel.GetKernelVersion()
-	if err != nil {
-		return err
-	}
-
-	logger.Log.Infof("Detected Linux kernel version: %s", kernelVersion)
-
 	t.bpfObjects = tlsTapperObjects{}
-	// TODO: cilium/ebpf does not support .kconfig Therefore; for now, we load object files according to kernel version.
-	if kernel.CompareKernelVersion(*kernelVersion, kernel.VersionInfo{Kernel: 4, Major: 6, Minor: 0}) < 1 {
-		if err := loadTlsTapper46Objects(&t.bpfObjects, nil); err != nil {
-			return errors.Wrap(err, 0)
-		}
-	} else {
-		if err := loadTlsTapperObjects(&t.bpfObjects, nil); err != nil {
-			return errors.Wrap(err, 0)
-		}
+	if err := loadTlsTapperObjects(&t.bpfObjects, nil); err != nil {
+		return errors.Wrap(err, 0)
 	}
 
 	t.syscallHooks = syscallHooks{}
@@ -64,11 +41,6 @@ func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string,
 		return err
 	}
 
-	t.tcpKprobeHooks = tcpKprobeHooks{}
-	if err := t.tcpKprobeHooks.installTcpKprobeHooks(&t.bpfObjects); err != nil {
-		return err
-	}
-
 	t.sslHooksStructs = make([]sslHooks, 0)
 
 	t.bpfLogger = newBpfLogger()
@@ -76,6 +48,7 @@ func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string,
 		return err
 	}
 
+	var err error
 	t.poller, err = newTlsPoller(t, extension, procfs)
 
 	if err != nil {
@@ -158,8 +131,6 @@ func (t *TlsTapper) Close() []error {
 
 	returnValue = append(returnValue, t.syscallHooks.close()...)
 
-	returnValue = append(returnValue, t.tcpKprobeHooks.close()...)
-
 	for _, sslHooks := range t.sslHooksStructs {
 		returnValue = append(returnValue, sslHooks.close()...)
 	}

tap/tlstapper/tlstapper46_bpfel_arm64.go

@@ -1,244 +0,0 @@
-// Code generated by bpf2go; DO NOT EDIT.
-//go:build arm64
-// +build arm64
-
-package tlstapper
-
-import (
-	"bytes"
-	_ "embed"
-	"fmt"
-	"io"
-
-	"github.com/cilium/ebpf"
-)
-
-type tlsTapper46GoidOffsets struct {
-	G_addrOffset uint64
-	GoidOffset   uint64
-}
-
-type tlsTapper46TlsChunk struct {
-	Pid         uint32
-	Tgid        uint32
-	Len         uint32
-	Start       uint32
-	Recorded    uint32
-	Fd          uint32
-	Flags       uint32
-	AddressInfo struct {
-		Mode  int32
-		Saddr uint32
-		Daddr uint32
-		Sport uint16
-		Dport uint16
-	}
-	Data [4096]uint8
-}
-
-// loadTlsTapper46 returns the embedded CollectionSpec for tlsTapper46.
-func loadTlsTapper46() (*ebpf.CollectionSpec, error) {
-	reader := bytes.NewReader(_TlsTapper46Bytes)
-	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
-	if err != nil {
-		return nil, fmt.Errorf("can't load tlsTapper46: %w", err)
-	}
-
-	return spec, err
-}
-
-// loadTlsTapper46Objects loads tlsTapper46 and converts it into a struct.
-//
-// The following types are suitable as obj argument:
-//
-//     *tlsTapper46Objects
-//     *tlsTapper46Programs
-//     *tlsTapper46Maps
-//
-// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
-func loadTlsTapper46Objects(obj interface{}, opts *ebpf.CollectionOptions) error {
-	spec, err := loadTlsTapper46()
-	if err != nil {
-		return err
-	}
-
-	return spec.LoadAndAssign(obj, opts)
-}
-
-// tlsTapper46Specs contains maps and programs before they are loaded into the kernel.
-//
-// It can be passed ebpf.CollectionSpec.Assign.
-type tlsTapper46Specs struct {
-	tlsTapper46ProgramSpecs
-	tlsTapper46MapSpecs
-}
-
-// tlsTapper46Specs contains programs before they are loaded into the kernel.
-//
-// It can be passed ebpf.CollectionSpec.Assign.
-type tlsTapper46ProgramSpecs struct {
-	GoCryptoTlsAbi0Read           *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_read"`
-	GoCryptoTlsAbi0ReadEx         *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_read_ex"`
-	GoCryptoTlsAbi0Write          *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_write"`
-	GoCryptoTlsAbi0WriteEx        *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi0_write_ex"`
-	GoCryptoTlsAbiInternalRead    *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_read"`
-	GoCryptoTlsAbiInternalReadEx  *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_read_ex"`
-	GoCryptoTlsAbiInternalWrite   *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_write"`
-	GoCryptoTlsAbiInternalWriteEx *ebpf.ProgramSpec `ebpf:"go_crypto_tls_abi_internal_write_ex"`
-	SslRead                       *ebpf.ProgramSpec `ebpf:"ssl_read"`
-	SslReadEx                     *ebpf.ProgramSpec `ebpf:"ssl_read_ex"`
-	SslRetRead                    *ebpf.ProgramSpec `ebpf:"ssl_ret_read"`
-	SslRetReadEx                  *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"`
-	SslRetWrite                   *ebpf.ProgramSpec `ebpf:"ssl_ret_write"`
-	SslRetWriteEx                 *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"`
-	SslWrite                      *ebpf.ProgramSpec `ebpf:"ssl_write"`
-	SslWriteEx                    *ebpf.ProgramSpec `ebpf:"ssl_write_ex"`
-	SysEnterAccept4               *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"`
-	SysEnterConnect               *ebpf.ProgramSpec `ebpf:"sys_enter_connect"`
-	SysEnterRead                  *ebpf.ProgramSpec `ebpf:"sys_enter_read"`
-	SysEnterWrite                 *ebpf.ProgramSpec `ebpf:"sys_enter_write"`
-	SysExitAccept4                *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"`
-	SysExitConnect                *ebpf.ProgramSpec `ebpf:"sys_exit_connect"`
-	TcpRecvmsg                    *ebpf.ProgramSpec `ebpf:"tcp_recvmsg"`
-	TcpSendmsg                    *ebpf.ProgramSpec `ebpf:"tcp_sendmsg"`
-}
-
-// tlsTapper46MapSpecs contains maps before they are loaded into the kernel.
-//
-// It can be passed ebpf.CollectionSpec.Assign.
-type tlsTapper46MapSpecs struct {
-	AcceptSyscallContext *ebpf.MapSpec `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.MapSpec `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.MapSpec `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"`
-	GoReadContext        *ebpf.MapSpec `ebpf:"go_read_context"`
-	GoWriteContext       *ebpf.MapSpec `ebpf:"go_write_context"`
-	GoidOffsetsMap       *ebpf.MapSpec `ebpf:"goid_offsets_map"`
-	Heap                 *ebpf.MapSpec `ebpf:"heap"`
-	LogBuffer            *ebpf.MapSpec `ebpf:"log_buffer"`
-	OpensslReadContext   *ebpf.MapSpec `ebpf:"openssl_read_context"`
-	OpensslWriteContext  *ebpf.MapSpec `ebpf:"openssl_write_context"`
-	PidsMap              *ebpf.MapSpec `ebpf:"pids_map"`
-}
-
-// tlsTapper46Objects contains all objects after they have been loaded into the kernel.
-//
-// It can be passed to loadTlsTapper46Objects or ebpf.CollectionSpec.LoadAndAssign.
-type tlsTapper46Objects struct {
-	tlsTapper46Programs
-	tlsTapper46Maps
-}
-
-func (o *tlsTapper46Objects) Close() error {
-	return _TlsTapper46Close(
-		&o.tlsTapper46Programs,
-		&o.tlsTapper46Maps,
-	)
-}
-
-// tlsTapper46Maps contains all maps after they have been loaded into the kernel.
-//
-// It can be passed to loadTlsTapper46Objects or ebpf.CollectionSpec.LoadAndAssign.
-type tlsTapper46Maps struct {
-	AcceptSyscallContext *ebpf.Map `ebpf:"accept_syscall_context"`
-	ChunksBuffer         *ebpf.Map `ebpf:"chunks_buffer"`
-	ConnectSyscallInfo   *ebpf.Map `ebpf:"connect_syscall_info"`
-	FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"`
-	GoReadContext        *ebpf.Map `ebpf:"go_read_context"`
-	GoWriteContext       *ebpf.Map `ebpf:"go_write_context"`
-	GoidOffsetsMap       *ebpf.Map `ebpf:"goid_offsets_map"`
-	Heap                 *ebpf.Map `ebpf:"heap"`
-	LogBuffer            *ebpf.Map `ebpf:"log_buffer"`
-	OpensslReadContext   *ebpf.Map `ebpf:"openssl_read_context"`
-	OpensslWriteContext  *ebpf.Map `ebpf:"openssl_write_context"`
-	PidsMap              *ebpf.Map `ebpf:"pids_map"`
-}
-
-func (m *tlsTapper46Maps) Close() error {
-	return _TlsTapper46Close(
-		m.AcceptSyscallContext,
-		m.ChunksBuffer,
-		m.ConnectSyscallInfo,
-		m.FileDescriptorToIpv4,
-		m.GoReadContext,
-		m.GoWriteContext,
-		m.GoidOffsetsMap,
-		m.Heap,
-		m.LogBuffer,
-		m.OpensslReadContext,
-		m.OpensslWriteContext,
-		m.PidsMap,
-	)
-}
-
-// tlsTapper46Programs contains all programs after they have been loaded into the kernel.
-//
-// It can be passed to loadTlsTapper46Objects or ebpf.CollectionSpec.LoadAndAssign.
-type tlsTapper46Programs struct {
-	GoCryptoTlsAbi0Read           *ebpf.Program `ebpf:"go_crypto_tls_abi0_read"`
-	GoCryptoTlsAbi0ReadEx         *ebpf.Program `ebpf:"go_crypto_tls_abi0_read_ex"`
-	GoCryptoTlsAbi0Write          *ebpf.Program `ebpf:"go_crypto_tls_abi0_write"`
-	GoCryptoTlsAbi0WriteEx        *ebpf.Program `ebpf:"go_crypto_tls_abi0_write_ex"`
-	GoCryptoTlsAbiInternalRead    *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_read"`
-	GoCryptoTlsAbiInternalReadEx  *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_read_ex"`
-	GoCryptoTlsAbiInternalWrite   *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_write"`
-	GoCryptoTlsAbiInternalWriteEx *ebpf.Program `ebpf:"go_crypto_tls_abi_internal_write_ex"`
-	SslRead                       *ebpf.Program `ebpf:"ssl_read"`
-	SslReadEx                     *ebpf.Program `ebpf:"ssl_read_ex"`
-	SslRetRead                    *ebpf.Program `ebpf:"ssl_ret_read"`
-	SslRetReadEx                  *ebpf.Program `ebpf:"ssl_ret_read_ex"`
-	SslRetWrite                   *ebpf.Program `ebpf:"ssl_ret_write"`
-	SslRetWriteEx                 *ebpf.Program `ebpf:"ssl_ret_write_ex"`
-	SslWrite                      *ebpf.Program `ebpf:"ssl_write"`
-	SslWriteEx                    *ebpf.Program `ebpf:"ssl_write_ex"`
-	SysEnterAccept4               *ebpf.Program `ebpf:"sys_enter_accept4"`
-	SysEnterConnect               *ebpf.Program `ebpf:"sys_enter_connect"`
-	SysEnterRead                  *ebpf.Program `ebpf:"sys_enter_read"`
-	SysEnterWrite                 *ebpf.Program `ebpf:"sys_enter_write"`
-	SysExitAccept4                *ebpf.Program `ebpf:"sys_exit_accept4"`
-	SysExitConnect                *ebpf.Program `ebpf:"sys_exit_connect"`
-	TcpRecvmsg                    *ebpf.Program `ebpf:"tcp_recvmsg"`
-	TcpSendmsg                    *ebpf.Program `ebpf:"tcp_sendmsg"`
-}
-
-func (p *tlsTapper46Programs) Close() error {
-	return _TlsTapper46Close(
-		p.GoCryptoTlsAbi0Read,
-		p.GoCryptoTlsAbi0ReadEx,
-		p.GoCryptoTlsAbi0Write,
-		p.GoCryptoTlsAbi0WriteEx,
-		p.GoCryptoTlsAbiInternalRead,
-		p.GoCryptoTlsAbiInternalReadEx,
-		p.GoCryptoTlsAbiInternalWrite,
-		p.GoCryptoTlsAbiInternalWriteEx,
-		p.SslRead,
-		p.SslReadEx,
-		p.SslRetRead,
-		p.SslRetReadEx,
-		p.SslRetWrite,
-		p.SslRetWriteEx,
-		p.SslWrite,
-		p.SslWriteEx,
-		p.SysEnterAccept4,
-		p.SysEnterConnect,
-		p.SysEnterRead,
-		p.SysEnterWrite,
-		p.SysExitAccept4,
-		p.SysExitConnect,
-		p.TcpRecvmsg,
-		p.TcpSendmsg,
-	)
-}
-
-func _TlsTapper46Close(closers ...io.Closer) error {
-	for _, closer := range closers {
-		if err := closer.Close(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// Do not access this directly.
-//go:embed tlstapper46_bpfel_arm64.o
-var _TlsTapper46Bytes []byte