Merge traffic stats endpoints to one and add auto interval logic (#1174)

* close ws on open

* chech if json before parsing

* setting defualt tab reponse and missing dep

* remove redundant

* space

* PR fixes

* remove redundant

* changed order

* Revert "remove redundant"

This reverts commit 2f0bef5d33.

* revert order change

* changes

* change

* changes

Co-authored-by: Leon <>

Dockerfile

@@ -54,14 +54,14 @@ ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
 
 
 ### Builder image for x86-64 to AArch64 cross-compilation
-FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf AS builder-from-amd64-to-arm64v8
+FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-amd64-to-arm64v8
 ENV CGO_ENABLED=1 GOOS=linux
 ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap -I/work/capstone/include"
 ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64 -I/usr/xcc/aarch64-linux-musl-cross/aarch64-linux-musl/include/"
 
 
 ### Builder image for AArch64 to x86-64 cross-compilation
-FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf AS builder-from-arm64v8-to-amd64
+FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-arm64v8-to-amd64
 ENV CGO_ENABLED=1 GOOS=linux
 ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap -I/capstone/include"
 ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86  -I/usr/local/musl/x86_64-unknown-linux-musl/include/"

acceptanceTests/setup.sh

@@ -57,7 +57,6 @@ kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --
 echo "Creating rabbitmq service"
 kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
-# TODO: need to understand how to fail if address already in use
 echo "Starting proxy"
 rm -f ${PROXY_LOG}
 kubectl proxy --port=8080 > ${PROXY_LOG} &
@@ -78,7 +77,6 @@ minikube image load "${MIZU_CI_IMAGE}"
 echo "Build cli"
 cd cli && make build GIT_BRANCH=ci SUFFIX=ci
 
-# TODO: need to understand how to fail if password is asked (sudo)
 echo "Starting tunnel"
 rm -f ${TUNNEL_LOG}
 minikube tunnel > ${TUNNEL_LOG} &

acceptanceTests/testsUtils.go

@@ -6,10 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/homedir"
 	"net/http"
 	"os"
 	"os/exec"
@@ -20,6 +16,11 @@ import (
 	"testing"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+
 	"github.com/up9inc/mizu/shared"
 )
 
@@ -175,7 +176,7 @@ func ApplyKubeFilesForTest(t *testing.T, kubeContext string, namespace string, f
 	return nil
 }
 
-func ApplyKubeFile(kubeContext string, namespace string, filename string) (error) {
+func ApplyKubeFile(kubeContext string, namespace string, filename string) error {
 	cmdArgs := []string{
 		"apply",
 		"--context", kubeContext,

agent/go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
 	github.com/chanced/openapi v0.0.8
 	github.com/djherbis/atime v1.1.0
-	github.com/getkin/kin-openapi v0.89.0
 	github.com/gin-contrib/pprof v1.3.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
@@ -31,7 +30,6 @@ require (
 	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
 	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
 	github.com/wI2L/jsondiff v0.1.1
-	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
 	k8s.io/api v0.23.3
 	k8s.io/apimachinery v0.23.3
 	k8s.io/client-go v0.23.3
@@ -59,7 +57,6 @@ require (
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.2 // indirect
@@ -85,7 +82,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.14.2 // indirect
-	github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e // indirect
+	github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

agent/go.sum

@@ -211,9 +211,6 @@ github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui72
 github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
 github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/getkin/kin-openapi v0.89.0 h1:p4nagHchUKGn85z/f+pse4aSh50nIBOYjOhMIku2hiA=
-github.com/getkin/kin-openapi v0.89.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0=
 github.com/gin-contrib/pprof v1.3.0/go.mod h1:waMjT1H9b179t3CxuG1cV3DHpga6ybizwfBaM5OXaB0=
@@ -373,7 +370,6 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c
 github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
@@ -457,8 +453,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
 github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE=
-github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
+github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 h1:1KszOoXSFt0aRQ6wxxcKm7QKgfLPI0TWO47UcY/f+vA=
+github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
@@ -711,8 +707,6 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6Ut
 github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
 github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

agent/main.go

@@ -131,6 +131,7 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engin
 	routes.MetadataRoutes(ginApp)
 	routes.StatusRoutes(ginApp)
 	routes.DbRoutes(ginApp)
+	routes.ReplayRoutes(ginApp)
 
 	return ginApp
 }
@@ -155,7 +156,7 @@ func runInTapperMode() {
 
 	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
 	tapOpts := &tap.TapOpts{
-		HostMode:     hostMode,
+		HostMode: hostMode,
 	}
 
 	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)

agent/pkg/api/contract_validation.go

@@ -1,107 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/getkin/kin-openapi/openapi3"
-	"github.com/getkin/kin-openapi/openapi3filter"
-	"github.com/getkin/kin-openapi/routers"
-	legacyrouter "github.com/getkin/kin-openapi/routers/legacy"
-
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap/api"
-)
-
-const (
-	ContractNotApplicable api.ContractStatus = 0
-	ContractPassed        api.ContractStatus = 1
-	ContractFailed        api.ContractStatus = 2
-)
-
-func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, router routers.Router, err error) {
-	path := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ContractFileName)
-	bytes, err := ioutil.ReadFile(path)
-	if err != nil {
-		return
-	}
-	contractContent = string(bytes)
-	loader := &openapi3.Loader{Context: ctx}
-	doc, _ = loader.LoadFromData(bytes)
-	err = doc.Validate(ctx)
-	if err != nil {
-		return
-	}
-	router, _ = legacyrouter.NewRouter(doc)
-	return
-}
-
-func validateOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response) (isValid bool, reqErr error, resErr error) {
-	isValid = true
-	reqErr = nil
-	resErr = nil
-
-	// Find route
-	route, pathParams, err := router.FindRoute(req)
-	if err != nil {
-		return
-	}
-
-	// Validate request
-	requestValidationInput := &openapi3filter.RequestValidationInput{
-		Request:    req,
-		PathParams: pathParams,
-		Route:      route,
-	}
-	if reqErr = openapi3filter.ValidateRequest(ctx, requestValidationInput); reqErr != nil {
-		isValid = false
-	}
-
-	responseValidationInput := &openapi3filter.ResponseValidationInput{
-		RequestValidationInput: requestValidationInput,
-		Status:                 res.StatusCode,
-		Header:                 res.Header,
-	}
-
-	if res.Body != nil {
-		body, _ := ioutil.ReadAll(res.Body)
-		res.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-		responseValidationInput.SetBodyBytes(body)
-	}
-
-	// Validate response.
-	if resErr = openapi3filter.ValidateResponse(ctx, responseValidationInput); resErr != nil {
-		isValid = false
-	}
-
-	return
-}
-
-func handleOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response, contractContent string) (contract api.Contract) {
-	contract = api.Contract{
-		Content: contractContent,
-		Status:  ContractNotApplicable,
-	}
-
-	isValid, reqErr, resErr := validateOAS(ctx, doc, router, req, res)
-	if isValid {
-		contract.Status = ContractPassed
-	} else {
-		contract.Status = ContractFailed
-		if reqErr != nil {
-			contract.RequestReason = reqErr.Error()
-		} else {
-			contract.RequestReason = ""
-		}
-		if resErr != nil {
-			contract.ResponseReason = resErr.Error()
-		} else {
-			contract.ResponseReason = ""
-		}
-	}
-
-	return
-}

agent/pkg/api/entry_streamer_socket_connector.go

@@ -22,7 +22,16 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
 	if params.EnableFullEntries {
 		message, _ = models.CreateFullEntryWebSocketMessage(entry)
 	} else {
-		extension := extensionsMap[entry.Protocol.Name]
+		protocol, ok := protocolsMap[entry.ProtocolId]
+		if !ok {
+			return fmt.Errorf("protocol not found, protocol: %v", protocol)
+		}
+
+		extension, ok := extensionsMap[protocol.Name]
+		if !ok {
+			return fmt.Errorf("extension not found, extension: %v", protocol.Name)
+		}
+
 		base := extension.Dissector.Summarize(entry)
 		message, _ = models.CreateBaseEntryWebSocketMessage(base)
 	}

agent/pkg/api/main.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/agent/pkg/oas"
 	"github.com/up9inc/mizu/agent/pkg/servicemap"
 
@@ -99,43 +98,15 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
-	disableOASValidation := false
-	ctx := context.Background()
-	doc, contractContent, router, err := loadOAS(ctx)
-	if err != nil {
-		logger.Log.Infof("Disabled OAS validation: %s", err.Error())
-		disableOASValidation = true
-	}
-
 	for item := range outputItems {
 		extension := extensionsMap[item.Protocol.Name]
-		resolvedSource, resolvedDestionation, namespace := resolveIP(item.ConnectionInfo)
+		resolvedSource, resolvedDestination, namespace := resolveIP(item.ConnectionInfo)
 
-		if namespace == "" && item.Namespace != tapApi.UNKNOWN_NAMESPACE {
+		if namespace == "" && item.Namespace != tapApi.UnknownNamespace {
 			namespace = item.Namespace
 		}
 
-		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation, namespace)
-		if extension.Protocol.Name == "http" {
-			if !disableOASValidation {
-				var httpPair tapApi.HTTPRequestResponsePair
-				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
-					logger.Log.Error(err)
-				} else {
-					contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-					mizuEntry.ContractStatus = contract.Status
-					mizuEntry.ContractRequestReason = contract.RequestReason
-					mizuEntry.ContractResponseReason = contract.ResponseReason
-					mizuEntry.ContractContent = contract.Content
-				}
-			}
-
-			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
-			if err == nil {
-				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
-				mizuEntry.Rules = rules
-			}
-		}
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestination, namespace)
 
 		data, err := json.Marshal(mizuEntry)
 		if err != nil {
@@ -155,7 +126,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
 
 		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
-		oasGenerator.HandleEntry(mizuEntry)
+		oasGenerator.HandleEntry(mizuEntry, &item.Protocol)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -14,10 +14,14 @@ import (
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
-var extensionsMap map[string]*tapApi.Extension // global
+var (
+	extensionsMap map[string]*tapApi.Extension // global
+	protocolsMap  map[string]*tapApi.Protocol  //global
+)
 
-func InitExtensionsMap(ref map[string]*tapApi.Extension) {
-	extensionsMap = ref
+func InitMaps(extensions map[string]*tapApi.Extension, protocols map[string]*tapApi.Protocol) {
+	extensionsMap = extensions
+	protocolsMap = protocols
 }
 
 type EventHandlers interface {

agent/pkg/app/main.go

@@ -9,10 +9,11 @@ import (
 	"github.com/op/go-logging"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/providers"
 	"github.com/up9inc/mizu/agent/pkg/utils"
 	"github.com/up9inc/mizu/logger"
-	"github.com/up9inc/mizu/tap/dbgctl"
 	tapApi "github.com/up9inc/mizu/tap/api"
+	"github.com/up9inc/mizu/tap/dbgctl"
 	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
 	httpExt "github.com/up9inc/mizu/tap/extensions/http"
 	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
@@ -22,11 +23,13 @@ import (
 var (
 	Extensions    []*tapApi.Extension          // global
 	ExtensionsMap map[string]*tapApi.Extension // global
+	ProtocolsMap  map[string]*tapApi.Protocol  //global
 )
 
 func LoadExtensions() {
 	Extensions = make([]*tapApi.Extension, 0)
 	ExtensionsMap = make(map[string]*tapApi.Extension)
+	ProtocolsMap = make(map[string]*tapApi.Protocol)
 
 	extensionHttp := &tapApi.Extension{}
 	dissectorHttp := httpExt.NewDissector()
@@ -34,6 +37,10 @@ func LoadExtensions() {
 	extensionHttp.Dissector = dissectorHttp
 	Extensions = append(Extensions, extensionHttp)
 	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
+	protocolsHttp := dissectorHttp.GetProtocols()
+	for k, v := range protocolsHttp {
+		ProtocolsMap[k] = v
+	}
 
 	if !dbgctl.MizuTapperDisableNonHttpExtensions {
 		extensionAmqp := &tapApi.Extension{}
@@ -42,6 +49,10 @@ func LoadExtensions() {
 		extensionAmqp.Dissector = dissectorAmqp
 		Extensions = append(Extensions, extensionAmqp)
 		ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+		protocolsAmqp := dissectorAmqp.GetProtocols()
+		for k, v := range protocolsAmqp {
+			ProtocolsMap[k] = v
+		}
 
 		extensionKafka := &tapApi.Extension{}
 		dissectorKafka := kafkaExt.NewDissector()
@@ -49,6 +60,10 @@ func LoadExtensions() {
 		extensionKafka.Dissector = dissectorKafka
 		Extensions = append(Extensions, extensionKafka)
 		ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+		protocolsKafka := dissectorKafka.GetProtocols()
+		for k, v := range protocolsKafka {
+			ProtocolsMap[k] = v
+		}
 
 		extensionRedis := &tapApi.Extension{}
 		dissectorRedis := redisExt.NewDissector()
@@ -56,13 +71,18 @@ func LoadExtensions() {
 		extensionRedis.Dissector = dissectorRedis
 		Extensions = append(Extensions, extensionRedis)
 		ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
+		protocolsRedis := dissectorRedis.GetProtocols()
+		for k, v := range protocolsRedis {
+			ProtocolsMap[k] = v
+		}
 	}
 
 	sort.Slice(Extensions, func(i, j int) bool {
 		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
 	})
 
-	api.InitExtensionsMap(ExtensionsMap)
+	api.InitMaps(ExtensionsMap, ProtocolsMap)
+	providers.InitProtocolToColor(ProtocolsMap)
 }
 
 func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level, insertionFilter string) {

agent/pkg/controllers/entries_controller.go

@@ -44,10 +44,10 @@ func GetEntries(c *gin.Context) {
 	}
 
 	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
-	entries, metadata, err := entriesProvider.GetEntries(entriesRequest)
+	entryWrappers, metadata, err := entriesProvider.GetEntries(entriesRequest)
 	if !HandleEntriesError(c, err) {
 		baseEntries := make([]interface{}, 0)
-		for _, entry := range entries {
+		for _, entry := range entryWrappers {
 			baseEntries = append(baseEntries, entry.Base)
 		}
 		c.JSON(http.StatusOK, models.EntriesResponse{

agent/pkg/controllers/replay_controller.go

@@ -0,0 +1,34 @@
+package controllers
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/replay"
+	"github.com/up9inc/mizu/agent/pkg/validation"
+	"github.com/up9inc/mizu/logger"
+)
+
+const (
+	replayTimeout = 10 * time.Second
+)
+
+func ReplayRequest(c *gin.Context) {
+	logger.Log.Debug("Starting replay")
+	replayDetails := &replay.Details{}
+	if err := c.Bind(replayDetails); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Debugf("Validating replay, %v", replayDetails)
+	if err := validation.Validate(replayDetails); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Debug("Executing replay, %v", replayDetails)
+	result := replay.ExecuteRequest(replayDetails, replayTimeout)
+	c.JSON(http.StatusOK, result)
+}

agent/pkg/controllers/status_controller.go

@@ -79,8 +79,8 @@ func GetGeneralStats(c *gin.Context) {
 	c.JSON(http.StatusOK, providers.GetGeneralStats())
 }
 
-func GetAccumulativeStats(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetAccumulativeStats())
+func GetTrafficStats(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetTrafficStats())
 }
 
 func GetCurrentResolvingInformation(c *gin.Context) {

agent/pkg/dependency/container.go

@@ -1,11 +1,11 @@
 package dependency
 
-var typeIntializerMap = make(map[DependencyContainerType]func() interface{}, 0)
+var typeInitializerMap = make(map[ContainerType]func() interface{}, 0)
 
-func RegisterGenerator(name DependencyContainerType, fn func() interface{}) {
-	typeIntializerMap[name] = fn
+func RegisterGenerator(name ContainerType, fn func() interface{}) {
+	typeInitializerMap[name] = fn
 }
 
-func GetInstance(name DependencyContainerType) interface{} {
-	return typeIntializerMap[name]()
+func GetInstance(name ContainerType) interface{} {
+	return typeInitializerMap[name]()
 }

agent/pkg/dependency/type_names.go

@@ -1,12 +1,12 @@
 package dependency
 
-type DependencyContainerType string
+type ContainerType string
 
 const (
-	ServiceMapGeneratorDependency = "ServiceMapGeneratorDependency"
-	OasGeneratorDependency        = "OasGeneratorDependency"
-	EntriesInserter               = "EntriesInserter"
-	EntriesProvider               = "EntriesProvider"
-	EntriesSocketStreamer         = "EntriesSocketStreamer"
-	EntryStreamerSocketConnector  = "EntryStreamerSocketConnector"
+	ServiceMapGeneratorDependency ContainerType = "ServiceMapGeneratorDependency"
+	OasGeneratorDependency        ContainerType = "OasGeneratorDependency"
+	EntriesInserter               ContainerType = "EntriesInserter"
+	EntriesProvider               ContainerType = "EntriesProvider"
+	EntriesSocketStreamer         ContainerType = "EntriesSocketStreamer"
+	EntryStreamerSocketConnector  ContainerType = "EntryStreamerSocketConnector"
 )

agent/pkg/entries/entries_provider.go

@@ -3,11 +3,11 @@ package entries
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/app"
-	"github.com/up9inc/mizu/agent/pkg/har"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
@@ -38,11 +38,20 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 			return nil, nil, err
 		}
 
-		extension := app.ExtensionsMap[entry.Protocol.Name]
+		protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+		if !ok {
+			return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
+		}
+
+		extension, ok := app.ExtensionsMap[protocol.Name]
+		if !ok {
+			return nil, nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
+		}
+
 		base := extension.Dissector.Summarize(entry)
 
 		dataSlice = append(dataSlice, &tapApi.EntryWrapper{
-			Protocol: entry.Protocol,
+			Protocol: *protocol,
 			Data:     entry,
 			Base:     base,
 		})
@@ -68,7 +77,16 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 		return nil, errors.New(string(bytes))
 	}
 
-	extension := app.ExtensionsMap[entry.Protocol.Name]
+	protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+	if !ok {
+		return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
+	}
+
+	extension, ok := app.ExtensionsMap[protocol.Name]
+	if !ok {
+		return nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
+	}
+
 	base := extension.Dissector.Summarize(entry)
 	var representation []byte
 	representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
@@ -76,24 +94,10 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 		return nil, err
 	}
 
-	var rules []map[string]interface{}
-	var isRulesEnabled bool
-	if entry.Protocol.Name == "http" {
-		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
-		isRulesEnabled = _isRulesEnabled
-		inrec, _ := json.Marshal(rulesMatched)
-		if err := json.Unmarshal(inrec, &rules); err != nil {
-			logger.Log.Error(err)
-		}
-	}
-
 	return &tapApi.EntryWrapper{
-		Protocol:       entry.Protocol,
+		Protocol:       *protocol,
 		Representation: string(representation),
 		Data:           entry,
 		Base:           base,
-		Rules:          rules,
-		IsRulesEnabled: isRulesEnabled,
 	}, nil
 }

agent/pkg/models/models.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 
 	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/rules"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	basenine "github.com/up9inc/basenine/client/go"
@@ -143,9 +142,3 @@ type ExtendedCreator struct {
 	*har.Creator
 	Source *string `json:"_source"`
 }
-
-func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.ApplicableRules, []rules.RulesMatched, bool) {
-	resultPolicyToSend, isEnabled := rules.MatchRequestPolicy(harEntry, service)
-	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
-	return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
-}

agent/pkg/oas/oas_generator.go

@@ -6,9 +6,8 @@ import (
 	"sync"
 
 	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/tap/api"
-
 	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/tap/api"
 )
 
 var (
@@ -17,7 +16,7 @@ var (
 )
 
 type OasGeneratorSink interface {
-	HandleEntry(mizuEntry *api.Entry)
+	HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol)
 }
 
 type OasGenerator interface {
@@ -59,12 +58,12 @@ func (g *defaultOasGenerator) IsStarted() bool {
 	return g.started
 }
 
-func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
+func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol) {
 	if !g.started {
 		return
 	}
 
-	if mizuEntry.Protocol.Name == "http" {
+	if protocol.Name == "http" {
 		dest := mizuEntry.Destination.Name
 		if dest == "" {
 			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
@@ -86,7 +85,7 @@ func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
 
 		g.handleHARWithSource(entryWSource)
 	} else {
-		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, mizuEntry.Protocol.Name)
+		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, protocol.Name)
 	}
 }
 

agent/pkg/providers/stats_provider.go

@@ -2,6 +2,8 @@ package providers
 
 import (
 	"reflect"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/jinzhu/copier"
@@ -19,18 +21,17 @@ type GeneralStats struct {
 type BucketStats []*TimeFrameStatsValue
 
 type TimeFrameStatsValue struct {
-	BucketTime    time.Time
-	ProtocolStats map[string]ProtocolStats
+	BucketTime    time.Time                `json:"timestamp"`
+	ProtocolStats map[string]ProtocolStats `json:"protocols"`
 }
 
 type ProtocolStats struct {
-	MethodsStats map[string]*SizeAndEntriesCount
-	Color        string
+	MethodsStats map[string]*SizeAndEntriesCount `json:"methods"`
 }
 
 type SizeAndEntriesCount struct {
-	EntriesCount  int
-	VolumeInBytes int
+	EntriesCount  int `json:"entriesCount"`
+	VolumeInBytes int `json:"volumeInBytes"`
 }
 
 type AccumulativeStatsCounter struct {
@@ -45,72 +46,108 @@ type AccumulativeStatsProtocol struct {
 	Methods []*AccumulativeStatsCounter `json:"methods"`
 }
 
+type AccumulativeStatsProtocolTime struct {
+	ProtocolsData []*AccumulativeStatsProtocol `json:"protocols"`
+	Time          int64                        `json:"timestamp"`
+}
+
+type TrafficStatsResponse struct {
+	PieStats      []*AccumulativeStatsProtocol     `json:"pie"`
+	TimelineStats []*AccumulativeStatsProtocolTime `json:"timeline"`
+}
+
 var (
-	generalStats = GeneralStats{}
-	bucketsStats = BucketStats{}
+	generalStats      = GeneralStats{}
+	bucketsStats      = BucketStats{}
+	bucketStatsLocker = sync.Mutex{}
+	protocolToColor   = map[string]string{}
+)
+
+const (
+	InternalBucketThreshold = time.Minute * 1
+	MaxNumberOfBars         = 30
 )
 
 func ResetGeneralStats() {
 	generalStats = GeneralStats{}
 }
 
-func GetGeneralStats() GeneralStats {
-	return generalStats
+func GetGeneralStats() *GeneralStats {
+	return &generalStats
 }
 
-func GetAccumulativeStats() []*AccumulativeStatsProtocol {
-	bucketStatsCopy := BucketStats{}
-	if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
-		logger.Log.Errorf("Error while copying src stats into temporary copied object")
+func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
+	for item, value := range protocolMap {
+		protocolToColor[strings.Split(item, "/")[2]] = value.BackgroundColor
+	}
+}
+
+func GetTrafficStats() *TrafficStatsResponse {
+	bucketsStatsCopy := getBucketStatsCopy()
+	interval := calculateInterval(bucketsStatsCopy[0].BucketTime.Unix(), bucketsStatsCopy[len(bucketsStatsCopy)-1].BucketTime.Unix()) // in seconds
+
+	return &TrafficStatsResponse{
+		PieStats:      getAccumulativeStats(bucketsStatsCopy),
+		TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy, interval),
+	}
+}
+
+func calculateInterval(firstTimestamp int64, lastTimestamp int64) time.Duration {
+	validDurations := []time.Duration{
+		time.Minute,
+		time.Minute * 2,
+		time.Minute * 3,
+		time.Minute * 5,
+		time.Minute * 10,
+		time.Minute * 15,
+		time.Minute * 20,
+		time.Minute * 30,
+		time.Minute * 45,
+		time.Minute * 60,
+		time.Minute * 75,
+		time.Minute * 90,   // 1.5 minutes
+		time.Minute * 120,  // 2 hours
+		time.Minute * 150,  // 2.5 hours
+		time.Minute * 180,  // 3 hours
+		time.Minute * 240,  // 4 hours
+		time.Minute * 300,  // 5 hours
+		time.Minute * 360,  // 6 hours
+		time.Minute * 420,  // 7 hours
+		time.Minute * 480,  // 8 hours
+		time.Minute * 540,  // 9 hours
+		time.Minute * 600,  // 10 hours
+		time.Minute * 660,  // 11 hours
+		time.Minute * 720,  // 12 hours
+		time.Minute * 1440, // 24 hours
+	}
+	duration := time.Duration(lastTimestamp-firstTimestamp) * time.Second / time.Duration(MaxNumberOfBars)
+	for _, validDuration := range validDurations {
+		if validDuration-duration >= 0 {
+			return validDuration
+		}
+	}
+	return duration.Round(validDurations[len(validDurations)-1])
+
+}
+
+func getAccumulativeStats(stats BucketStats) []*AccumulativeStatsProtocol {
+	if len(stats) == 0 {
 		return make([]*AccumulativeStatsProtocol, 0)
 	}
 
-	result := make(map[string]*AccumulativeStatsProtocol, 0)
-	methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
-	for _, countersOfTimeFrame := range bucketStatsCopy {
-		for protocolName, value := range countersOfTimeFrame.ProtocolStats {
+	methodsPerProtocolAggregated := getAggregatedStats(stats)
 
-			if _, found := result[protocolName]; !found {
-				result[protocolName] = &AccumulativeStatsProtocol{
-					AccumulativeStatsCounter: AccumulativeStatsCounter{
-						Name:            protocolName,
-						EntriesCount:    0,
-						VolumeSizeBytes: 0,
-					},
-					Color: value.Color,
-				}
-			}
-			if _, found := methodsPerProtocolAggregated[protocolName]; !found {
-				methodsPerProtocolAggregated[protocolName] = map[string]*AccumulativeStatsCounter{}
-			}
+	return convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated)
+}
 
-			for method, countersValue := range value.MethodsStats {
-				if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
-					methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
-						Name:            method,
-						EntriesCount:    0,
-						VolumeSizeBytes: 0,
-					}
-				}
-
-				result[protocolName].AccumulativeStatsCounter.EntriesCount += countersValue.EntriesCount
-				methodsPerProtocolAggregated[protocolName][method].EntriesCount += countersValue.EntriesCount
-				result[protocolName].AccumulativeStatsCounter.VolumeSizeBytes += countersValue.VolumeInBytes
-				methodsPerProtocolAggregated[protocolName][method].VolumeSizeBytes += countersValue.VolumeInBytes
-			}
-		}
+func getAccumulativeStatsTiming(stats BucketStats, interval time.Duration) []*AccumulativeStatsProtocolTime {
+	if len(stats) == 0 {
+		return make([]*AccumulativeStatsProtocolTime, 0)
 	}
 
-	finalResult := make([]*AccumulativeStatsProtocol, 0)
-	for _, value := range result {
-		methodsForProtocol := make([]*AccumulativeStatsCounter, 0)
-		for _, methodValue := range methodsPerProtocolAggregated[value.Name] {
-			methodsForProtocol = append(methodsForProtocol, methodValue)
-		}
-		value.Methods = methodsForProtocol
-		finalResult = append(finalResult, value)
-	}
-	return finalResult
+	methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(interval, stats)
+
+	return convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated)
 }
 
 func EntryAdded(size int, summery *api.BaseEntry) {
@@ -129,7 +166,8 @@ func EntryAdded(size int, summery *api.BaseEntry) {
 }
 
 func addToBucketStats(size int, summery *api.BaseEntry) {
-	entryTimeBucketRounded := time.Unix(summery.Timestamp, 0).Round(time.Minute * 1)
+	entryTimeBucketRounded := getBucketFromTimeStamp(summery.Timestamp)
+
 	if len(bucketsStats) == 0 {
 		bucketsStats = append(bucketsStats, &TimeFrameStatsValue{
 			BucketTime:    entryTimeBucketRounded,
@@ -147,7 +185,6 @@ func addToBucketStats(size int, summery *api.BaseEntry) {
 	if _, found := bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation]; !found {
 		bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation] = ProtocolStats{
 			MethodsStats: map[string]*SizeAndEntriesCount{},
-			Color:        summery.Protocol.BackgroundColor,
 		}
 	}
 	if _, found := bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method]; !found {
@@ -160,3 +197,131 @@ func addToBucketStats(size int, summery *api.BaseEntry) {
 	bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method].EntriesCount += 1
 	bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method].VolumeInBytes += size
 }
+
+func getBucketFromTimeStamp(timestamp int64) time.Time {
+	entryTimeStampAsTime := time.UnixMilli(timestamp)
+	return entryTimeStampAsTime.Add(-1 * InternalBucketThreshold / 2).Round(InternalBucketThreshold)
+}
+
+func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated map[time.Time]map[string]map[string]*AccumulativeStatsCounter) []*AccumulativeStatsProtocolTime {
+	finalResult := make([]*AccumulativeStatsProtocolTime, 0)
+	for timeKey, item := range methodsPerProtocolPerTimeAggregated {
+		protocolsData := make([]*AccumulativeStatsProtocol, 0)
+		for protocolName := range item {
+			entriesCount := 0
+			volumeSizeBytes := 0
+			methods := make([]*AccumulativeStatsCounter, 0)
+			for _, methodAccData := range methodsPerProtocolPerTimeAggregated[timeKey][protocolName] {
+				entriesCount += methodAccData.EntriesCount
+				volumeSizeBytes += methodAccData.VolumeSizeBytes
+				methods = append(methods, methodAccData)
+			}
+			protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
+				AccumulativeStatsCounter: AccumulativeStatsCounter{
+					Name:            protocolName,
+					EntriesCount:    entriesCount,
+					VolumeSizeBytes: volumeSizeBytes,
+				},
+				Color:   protocolToColor[protocolName],
+				Methods: methods,
+			})
+		}
+		finalResult = append(finalResult, &AccumulativeStatsProtocolTime{
+			Time:          timeKey.UnixMilli(),
+			ProtocolsData: protocolsData,
+		})
+	}
+	return finalResult
+}
+
+func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string]map[string]*AccumulativeStatsCounter) []*AccumulativeStatsProtocol {
+	protocolsData := make([]*AccumulativeStatsProtocol, 0)
+	for protocolName, value := range methodsPerProtocolAggregated {
+		entriesCount := 0
+		volumeSizeBytes := 0
+		methods := make([]*AccumulativeStatsCounter, 0)
+		for _, methodAccData := range value {
+			entriesCount += methodAccData.EntriesCount
+			volumeSizeBytes += methodAccData.VolumeSizeBytes
+			methods = append(methods, methodAccData)
+		}
+		protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
+			AccumulativeStatsCounter: AccumulativeStatsCounter{
+				Name:            protocolName,
+				EntriesCount:    entriesCount,
+				VolumeSizeBytes: volumeSizeBytes,
+			},
+			Color:   protocolToColor[protocolName],
+			Methods: methods,
+		})
+	}
+	return protocolsData
+}
+
+func getBucketStatsCopy() BucketStats {
+	bucketStatsCopy := BucketStats{}
+	bucketStatsLocker.Lock()
+	if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
+		logger.Log.Errorf("Error while copying src stats into temporary copied object")
+		return nil
+	}
+	bucketStatsLocker.Unlock()
+	return bucketStatsCopy
+}
+
+func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
+	methodsPerProtocolPerTimeAggregated := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{}
+
+	bucketStatsIndex := len(stats) - 1
+	for bucketStatsIndex >= 0 {
+		currentBucketTime := stats[bucketStatsIndex].BucketTime
+		resultBucketRoundedKey := currentBucketTime.Add(-1 * interval / 2).Round(interval)
+
+		for protocolName, data := range stats[bucketStatsIndex].ProtocolStats {
+			for methodName, dataOfMethod := range data.MethodsStats {
+
+				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey]; !ok {
+					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey] = map[string]map[string]*AccumulativeStatsCounter{}
+				}
+				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName]; !ok {
+					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName] = map[string]*AccumulativeStatsCounter{}
+				}
+				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName]; !ok {
+					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName] = &AccumulativeStatsCounter{
+						Name:            methodName,
+						EntriesCount:    0,
+						VolumeSizeBytes: 0,
+					}
+				}
+				methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].EntriesCount += dataOfMethod.EntriesCount
+				methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].VolumeSizeBytes += dataOfMethod.VolumeInBytes
+			}
+		}
+
+		bucketStatsIndex--
+	}
+	return methodsPerProtocolPerTimeAggregated
+}
+
+func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*AccumulativeStatsCounter {
+	methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
+	for _, countersOfTimeFrame := range bucketStatsCopy {
+		for protocolName, value := range countersOfTimeFrame.ProtocolStats {
+			for method, countersValue := range value.MethodsStats {
+				if _, found := methodsPerProtocolAggregated[protocolName]; !found {
+					methodsPerProtocolAggregated[protocolName] = map[string]*AccumulativeStatsCounter{}
+				}
+				if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
+					methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
+						Name:            method,
+						EntriesCount:    0,
+						VolumeSizeBytes: 0,
+					}
+				}
+				methodsPerProtocolAggregated[protocolName][method].EntriesCount += countersValue.EntriesCount
+				methodsPerProtocolAggregated[protocolName][method].VolumeSizeBytes += countersValue.VolumeInBytes
+			}
+		}
+	}
+	return methodsPerProtocolAggregated
+}

agent/pkg/providers/stats_provider_internal_test.go

@@ -0,0 +1,299 @@
+package providers
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestGetBucketOfTimeStamp(t *testing.T) {
+	tests := map[int64]time.Time{
+		time.Date(2022, time.Month(1), 1, 10, 34, 45, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
+		time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
+		time.Date(2022, time.Month(1), 1, 10, 59, 01, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 59, 00, 0, time.Local),
+	}
+
+	for key, value := range tests {
+		t.Run(fmt.Sprintf("%v", key), func(t *testing.T) {
+
+			actual := getBucketFromTimeStamp(key)
+
+			if actual != value {
+				t.Errorf("unexpected result - expected: %v, actual: %v", value, actual)
+			}
+		})
+	}
+}
+
+func TestGetAggregatedStatsAllTime(t *testing.T) {
+	bucketStatsForTest := BucketStats{
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"kafka": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"listTopics": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"redis": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"set": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	expected := map[string]map[string]*AccumulativeStatsCounter{
+		"http": {
+			"post": {
+				Name:            "post",
+				EntriesCount:    4,
+				VolumeSizeBytes: 6,
+			},
+			"get": {
+				Name:            "get",
+				EntriesCount:    2,
+				VolumeSizeBytes: 4,
+			},
+		},
+		"kafka": {
+			"listTopics": {
+				Name:            "listTopics",
+				EntriesCount:    5,
+				VolumeSizeBytes: 6,
+			},
+		},
+		"redis": {
+			"set": {
+				Name:            "set",
+				EntriesCount:    5,
+				VolumeSizeBytes: 6,
+			},
+		},
+	}
+	actual := getAggregatedStats(bucketStatsForTest)
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
+	}
+}
+
+func TestGetAggregatedStatsFromSpecificTime(t *testing.T) {
+	bucketStatsForTest := BucketStats{
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+					},
+				},
+				"kafka": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"listTopics": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"redis": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"set": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	expected := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{
+		time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC): {
+			"http": {
+				"post": {
+					Name:            "post",
+					EntriesCount:    2,
+					VolumeSizeBytes: 3,
+				},
+				"get": {
+					Name:            "get",
+					EntriesCount:    2,
+					VolumeSizeBytes: 4,
+				},
+			},
+			"kafka": {
+				"listTopics": {
+					Name:            "listTopics",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+			"redis": {
+				"set": {
+					Name:            "set",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+		},
+	}
+	actual := getAggregatedResultTiming(time.Minute*5, bucketStatsForTest)
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
+	}
+}
+
+func TestGetAggregatedStatsFromSpecificTimeMultipleBuckets(t *testing.T) {
+	bucketStatsForTest := BucketStats{
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+					},
+				},
+				"kafka": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"listTopics": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"redis": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"set": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	expected := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{
+		time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC): {
+			"http": {
+				"get": {
+					Name:            "get",
+					EntriesCount:    1,
+					VolumeSizeBytes: 2,
+				},
+			},
+			"kafka": {
+				"listTopics": {
+					Name:            "listTopics",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+		},
+		time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC): {
+			"http": {
+				"post": {
+					Name:            "post",
+					EntriesCount:    2,
+					VolumeSizeBytes: 3,
+				},
+				"get": {
+					Name:            "get",
+					EntriesCount:    1,
+					VolumeSizeBytes: 2,
+				},
+			},
+			"redis": {
+				"set": {
+					Name:            "set",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+		},
+	}
+	actual := getAggregatedResultTiming(time.Minute, bucketStatsForTest)
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
+	}
+}

agent/pkg/providers/stats_provider_test.go

@@ -81,5 +81,4 @@ func TestEntryAddedVolume(t *testing.T) {
 			}
 		})
 	}
-
 }

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -44,9 +44,10 @@ func Set(tappedPodsToSet []*shared.PodInfo) {
 
 func GetTappedPodsStatus() []shared.TappedPodStatus {
 	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
+	tapperStatus := tappers.GetStatus()
 	for _, pod := range Get() {
 		var status string
-		if tapperStatus, ok := tappers.GetStatus()[pod.NodeName]; ok {
+		if tapperStatus, ok := tapperStatus[pod.NodeName]; ok {
 			status = strings.ToLower(tapperStatus.Status)
 		}
 
@@ -59,7 +60,7 @@ func GetTappedPodsStatus() []shared.TappedPodStatus {
 
 func SetNodeToTappedPodMap(nodeToTappedPodsMap shared.NodeToPodsMap) {
 	summary := nodeToTappedPodsMap.Summary()
-	logger.Log.Infof("Setting node to tapped pods map to %v", summary)
+	logger.Log.Debugf("Setting node to tapped pods map to %v", summary)
 
 	nodeHostToTappedPodsMap = nodeToTappedPodsMap
 }

agent/pkg/replay/replay.go

@@ -0,0 +1,184 @@
+package replay
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/up9inc/mizu/agent/pkg/app"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	mizuhttp "github.com/up9inc/mizu/tap/extensions/http"
+)
+
+var (
+	inProcessRequestsLocker = sync.Mutex{}
+	inProcessRequests       = 0
+)
+
+const maxParallelAction = 5
+
+type Details struct {
+	Method  string            `json:"method"`
+	Url     string            `json:"url"`
+	Body    string            `json:"body"`
+	Headers map[string]string `json:"headers"`
+}
+
+type Response struct {
+	Success      bool        `json:"status"`
+	Data         interface{} `json:"data"`
+	ErrorMessage string      `json:"errorMessage"`
+}
+
+func incrementCounter() bool {
+	result := false
+	inProcessRequestsLocker.Lock()
+	if inProcessRequests < maxParallelAction {
+		inProcessRequests++
+		result = true
+	}
+	inProcessRequestsLocker.Unlock()
+	return result
+}
+
+func decrementCounter() {
+	inProcessRequestsLocker.Lock()
+	inProcessRequests--
+	inProcessRequestsLocker.Unlock()
+}
+
+func getEntryFromRequestResponse(extension *tapApi.Extension, request *http.Request, response *http.Response) *tapApi.Entry {
+	captureTime := time.Now()
+
+	itemTmp := tapApi.OutputChannelItem{
+		Protocol: *extension.Protocol,
+		ConnectionInfo: &tapApi.ConnectionInfo{
+			ClientIP:   "",
+			ClientPort: "1",
+			ServerIP:   "",
+			ServerPort: "1",
+			IsOutgoing: false,
+		},
+		Capture:   "",
+		Timestamp: time.Now().UnixMilli(),
+		Pair: &tapApi.RequestResponsePair{
+			Request: tapApi.GenericMessage{
+				IsRequest:   true,
+				CaptureTime: captureTime,
+				CaptureSize: 0,
+				Payload: &mizuhttp.HTTPPayload{
+					Type: mizuhttp.TypeHttpRequest,
+					Data: request,
+				},
+			},
+			Response: tapApi.GenericMessage{
+				IsRequest:   false,
+				CaptureTime: captureTime,
+				CaptureSize: 0,
+				Payload: &mizuhttp.HTTPPayload{
+					Type: mizuhttp.TypeHttpResponse,
+					Data: response,
+				},
+			},
+		},
+	}
+
+	// Analyze is expecting an item that's marshalled and unmarshalled
+	itemMarshalled, err := json.Marshal(itemTmp)
+	if err != nil {
+		return nil
+	}
+	var finalItem *tapApi.OutputChannelItem
+	if err := json.Unmarshal(itemMarshalled, &finalItem); err != nil {
+		return nil
+	}
+
+	return extension.Dissector.Analyze(finalItem, "", "", "")
+}
+
+func ExecuteRequest(replayData *Details, timeout time.Duration) *Response {
+	if incrementCounter() {
+		defer decrementCounter()
+
+		client := &http.Client{
+			Timeout: timeout,
+		}
+
+		request, err := http.NewRequest(strings.ToUpper(replayData.Method), replayData.Url, bytes.NewBufferString(replayData.Body))
+		if err != nil {
+			return &Response{
+				Success:      false,
+				Data:         nil,
+				ErrorMessage: err.Error(),
+			}
+		}
+
+		for headerKey, headerValue := range replayData.Headers {
+			request.Header.Add(headerKey, headerValue)
+		}
+		request.Header.Add("x-mizu", uuid.New().String())
+		response, requestErr := client.Do(request)
+
+		if requestErr != nil {
+			return &Response{
+				Success:      false,
+				Data:         nil,
+				ErrorMessage: requestErr.Error(),
+			}
+		}
+
+		extension := app.ExtensionsMap["http"] // # TODO: maybe pass the extension to the function so it can be tested
+		entry := getEntryFromRequestResponse(extension, request, response)
+		base := extension.Dissector.Summarize(entry)
+		var representation []byte
+
+		// Represent is expecting an entry that's marshalled and unmarshalled
+		entryMarshalled, err := json.Marshal(entry)
+		if err != nil {
+			return &Response{
+				Success:      false,
+				Data:         nil,
+				ErrorMessage: err.Error(),
+			}
+		}
+		var entryUnmarshalled *tapApi.Entry
+		if err := json.Unmarshal(entryMarshalled, &entryUnmarshalled); err != nil {
+			return &Response{
+				Success:      false,
+				Data:         nil,
+				ErrorMessage: err.Error(),
+			}
+		}
+
+		representation, err = extension.Dissector.Represent(entryUnmarshalled.Request, entryUnmarshalled.Response)
+		if err != nil {
+			return &Response{
+				Success:      false,
+				Data:         nil,
+				ErrorMessage: err.Error(),
+			}
+		}
+
+		return &Response{
+			Success: true,
+			Data: &tapApi.EntryWrapper{
+				Protocol:       *extension.Protocol,
+				Representation: string(representation),
+				Data:           entryUnmarshalled,
+				Base:           base,
+			},
+			ErrorMessage: "",
+		}
+	} else {
+		return &Response{
+			Success:      false,
+			Data:         nil,
+			ErrorMessage: fmt.Sprintf("reached threshold of %d requests", maxParallelAction),
+		}
+	}
+}

agent/pkg/replay/replay_internal_test.go

@@ -0,0 +1,106 @@
+package replay
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"encoding/json"
+
+	"github.com/google/uuid"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	mizuhttp "github.com/up9inc/mizu/tap/extensions/http"
+)
+
+func TestValid(t *testing.T) {
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	tests := map[string]*Details{
+		"40x": {
+			Method:  "GET",
+			Url:     "http://httpbin.org/status/404",
+			Body:    "",
+			Headers: map[string]string{},
+		},
+		"20x": {
+			Method:  "GET",
+			Url:     "http://httpbin.org/status/200",
+			Body:    "",
+			Headers: map[string]string{},
+		},
+		"50x": {
+			Method:  "GET",
+			Url:     "http://httpbin.org/status/500",
+			Body:    "",
+			Headers: map[string]string{},
+		},
+		// TODO: this should be fixes, currently not working because of header name with ":"
+		//":path-header": {
+		//	Method: "GET",
+		//	Url:    "http://httpbin.org/get",
+		//	Body:   "",
+		//	Headers: map[string]string{
+		//		":path": "/get",
+		//	},
+		// },
+	}
+
+	for testCaseName, replayData := range tests {
+		t.Run(fmt.Sprintf("%+v", testCaseName), func(t *testing.T) {
+			request, err := http.NewRequest(strings.ToUpper(replayData.Method), replayData.Url, bytes.NewBufferString(replayData.Body))
+			if err != nil {
+				t.Errorf("Error executing request")
+			}
+
+			for headerKey, headerValue := range replayData.Headers {
+				request.Header.Add(headerKey, headerValue)
+			}
+			request.Header.Add("x-mizu", uuid.New().String())
+			response, requestErr := client.Do(request)
+
+			if requestErr != nil {
+				t.Errorf("failed: %v, ", requestErr)
+			}
+
+			extensionHttp := &tapApi.Extension{}
+			dissectorHttp := mizuhttp.NewDissector()
+			dissectorHttp.Register(extensionHttp)
+			extensionHttp.Dissector = dissectorHttp
+			extension := extensionHttp
+
+			entry := getEntryFromRequestResponse(extension, request, response)
+			base := extension.Dissector.Summarize(entry)
+
+			// Represent is expecting an entry that's marshalled and unmarshalled
+			entryMarshalled, err := json.Marshal(entry)
+			if err != nil {
+				t.Errorf("failed marshaling entry: %v, ", err)
+			}
+			var entryUnmarshalled *tapApi.Entry
+			if err := json.Unmarshal(entryMarshalled, &entryUnmarshalled); err != nil {
+				t.Errorf("failed unmarshaling entry: %v, ", err)
+			}
+
+			var representation []byte
+			representation, err = extension.Dissector.Represent(entryUnmarshalled.Request, entryUnmarshalled.Response)
+			if err != nil {
+				t.Errorf("failed: %v, ", err)
+			}
+
+			result := &tapApi.EntryWrapper{
+				Protocol:       *extension.Protocol,
+				Representation: string(representation),
+				Data:           entry,
+				Base:           base,
+			}
+			t.Logf("%+v", result)
+			//data, _ := json.MarshalIndent(result, "", "  ")
+			//t.Logf("%+v", string(data))
+		})
+	}
+}

agent/pkg/resolver/go.sum

@@ -1,522 +0,0 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE=
-github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
-github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1 h1:ocYkMQY5RrXTYgXl7ICpV0IXwlEQGwKIsery4gyXa1U=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7PEeBTyK1rj1gABdCO2mbri6GKO1cMDs=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
-gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y=
-k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU=
-k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA=
-k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
-k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag=
-k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac h1:sAvhNk5RRuc6FNYGqe7Ygz3PSo/2wGWbulskmzRX8Vs=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

agent/pkg/resolver/loader.go

@@ -15,9 +15,9 @@ func NewFromInCluster(errOut chan error, namespace string) (*Resolver, error) {
 	if err != nil {
 		return nil, err
 	}
-	clientset, err := kubernetes.NewForConfig(config)
+	clientSet, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		return nil, err
 	}
-	return &Resolver{clientConfig: config, clientSet: clientset, nameMap: cmap.New(), serviceMap: cmap.New(), errOut: errOut, namespace: namespace}, nil
+	return &Resolver{clientConfig: config, clientSet: clientSet, nameMap: cmap.New(), serviceMap: cmap.New(), errOut: errOut, namespace: namespace}, nil
 }

agent/pkg/routes/replay_routes.go

@@ -0,0 +1,13 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+)
+
+// ReplayRoutes defines the group of replay routes.
+func ReplayRoutes(app *gin.Engine) {
+	routeGroup := app.Group("/replay")
+
+	routeGroup.POST("/", controllers.ReplayRequest)
+}

agent/pkg/routes/status_routes.go

@@ -15,8 +15,8 @@ func StatusRoutes(ginApp *gin.Engine) {
 	routeGroup.GET("/connectedTappersCount", controllers.GetConnectedTappersCount)
 	routeGroup.GET("/tap", controllers.GetTappingStatus)
 
-	routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB
-	routeGroup.GET("/accumulative", controllers.GetAccumulativeStats) // get general stats about entries in DB
+	routeGroup.GET("/general", controllers.GetGeneralStats)
+	routeGroup.GET("/trafficStats", controllers.GetTrafficStats)
 
 	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
 }

agent/pkg/rules/rulesHTTP.go

@@ -1,124 +0,0 @@
-package rules
-
-import (
-	"encoding/base64"
-	"encoding/json"
-	"fmt"
-	"reflect"
-	"regexp"
-	"strings"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-
-	"github.com/up9inc/mizu/logger"
-
-	"github.com/up9inc/mizu/shared"
-	"github.com/yalp/jsonpath"
-)
-
-type RulesMatched struct {
-	Matched bool              `json:"matched"`
-	Rule    shared.RulePolicy `json:"rule"`
-}
-
-func appendRulesMatched(rulesMatched []RulesMatched, matched bool, rule shared.RulePolicy) []RulesMatched {
-	return append(rulesMatched, RulesMatched{Matched: matched, Rule: rule})
-}
-
-func ValidatePath(URLFromRule string, URL string) bool {
-	if URLFromRule != "" {
-		matchPath, err := regexp.MatchString(URLFromRule, URL)
-		if err != nil || !matchPath {
-			return false
-		}
-	}
-	return true
-}
-
-func ValidateService(serviceFromRule string, service string) bool {
-	if serviceFromRule != "" {
-		matchService, err := regexp.MatchString(serviceFromRule, service)
-		if err != nil || !matchService {
-			return false
-		}
-	}
-	return true
-}
-
-func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend []RulesMatched, isEnabled bool) {
-	enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ValidationRulesFileName))
-	if err == nil && len(enforcePolicy.Rules) > 0 {
-		isEnabled = true
-	}
-	for _, rule := range enforcePolicy.Rules {
-		if !ValidatePath(rule.Path, harEntry.Request.URL) || !ValidateService(rule.Service, service) {
-			continue
-		}
-		if rule.Type == "json" {
-			var bodyJsonMap interface{}
-			contentTextDecoded, _ := base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text))
-			if err := json.Unmarshal(contentTextDecoded, &bodyJsonMap); err != nil {
-				continue
-			}
-			out, err := jsonpath.Read(bodyJsonMap, rule.Key)
-			if err != nil || out == nil {
-				continue
-			}
-			var matchValue bool
-			if reflect.TypeOf(out).Kind() == reflect.String {
-				matchValue, err = regexp.MatchString(rule.Value, out.(string))
-				if err != nil {
-					continue
-				}
-				logger.Log.Info(matchValue, rule.Value)
-			} else {
-				val := fmt.Sprint(out)
-				matchValue, err = regexp.MatchString(rule.Value, val)
-				if err != nil {
-					continue
-				}
-			}
-			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
-		} else if rule.Type == "header" {
-			for j := range harEntry.Response.Headers {
-				matchKey, err := regexp.MatchString(rule.Key, harEntry.Response.Headers[j].Name)
-				if err != nil {
-					continue
-				}
-				if matchKey {
-					matchValue, err := regexp.MatchString(rule.Value, harEntry.Response.Headers[j].Value)
-					if err != nil {
-						continue
-					}
-					resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
-				}
-			}
-		} else {
-			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, true, rule)
-		}
-	}
-	return
-}
-
-func PassedValidationRules(rulesMatched []RulesMatched) (bool, int64, int) {
-	var numberOfRulesMatched = len(rulesMatched)
-	var responseTime int64 = -1
-
-	if numberOfRulesMatched == 0 {
-		return false, 0, numberOfRulesMatched
-	}
-
-	for _, rule := range rulesMatched {
-		if !rule.Matched {
-			return false, responseTime, numberOfRulesMatched
-		} else {
-			if strings.ToLower(rule.Rule.Type) == "slo" {
-				if rule.Rule.ResponseTime < responseTime || responseTime == -1 {
-					responseTime = rule.Rule.ResponseTime
-				}
-			}
-		}
-	}
-
-	return true, responseTime, numberOfRulesMatched
-}

agent/pkg/utils/pathUtils.go

@@ -1,8 +1,9 @@
 package utils
 
 import (
-	"github.com/djherbis/atime"
 	"os"
+
+	"github.com/djherbis/atime"
 )
 
 type ByModTime []os.FileInfo
@@ -19,7 +20,6 @@ func (fis ByModTime) Less(i, j int) bool {
 	return fis[i].ModTime().Before(fis[j].ModTime())
 }
 
-
 type ByName []os.FileInfo
 
 func (fis ByName) Len() int {
@@ -34,7 +34,6 @@ func (fis ByName) Less(i, j int) bool {
 	return fis[i].Name() < fis[j].Name()
 }
 
-
 type ByCreationTime []os.FileInfo
 
 func (fis ByCreationTime) Len() int {
@@ -47,4 +46,4 @@ func (fis ByCreationTime) Swap(i, j int) {
 
 func (fis ByCreationTime) Less(i, j int) bool {
 	return atime.Get(fis[i]).Unix() < atime.Get(fis[j]).Unix()
-}
+}

agent/pkg/utils/utils.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
@@ -59,17 +58,6 @@ func CheckErr(e error) {
 	}
 }
 
-func SetHostname(address, newHostname string) string {
-	replacedUrl, err := url.Parse(address)
-	if err != nil {
-		logger.Log.Errorf("error replacing hostname to %s in address %s, returning original %v", newHostname, address, err)
-		return address
-	}
-	replacedUrl.Host = newHostname
-	return replacedUrl.String()
-
-}
-
 func ReadJsonFile(filePath string, value interface{}) error {
 	if content, err := ioutil.ReadFile(filePath); err != nil {
 		return err

cli/cmd/tap.go

@@ -53,9 +53,8 @@ func init() {
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
-	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
-	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
 	tapCmd.Flags().Bool(configStructs.ServiceMeshName, defaultTapConfig.ServiceMesh, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls")
 	tapCmd.Flags().Bool(configStructs.TlsName, defaultTapConfig.Tls, "Record tls traffic")
 	tapCmd.Flags().Bool(configStructs.ProfilerName, defaultTapConfig.Profiler, "Run pprof server")
+	tapCmd.Flags().Int(configStructs.MaxLiveStreamsName, defaultTapConfig.MaxLiveStreams, "Maximum live tcp streams to handle concurrently")
 }

cli/cmd/tapRunner.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"regexp"
 	"strings"
 	"time"
@@ -13,8 +12,6 @@ import (
 	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/utils"
 
-	"github.com/getkin/kin-openapi/openapi3"
-	"gopkg.in/yaml.v3"
 	core "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -47,40 +44,6 @@ func RunMizuTap() {
 
 	apiProvider = apiserver.NewProvider(GetApiServerUrl(config.Config.Tap.GuiPort), apiserver.DefaultRetries, apiserver.DefaultTimeout)
 
-	var err error
-	var serializedValidationRules string
-	if config.Config.Tap.EnforcePolicyFile != "" {
-		serializedValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
-			return
-		}
-	}
-
-	// Read and validate the OAS file
-	var serializedContract string
-	if config.Config.Tap.ContractFile != "" {
-		bytes, err := ioutil.ReadFile(config.Config.Tap.ContractFile)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading contract file: %v", errormessage.FormatError(err)))
-			return
-		}
-		serializedContract = string(bytes)
-
-		ctx := context.Background()
-		loader := &openapi3.Loader{Context: ctx}
-		doc, err := loader.LoadFromData(bytes)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error loading contract file: %v", errormessage.FormatError(err)))
-			return
-		}
-		err = doc.Validate(ctx)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error validating contract file: %v", errormessage.FormatError(err)))
-			return
-		}
-	}
-
 	kubernetesProvider, err := getKubernetesProviderForCli()
 	if err != nil {
 		return
@@ -124,7 +87,7 @@ func RunMizuTap() {
 	}
 
 	logger.Log.Infof("Waiting for Mizu Agent to start...")
-	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), config.Config.Tap.Profiler); err != nil {
+	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), config.Config.Tap.Profiler); err != nil {
 		var statusError *k8serrors.StatusError
 		if errors.As(err, &statusError) && (statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists) {
 			logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
@@ -202,6 +165,7 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
 		ServiceMesh:              config.Config.Tap.ServiceMesh,
 		Tls:                      config.Config.Tap.Tls,
+		MaxLiveStreams:           config.Config.Tap.MaxLiveStreams,
 	}, startTime)
 
 	if err != nil {
@@ -265,15 +229,6 @@ func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError)
 	}
 }
 
-func readValidationRules(file string) (string, error) {
-	rules, err := shared.DecodeEnforcePolicy(file)
-	if err != nil {
-		return "", err
-	}
-	newContent, _ := yaml.Marshal(&rules)
-	return string(newContent), nil
-}
-
 func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
 	var compiledRegexSlice []*api.SerializableRegexp
 

cli/config/configStructs/tapConfig.go

@@ -23,11 +23,10 @@ const (
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	InsertionFilterName           = "insertion-filter"
 	DryRunTapName                 = "dry-run"
-	EnforcePolicyFile             = "traffic-validation-file"
-	ContractFile                  = "contract"
 	ServiceMeshName               = "service-mesh"
 	TlsName                       = "tls"
 	ProfilerName                  = "profiler"
+	MaxLiveStreamsName            = "max-live-streams"
 )
 
 type TapConfig struct {
@@ -42,13 +41,12 @@ type TapConfig struct {
 	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
 	InsertionFilter        string           `yaml:"insertion-filter" default:""`
 	DryRun                 bool             `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile      string           `yaml:"traffic-validation-file"`
-	ContractFile           string           `yaml:"contract"`
 	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
 	TapperResources        shared.Resources `yaml:"tapper-resources"`
 	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
 	Tls                    bool             `yaml:"tls" default:"false"`
 	Profiler               bool             `yaml:"profiler" default:"false"`
+	MaxLiveStreams         int              `yaml:"max-live-streams" default:"500"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

cli/go.mod

@@ -5,16 +5,13 @@ go 1.17
 require (
 	github.com/creasty/defaults v1.5.2
 	github.com/denisbrodbeck/machineid v1.0.1
-	github.com/getkin/kin-openapi v0.89.0
 	github.com/google/go-github/v37 v37.0.0
-	github.com/google/uuid v1.3.0
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.23.3
 	k8s.io/apimachinery v0.23.3
@@ -39,7 +36,6 @@ require (
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
@@ -52,8 +48,8 @@ require (
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
@@ -79,6 +75,7 @@ require (
 	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
 	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect

cli/go.sum

@@ -190,9 +190,6 @@ github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui72
 github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
 github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/getkin/kin-openapi v0.89.0 h1:p4nagHchUKGn85z/f+pse4aSh50nIBOYjOhMIku2hiA=
-github.com/getkin/kin-openapi v0.89.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -294,7 +291,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=

cli/mizu/version/versionCheck.go

@@ -10,32 +10,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/pkg/version"
 	"github.com/up9inc/mizu/logger"
 
 	"github.com/google/go-github/v37/github"
-	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
-func CheckVersionCompatibility(apiServerProvider *apiserver.Provider) (bool, error) {
-	apiVer, err := apiServerProvider.GetVersion()
-	if err != nil {
-		return false, err
-	}
-
-	if equals, err := version.AreEquals(apiVer, mizu.Ver); err != nil {
-		return false, fmt.Errorf("Failed to check version equality between mizuVer: %s and apiVer: %s, error: %w", mizu.Ver, apiVer, err)
-	} else if !equals {
-		logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.Ver, apiVer))
-		return false, nil
-	}
-
-	logger.Log.Debug("cli version %s is compatible with api version %s", mizu.Ver, apiVer)
-	return true, nil
-}
-
 func CheckNewerVersion(versionChan chan string) {
 	if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
 		versionChan <- ""

cli/resources/createResources.go

@@ -14,14 +14,14 @@ import (
 	core "k8s.io/api/core/v1"
 )
 
-func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, profiler bool) (bool, error) {
+func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, profiler bool) (bool, error) {
 	if !isNsRestrictedMode {
 		if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 			return false, err
 		}
 	}
 
-	if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil {
+	if err := createMizuConfigmap(ctx, kubernetesProvider, serializedMizuConfig, mizuResourcesNamespace); err != nil {
 		return false, err
 	}
 
@@ -71,8 +71,8 @@ func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	return err
 }
 
-func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, mizuResourcesNamespace string) error {
-	err := kubernetesProvider.CreateConfigMap(ctx, mizuResourcesNamespace, kubernetes.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig)
+func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedMizuConfig string, mizuResourcesNamespace string) error {
+	err := kubernetesProvider.CreateConfigMap(ctx, mizuResourcesNamespace, kubernetes.ConfigMapName, serializedMizuConfig)
 	return err
 }
 

cli/telemetry/telemetry.go

@@ -73,11 +73,7 @@ func shouldRunTelemetry() bool {
 		return false
 	}
 
-	if mizu.Branch != "main" && mizu.Branch != "develop" {
-		return false
-	}
-
-	return true
+	return mizu.Branch == "main" || mizu.Branch == "develop"
 }
 
 func sendTelemetry(argsMap map[string]interface{}) error {

devops/install-capstone.sh

@@ -5,8 +5,8 @@ if (( $EUID != 0 )); then
     SUDO='sudo'
 fi
 
-curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \
- && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone \
+curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \
+ && tar -xf capstone.tar.xz && mv ./capstone-* ./capstone \
  && cd capstone \
  && CAPSTONE_ARCHS="aarch64 x86" ./make.sh \
  && $SUDO ./make.sh install

devops/linux-arm64-musl-go-libpcap-capstone-bpf/Dockerfile

@@ -20,8 +20,8 @@ RUN ./configure --host=arm && make \
 WORKDIR /work
 
 # Build and install Capstone from source
-RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \
- && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone
+RUN curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \
+ && tar -xf capstone.tar.xz && mv ./capstone-* ./capstone
 WORKDIR /work/capstone
 RUN CAPSTONE_ARCHS="aarch64" CAPSTONE_STATIC=yes ./make.sh \
 && cp /work/capstone/libcapstone.a /usr/xcc/aarch64-linux-musl-cross/lib/gcc/aarch64-linux-musl/*/

devops/linux-arm64-musl-go-libpcap-capstone-bpf/build-push.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 set -e
 
-docker build . -t up9inc/linux-arm64-musl-go-libpcap-capstone-bpf && docker push up9inc/linux-arm64-musl-go-libpcap-capstone-bpf
+# Build it on x86_64
+docker build . -t up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 && docker push up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2

devops/linux-x86_64-musl-go-libpcap-capstone-bpf/Dockerfile

@@ -44,8 +44,8 @@ RUN ./configure --host=x86_64 && make \
 WORKDIR /
 
 # Build and install Capstone from source
-RUN curl https://github.com/capstone-engine/capstone/archive/4.0.2.tar.gz -Lo ./capstone.tar.gz \
- && tar -xzf capstone.tar.gz && mv ./capstone-* ./capstone
+RUN curl https://github.com/capstone-engine/capstone/releases/download/5.0-rc2/capstone-5.0-rc2.tar.xz -Lo ./capstone.tar.xz \
+ && tar -xf capstone.tar.xz && mv ./capstone-* ./capstone
 WORKDIR /capstone
 RUN CAPSTONE_ARCHS="x86" CAPSTONE_STATIC=yes ./make.sh \
  && cp /capstone/libcapstone.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/

devops/linux-x86_64-musl-go-libpcap-capstone-bpf/build-push.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 set -e
 
-docker build . -t up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf && docker push up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf
+# Build it on arm64
+docker build . -t up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 && docker push up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2

shared/consts.go

@@ -6,8 +6,6 @@ const (
 	NodeNameEnvVar             = "NODE_NAME"
 	ConfigDirPath              = "/app/config/"
 	DataDirPath                = "/app/data/"
-	ValidationRulesFileName    = "validation-rules.yaml"
-	ContractFileName           = "contract-oas.yaml"
 	ConfigFileName             = "mizu-config.json"
 	DefaultApiServerPort       = 8899
 	LogLevelEnvVar             = "LOG_LEVEL"

shared/go.mod

@@ -4,11 +4,9 @@ go 1.17
 
 require (
 	github.com/docker/go-units v0.4.0
-	github.com/golang-jwt/jwt/v4 v4.2.0
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.23.3
 	k8s.io/apimachinery v0.23.3
 	k8s.io/client-go v0.23.3
@@ -38,11 +36,11 @@ require (
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
@@ -81,6 +79,7 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/cli-runtime v0.23.3 // indirect
 	k8s.io/component-base v0.23.3 // indirect
 	k8s.io/klog/v2 v2.40.1 // indirect

shared/go.sum

@@ -282,7 +282,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=

shared/kubernetes/mizuTapperSyncer.go

@@ -48,6 +48,7 @@ type TapperSyncerConfig struct {
 	MizuServiceAccountExists bool
 	ServiceMesh              bool
 	Tls                      bool
+	MaxLiveStreams           int
 }
 
 func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig, startTime time.Time) (*MizuTapperSyncer, error) {
@@ -337,7 +338,8 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
 			tapperSyncer.config.MizuApiFilteringOptions,
 			tapperSyncer.config.LogLevel,
 			tapperSyncer.config.ServiceMesh,
-			tapperSyncer.config.Tls); err != nil {
+			tapperSyncer.config.Tls,
+			tapperSyncer.config.MaxLiveStreams); err != nil {
 			return err
 		}
 

shared/kubernetes/provider.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"path/filepath"
 	"regexp"
+	"strconv"
 
 	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/logger"
@@ -382,11 +383,11 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 			Tolerations: []core.Toleration{
 				{
 					Operator: core.TolerationOpExists,
-					Effect: core.TaintEffectNoExecute,
+					Effect:   core.TaintEffectNoExecute,
 				},
 				{
 					Operator: core.TolerationOpExists,
-					Effect: core.TaintEffectNoSchedule,
+					Effect:   core.TaintEffectNoSchedule,
 				},
 			},
 		},
@@ -684,14 +685,8 @@ func (provider *Provider) handleRemovalError(err error) error {
 	return err
 }
 
-func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
+func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, serializedMizuConfig string) error {
 	configMapData := make(map[string]string)
-	if serializedValidationRules != "" {
-		configMapData[shared.ValidationRulesFileName] = serializedValidationRules
-	}
-	if serializedContract != "" {
-		configMapData[shared.ContractFileName] = serializedContract
-	}
 	configMapData[shared.ConfigFileName] = serializedMizuConfig
 
 	configMap := &core.ConfigMap{
@@ -714,7 +709,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeNames []string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool, tls bool) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeNames []string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool, tls bool, maxLiveStreams int) error {
 	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeNames), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeNames) == 0 {
@@ -732,6 +727,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"--tap",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
 		"--nodefrag",
+		"--max-live-streams", strconv.Itoa(maxLiveStreams),
 	}
 
 	if serviceMesh {

shared/kubernetes/watchEvent.go

@@ -11,11 +11,11 @@ import (
 )
 
 const (
-	EventAdded    watch.EventType = watch.Added
-	EventModified watch.EventType = watch.Modified
-	EventDeleted  watch.EventType = watch.Deleted
-	EventBookmark watch.EventType = watch.Bookmark
-	EventError    watch.EventType = watch.Error
+	EventAdded    = watch.Added
+	EventModified = watch.Modified
+	EventDeleted  = watch.Deleted
+	EventBookmark = watch.Bookmark
+	EventError    = watch.Error
 )
 
 type InvalidObjectType struct {

shared/models.go

@@ -1,13 +1,8 @@
 package shared
 
 import (
-	"io/ioutil"
-	"strings"
-
 	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/logger"
 
-	"gopkg.in/yaml.v3"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -55,7 +50,6 @@ type WebSocketMessageMetadata struct {
 	MessageType WebSocketMessageType `json:"messageType,omitempty"`
 }
 
-
 type WebSocketStatusMessage struct {
 	*WebSocketMessageMetadata
 	TappingStatus []TappedPodStatus `json:"tappingStatus"`
@@ -136,83 +130,3 @@ type HealthResponse struct {
 type VersionResponse struct {
 	Ver string `json:"ver"`
 }
-
-type RulesPolicy struct {
-	Rules []RulePolicy `yaml:"rules"`
-}
-
-type RulePolicy struct {
-	Type         string `yaml:"type"`
-	Service      string `yaml:"service"`
-	Path         string `yaml:"path"`
-	Method       string `yaml:"method"`
-	Key          string `yaml:"key"`
-	Value        string `yaml:"value"`
-	ResponseTime int64  `yaml:"response-time"`
-	Name         string `yaml:"name"`
-}
-
-type RulesMatched struct {
-	Matched bool       `json:"matched"`
-	Rule    RulePolicy `json:"rule"`
-}
-
-func (r *RulePolicy) validateType() bool {
-	permitedTypes := []string{"json", "header", "slo"}
-	_, found := Find(permitedTypes, r.Type)
-	if !found {
-		logger.Log.Errorf("Only json, header and slo types are supported on rule definition. This rule will be ignored. rule name: %s", r.Name)
-		found = false
-	}
-	if strings.ToLower(r.Type) == "slo" {
-		if r.ResponseTime <= 0 {
-			logger.Log.Errorf("When rule type is slo, the field response-time should be specified and have a value >= 1. rule name: %s", r.Name)
-			found = false
-		}
-	}
-	return found
-}
-
-func (rules *RulesPolicy) ValidateRulesPolicy() []int {
-	invalidIndex := make([]int, 0)
-	for i := range rules.Rules {
-		validated := rules.Rules[i].validateType()
-		if !validated {
-			invalidIndex = append(invalidIndex, i)
-		}
-	}
-	return invalidIndex
-}
-
-func Find(slice []string, val string) (int, bool) {
-	for i, item := range slice {
-		if item == val {
-			return i, true
-		}
-	}
-	return -1, false
-}
-
-func DecodeEnforcePolicy(path string) (RulesPolicy, error) {
-	content, err := ioutil.ReadFile(path)
-	enforcePolicy := RulesPolicy{}
-	if err != nil {
-		return enforcePolicy, err
-	}
-	err = yaml.Unmarshal([]byte(content), &enforcePolicy)
-	if err != nil {
-		return enforcePolicy, err
-	}
-	invalidIndex := enforcePolicy.ValidateRulesPolicy()
-	var k = 0
-	if len(invalidIndex) != 0 {
-		for i, rule := range enforcePolicy.Rules {
-			if !ContainsInt(invalidIndex, i) {
-				enforcePolicy.Rules[k] = rule
-				k++
-			}
-		}
-		enforcePolicy.Rules = enforcePolicy.Rules[:k]
-	}
-	return enforcePolicy, nil
-}

shared/sliceUtils.go

@@ -10,15 +10,6 @@ func Contains(slice []string, containsValue string) bool {
 	return false
 }
 
-func ContainsInt(slice []int, containsValue int) bool {
-	for _, sliceValue := range slice {
-		if sliceValue == containsValue {
-			return true
-		}
-	}
-	return false
-}
-
 func Unique(slice []string) []string {
 	keys := make(map[string]bool)
 	var list []string

tap/api/api.go

@@ -2,27 +2,16 @@ package api
 
 import (
 	"bufio"
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"net"
-	"net/http"
-	"os"
-	"sort"
 	"sync"
 	"time"
 
-	"github.com/google/martian/har"
-
 	"github.com/up9inc/mizu/tap/dbgctl"
 )
 
-const mizuTestEnvVar = "MIZU_TEST"
-const UNKNOWN_NAMESPACE = ""
+const UnknownNamespace = ""
 
-var UnknownIp net.IP = net.IP{0, 0, 0, 0}
+var UnknownIp = net.IP{0, 0, 0, 0}
 var UnknownPort uint16 = 0
 
 type Protocol struct {
@@ -102,7 +91,6 @@ type OutputChannelItem struct {
 	Timestamp      int64
 	ConnectionInfo *ConnectionInfo
 	Pair           *RequestResponsePair
-	Summary        *BaseEntry
 	Namespace      string
 }
 
@@ -127,6 +115,7 @@ func (p *ReadProgress) Reset() {
 
 type Dissector interface {
 	Register(*Extension)
+	GetProtocols() map[string]*Protocol
 	Ping()
 	Dissect(b *bufio.Reader, reader TcpReader, options *TrafficFilteringOptions) error
 	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *Entry
@@ -161,69 +150,44 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 }
 
 type Entry struct {
-	Id                     string                 `json:"id"`
-	Protocol               Protocol               `json:"proto"`
-	Capture                Capture                `json:"capture"`
-	Source                 *TCP                   `json:"src"`
-	Destination            *TCP                   `json:"dst"`
-	Namespace              string                 `json:"namespace"`
-	Outgoing               bool                   `json:"outgoing"`
-	Timestamp              int64                  `json:"timestamp"`
-	StartTime              time.Time              `json:"startTime"`
-	Request                map[string]interface{} `json:"request"`
-	Response               map[string]interface{} `json:"response"`
-	RequestSize            int                    `json:"requestSize"`
-	ResponseSize           int                    `json:"responseSize"`
-	ElapsedTime            int64                  `json:"elapsedTime"`
-	Rules                  ApplicableRules        `json:"rules,omitempty"`
-	ContractStatus         ContractStatus         `json:"contractStatus,omitempty"`
-	ContractRequestReason  string                 `json:"contractRequestReason,omitempty"`
-	ContractResponseReason string                 `json:"contractResponseReason,omitempty"`
-	ContractContent        string                 `json:"contractContent,omitempty"`
-	HTTPPair               string                 `json:"httpPair,omitempty"`
+	Id           string                 `json:"id"`
+	ProtocolId   string                 `json:"protocol"`
+	Capture      Capture                `json:"capture"`
+	Source       *TCP                   `json:"src"`
+	Destination  *TCP                   `json:"dst"`
+	Namespace    string                 `json:"namespace"`
+	Outgoing     bool                   `json:"outgoing"`
+	Timestamp    int64                  `json:"timestamp"`
+	StartTime    time.Time              `json:"startTime"`
+	Request      map[string]interface{} `json:"request"`
+	Response     map[string]interface{} `json:"response"`
+	RequestSize  int                    `json:"requestSize"`
+	ResponseSize int                    `json:"responseSize"`
+	ElapsedTime  int64                  `json:"elapsedTime"`
 }
 
 type EntryWrapper struct {
-	Protocol       Protocol                 `json:"protocol"`
-	Representation string                   `json:"representation"`
-	Data           *Entry                   `json:"data"`
-	Base           *BaseEntry               `json:"base"`
-	Rules          []map[string]interface{} `json:"rulesMatched,omitempty"`
-	IsRulesEnabled bool                     `json:"isRulesEnabled"`
+	Protocol       Protocol   `json:"protocol"`
+	Representation string     `json:"representation"`
+	Data           *Entry     `json:"data"`
+	Base           *BaseEntry `json:"base"`
 }
 
 type BaseEntry struct {
-	Id             string          `json:"id"`
-	Protocol       Protocol        `json:"proto,omitempty"`
-	Capture        Capture         `json:"capture"`
-	Summary        string          `json:"summary,omitempty"`
-	SummaryQuery   string          `json:"summaryQuery,omitempty"`
-	Status         int             `json:"status"`
-	StatusQuery    string          `json:"statusQuery"`
-	Method         string          `json:"method,omitempty"`
-	MethodQuery    string          `json:"methodQuery,omitempty"`
-	Timestamp      int64           `json:"timestamp,omitempty"`
-	Source         *TCP            `json:"src"`
-	Destination    *TCP            `json:"dst"`
-	IsOutgoing     bool            `json:"isOutgoing,omitempty"`
-	Latency        int64           `json:"latency"`
-	Rules          ApplicableRules `json:"rules,omitempty"`
-	ContractStatus ContractStatus  `json:"contractStatus"`
-}
-
-type ApplicableRules struct {
-	Latency       int64 `json:"latency,omitempty"`
-	Status        bool  `json:"status,omitempty"`
-	NumberOfRules int   `json:"numberOfRules,omitempty"`
-}
-
-type ContractStatus int
-
-type Contract struct {
-	Status         ContractStatus `json:"status"`
-	RequestReason  string         `json:"requestReason"`
-	ResponseReason string         `json:"responseReason"`
-	Content        string         `json:"content"`
+	Id           string   `json:"id"`
+	Protocol     Protocol `json:"proto,omitempty"`
+	Capture      Capture  `json:"capture"`
+	Summary      string   `json:"summary,omitempty"`
+	SummaryQuery string   `json:"summaryQuery,omitempty"`
+	Status       int      `json:"status"`
+	StatusQuery  string   `json:"statusQuery"`
+	Method       string   `json:"method,omitempty"`
+	MethodQuery  string   `json:"methodQuery,omitempty"`
+	Timestamp    int64    `json:"timestamp,omitempty"`
+	Source       *TCP     `json:"src"`
+	Destination  *TCP     `json:"dst"`
+	IsOutgoing   bool     `json:"isOutgoing,omitempty"`
+	Latency      int64    `json:"latency"`
 }
 
 const (
@@ -246,170 +210,6 @@ type TableData struct {
 	Selector string      `json:"selector"`
 }
 
-const (
-	TypeHttpRequest = iota
-	TypeHttpResponse
-)
-
-type HTTPPayload struct {
-	Type uint8
-	Data interface{}
-}
-
-type HTTPPayloader interface {
-	MarshalJSON() ([]byte, error)
-}
-
-type HTTPWrapper struct {
-	Method      string               `json:"method"`
-	Url         string               `json:"url"`
-	Details     interface{}          `json:"details"`
-	RawRequest  *HTTPRequestWrapper  `json:"rawRequest"`
-	RawResponse *HTTPResponseWrapper `json:"rawResponse"`
-}
-
-func (h HTTPPayload) MarshalJSON() ([]byte, error) {
-	_, testEnvEnabled := os.LookupEnv(mizuTestEnvVar)
-	switch h.Type {
-	case TypeHttpRequest:
-		harRequest, err := har.NewRequest(h.Data.(*http.Request), true)
-		if err != nil {
-			return nil, errors.New("Failed converting request to HAR")
-		}
-		sort.Slice(harRequest.Headers, func(i, j int) bool {
-			if harRequest.Headers[i].Name < harRequest.Headers[j].Name {
-				return true
-			}
-			if harRequest.Headers[i].Name > harRequest.Headers[j].Name {
-				return false
-			}
-			return harRequest.Headers[i].Value < harRequest.Headers[j].Value
-		})
-		sort.Slice(harRequest.QueryString, func(i, j int) bool {
-			if harRequest.QueryString[i].Name < harRequest.QueryString[j].Name {
-				return true
-			}
-			if harRequest.QueryString[i].Name > harRequest.QueryString[j].Name {
-				return false
-			}
-			return harRequest.QueryString[i].Value < harRequest.QueryString[j].Value
-		})
-		if harRequest.PostData != nil {
-			sort.Slice(harRequest.PostData.Params, func(i, j int) bool {
-				if harRequest.PostData.Params[i].Name < harRequest.PostData.Params[j].Name {
-					return true
-				}
-				if harRequest.PostData.Params[i].Name > harRequest.PostData.Params[j].Name {
-					return false
-				}
-				return harRequest.PostData.Params[i].Value < harRequest.PostData.Params[j].Value
-			})
-		}
-		if testEnvEnabled {
-			harRequest.URL = ""
-		}
-		var reqWrapper *HTTPRequestWrapper
-		if !testEnvEnabled {
-			reqWrapper = &HTTPRequestWrapper{Request: h.Data.(*http.Request)}
-		}
-		return json.Marshal(&HTTPWrapper{
-			Method:     harRequest.Method,
-			Details:    harRequest,
-			RawRequest: reqWrapper,
-		})
-	case TypeHttpResponse:
-		harResponse, err := har.NewResponse(h.Data.(*http.Response), true)
-		if err != nil {
-			return nil, errors.New("Failed converting response to HAR")
-		}
-		sort.Slice(harResponse.Headers, func(i, j int) bool {
-			if harResponse.Headers[i].Name < harResponse.Headers[j].Name {
-				return true
-			}
-			if harResponse.Headers[i].Name > harResponse.Headers[j].Name {
-				return false
-			}
-			return harResponse.Headers[i].Value < harResponse.Headers[j].Value
-		})
-		sort.Slice(harResponse.Cookies, func(i, j int) bool {
-			if harResponse.Cookies[i].Name < harResponse.Cookies[j].Name {
-				return true
-			}
-			if harResponse.Cookies[i].Name > harResponse.Cookies[j].Name {
-				return false
-			}
-			return harResponse.Cookies[i].Value < harResponse.Cookies[j].Value
-		})
-		var resWrapper *HTTPResponseWrapper
-		if !testEnvEnabled {
-			resWrapper = &HTTPResponseWrapper{Response: h.Data.(*http.Response)}
-		}
-		return json.Marshal(&HTTPWrapper{
-			Method:      "",
-			Url:         "",
-			Details:     harResponse,
-			RawResponse: resWrapper,
-		})
-	default:
-		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %v", h.Type))
-	}
-}
-
-type HTTPWrapperTricky struct {
-	Method      string         `json:"method"`
-	Url         string         `json:"url"`
-	Details     interface{}    `json:"details"`
-	RawRequest  *http.Request  `json:"rawRequest"`
-	RawResponse *http.Response `json:"rawResponse"`
-}
-
-type HTTPMessage struct {
-	IsRequest   bool              `json:"isRequest"`
-	CaptureTime time.Time         `json:"captureTime"`
-	Payload     HTTPWrapperTricky `json:"payload"`
-}
-
-type HTTPRequestResponsePair struct {
-	Request  HTTPMessage `json:"request"`
-	Response HTTPMessage `json:"response"`
-}
-
-type HTTPRequestWrapper struct {
-	*http.Request
-}
-
-func (r *HTTPRequestWrapper) MarshalJSON() ([]byte, error) {
-	body, _ := ioutil.ReadAll(r.Request.Body)
-	r.Request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	return json.Marshal(&struct { //nolint
-		Body    string `json:"Body,omitempty"`
-		GetBody string `json:"GetBody,omitempty"`
-		Cancel  string `json:"Cancel,omitempty"`
-		*http.Request
-	}{
-		Body:    string(body),
-		Request: r.Request,
-	})
-}
-
-type HTTPResponseWrapper struct {
-	*http.Response
-}
-
-func (r *HTTPResponseWrapper) MarshalJSON() ([]byte, error) {
-	body, _ := ioutil.ReadAll(r.Response.Body)
-	r.Response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	return json.Marshal(&struct { //nolint
-		Body    string `json:"Body,omitempty"`
-		GetBody string `json:"GetBody,omitempty"`
-		Cancel  string `json:"Cancel,omitempty"`
-		*http.Response
-	}{
-		Body:     string(body),
-		Response: r.Response,
-	})
-}
-
 type TcpReaderDataMsg interface {
 	GetBytes() []byte
 	GetTimestamp() time.Time

tap/api/go.mod

@@ -2,9 +2,6 @@ module github.com/up9inc/mizu/tap/api
 
 go 1.17
 
-require (
-	github.com/google/martian v2.1.0+incompatible
-	github.com/up9inc/mizu/tap/dbgctl v0.0.0
-)
+require github.com/up9inc/mizu/tap/dbgctl v0.0.0
 
 replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../dbgctl

tap/api/go.sum

@@ -1,2 +0,0 @@
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=

tap/dbgctl/debug_control.go

@@ -5,11 +5,11 @@ import (
 )
 
 var (
-	MizuTapperDisablePcap              bool = os.Getenv("MIZU_DEBUG_DISABLE_PCAP") == "true"
-	MizuTapperDisableTcpReassembly     bool = os.Getenv("MIZU_DEBUG_DISABLE_TCP_REASSEMBLY") == "true"
-	MizuTapperDisableTcpStream         bool = os.Getenv("MIZU_DEBUG_DISABLE_TCP_STREAM") == "true"
-	MizuTapperDisableDissectors        bool = os.Getenv("MIZU_DEBUG_DISABLE_DISSECTORS") == "true"
-	MizuTapperDisableEmitting          bool = os.Getenv("MIZU_DEBUG_DISABLE_EMITTING") == "true"
-	MizuTapperDisableSending           bool = os.Getenv("MIZU_DEBUG_DISABLE_SENDING") == "true"
-	MizuTapperDisableNonHttpExtensions bool = os.Getenv("MIZU_DEBUG_DISABLE_NON_HTTP_EXTENSSION") == "true"
+	MizuTapperDisablePcap              = os.Getenv("MIZU_DEBUG_DISABLE_PCAP") == "true"
+	MizuTapperDisableTcpReassembly     = os.Getenv("MIZU_DEBUG_DISABLE_TCP_REASSEMBLY") == "true"
+	MizuTapperDisableTcpStream         = os.Getenv("MIZU_DEBUG_DISABLE_TCP_STREAM") == "true"
+	MizuTapperDisableDissectors        = os.Getenv("MIZU_DEBUG_DISABLE_DISSECTORS") == "true"
+	MizuTapperDisableEmitting          = os.Getenv("MIZU_DEBUG_DISABLE_EMITTING") == "true"
+	MizuTapperDisableSending           = os.Getenv("MIZU_DEBUG_DISABLE_SENDING") == "true"
+	MizuTapperDisableNonHttpExtensions = os.Getenv("MIZU_DEBUG_DISABLE_NON_HTTP_EXTENSSION") == "true"
 )

tap/extensions/amqp/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/amqp/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/amqp/\* expect

tap/extensions/amqp/go.mod

@@ -9,7 +9,6 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect

tap/extensions/amqp/go.sum

@@ -1,7 +1,5 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

tap/extensions/amqp/main.go

@@ -12,7 +12,7 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-var protocol api.Protocol = api.Protocol{
+var protocol = api.Protocol{
 	Name:            "amqp",
 	LongName:        "Advanced Message Queuing Protocol 0-9-1",
 	Abbreviation:    "AMQP",
@@ -26,12 +26,20 @@ var protocol api.Protocol = api.Protocol{
 	Priority:        1,
 }
 
+var protocolsMap = map[string]*api.Protocol{
+	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
+}
+
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
 	extension.Protocol = &protocol
 }
 
+func (d dissecting) GetProtocols() map[string]*api.Protocol {
+	return protocolsMap
+}
+
 func (d dissecting) Ping() {
 	log.Printf("pong %s", protocol.Name)
 }
@@ -74,13 +82,13 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 	var lastMethodFrameMessage Message
 
 	for {
-		frame, err := r.ReadFrame()
+		frameVal, err := r.readFrame()
 		if err == io.EOF {
 			// We must read until we see an EOF... very important!
 			return err
 		}
 
-		switch f := frame.(type) {
+		switch f := frameVal.(type) {
 		case *HeartbeatFrame:
 			// drop
 
@@ -203,7 +211,7 @@ func (d dissecting) Dissect(b *bufio.Reader, reader api.TcpReader, options *api.
 			}
 
 		default:
-			// log.Printf("unexpected frame: %+v", f)
+			// log.Printf("unexpected frameVal: %+v", f)
 		}
 	}
 }
@@ -214,8 +222,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 	reqDetails["method"] = request["method"]
 	return &api.Entry{
-		Protocol: protocol,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -276,22 +284,20 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	}
 
 	return &api.BaseEntry{
-		Id:             entry.Id,
-		Protocol:       entry.Protocol,
-		Capture:        entry.Capture,
-		Summary:        summary,
-		SummaryQuery:   summaryQuery,
-		Status:         0,
-		StatusQuery:    "",
-		Method:         method,
-		MethodQuery:    methodQuery,
-		Timestamp:      entry.Timestamp,
-		Source:         entry.Source,
-		Destination:    entry.Destination,
-		IsOutgoing:     entry.Outgoing,
-		Latency:        entry.ElapsedTime,
-		Rules:          entry.Rules,
-		ContractStatus: entry.ContractStatus,
+		Id:           entry.Id,
+		Protocol:     *protocolsMap[entry.ProtocolId],
+		Capture:      entry.Capture,
+		Summary:      summary,
+		SummaryQuery: summaryQuery,
+		Status:       0,
+		StatusQuery:  "",
+		Method:       method,
+		MethodQuery:  methodQuery,
+		Timestamp:    entry.Timestamp,
+		Source:       entry.Source,
+		Destination:  entry.Destination,
+		IsOutgoing:   entry.Outgoing,
+		Latency:      entry.ElapsedTime,
 	}
 }
 
@@ -323,7 +329,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`amqp`: fmt.Sprintf(`proto.name == "%s"`, protocol.Name),
+		`amqp`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
 	}
 }
 

tap/extensions/amqp/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"amqp": `proto.name == "amqp"`,
+		"amqp": `protocol == "amqp/0-9-1/AMQP"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/amqp/read.go

@@ -43,14 +43,14 @@ In realistic implementations where performance is a concern, we would use
 
 “gathering reads” to avoid doing three separate system calls to read a frame.
 */
-func (r *AmqpReader) ReadFrame() (frame frame, err error) {
+func (r *AmqpReader) readFrame() (frame frame, err error) {
 	var scratch [7]byte
 
 	if _, err = io.ReadFull(r.R, scratch[:7]); err != nil {
 		return
 	}
 
-	typ := uint8(scratch[0])
+	typ := scratch[0]
 	channel := binary.BigEndian.Uint16(scratch[1:3])
 	size := binary.BigEndian.Uint32(scratch[3:7])
 
@@ -94,20 +94,20 @@ func (r *AmqpReader) ReadFrame() (frame frame, err error) {
 	return
 }
 
-func readShortstr(r io.Reader) (v string, err error) {
+func readShortStr(r io.Reader) (v string, err error) {
 	var length uint8
 	if err = binary.Read(r, binary.BigEndian, &length); err != nil {
 		return
 	}
 
-	bytes := make([]byte, length)
-	if _, err = io.ReadFull(r, bytes); err != nil {
+	bytesValue := make([]byte, length)
+	if _, err = io.ReadFull(r, bytesValue); err != nil {
 		return
 	}
-	return string(bytes), nil
+	return string(bytesValue), nil
 }
 
-func readLongstr(r io.Reader) (v string, err error) {
+func readLongStr(r io.Reader) (v string, err error) {
 	var length uint32
 	if err = binary.Read(r, binary.BigEndian, &length); err != nil {
 		return
@@ -118,11 +118,11 @@ func readLongstr(r io.Reader) (v string, err error) {
 		return
 	}
 
-	bytes := make([]byte, length)
-	if _, err = io.ReadFull(r, bytes); err != nil {
+	bytesValue := make([]byte, length)
+	if _, err = io.ReadFull(r, bytesValue); err != nil {
 		return
 	}
-	return string(bytes), nil
+	return string(bytesValue), nil
 }
 
 func readDecimal(r io.Reader) (v Decimal, err error) {
@@ -171,7 +171,7 @@ func readField(r io.Reader) (v interface{}, err error) {
 		if err = binary.Read(r, binary.BigEndian, &value); err != nil {
 			return
 		}
-		return (value != 0), nil
+		return value != 0, nil
 
 	case 'b':
 		var value [1]byte
@@ -219,7 +219,7 @@ func readField(r io.Reader) (v interface{}, err error) {
 		return readDecimal(r)
 
 	case 'S':
-		return readLongstr(r)
+		return readLongStr(r)
 
 	case 'A':
 		return readArray(r)
@@ -231,12 +231,12 @@ func readField(r io.Reader) (v interface{}, err error) {
 		return readTable(r)
 
 	case 'x':
-		var len int32
-		if err = binary.Read(r, binary.BigEndian, &len); err != nil {
+		var lenVal int32
+		if err = binary.Read(r, binary.BigEndian, &lenVal); err != nil {
 			return nil, err
 		}
 
-		value := make([]byte, len)
+		value := make([]byte, lenVal)
 		if _, err = io.ReadFull(r, value); err != nil {
 			return nil, err
 		}
@@ -261,7 +261,7 @@ func readTable(r io.Reader) (table Table, err error) {
 	var nested bytes.Buffer
 	var str string
 
-	if str, err = readLongstr(r); err != nil {
+	if str, err = readLongStr(r); err != nil {
 		return
 	}
 
@@ -273,7 +273,7 @@ func readTable(r io.Reader) (table Table, err error) {
 		var key string
 		var value interface{}
 
-		if key, err = readShortstr(&nested); err != nil {
+		if key, err = readShortStr(&nested); err != nil {
 			return
 		}
 
@@ -349,12 +349,12 @@ func (r *AmqpReader) parseHeaderFrame(channel uint16, size uint32) (frame frame,
 	}
 
 	if hasProperty(flags, flagContentType) {
-		if hf.Properties.ContentType, err = readShortstr(r.R); err != nil {
+		if hf.Properties.ContentType, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
 	if hasProperty(flags, flagContentEncoding) {
-		if hf.Properties.ContentEncoding, err = readShortstr(r.R); err != nil {
+		if hf.Properties.ContentEncoding, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
@@ -374,22 +374,22 @@ func (r *AmqpReader) parseHeaderFrame(channel uint16, size uint32) (frame frame,
 		}
 	}
 	if hasProperty(flags, flagCorrelationId) {
-		if hf.Properties.CorrelationId, err = readShortstr(r.R); err != nil {
+		if hf.Properties.CorrelationId, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
 	if hasProperty(flags, flagReplyTo) {
-		if hf.Properties.ReplyTo, err = readShortstr(r.R); err != nil {
+		if hf.Properties.ReplyTo, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
 	if hasProperty(flags, flagExpiration) {
-		if hf.Properties.Expiration, err = readShortstr(r.R); err != nil {
+		if hf.Properties.Expiration, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
 	if hasProperty(flags, flagMessageId) {
-		if hf.Properties.MessageId, err = readShortstr(r.R); err != nil {
+		if hf.Properties.MessageId, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
@@ -399,22 +399,22 @@ func (r *AmqpReader) parseHeaderFrame(channel uint16, size uint32) (frame frame,
 		}
 	}
 	if hasProperty(flags, flagType) {
-		if hf.Properties.Type, err = readShortstr(r.R); err != nil {
+		if hf.Properties.Type, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
 	if hasProperty(flags, flagUserId) {
-		if hf.Properties.UserId, err = readShortstr(r.R); err != nil {
+		if hf.Properties.UserId, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
 	if hasProperty(flags, flagAppId) {
-		if hf.Properties.AppId, err = readShortstr(r.R); err != nil {
+		if hf.Properties.AppId, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
 	if hasProperty(flags, flagReserved1) {
-		if hf.Properties.reserved1, err = readShortstr(r.R); err != nil {
+		if hf.Properties.reserved1, err = readShortStr(r.R); err != nil {
 			return
 		}
 	}
@@ -436,7 +436,7 @@ func (r *AmqpReader) parseBodyFrame(channel uint16, size uint32) (frame frame, e
 	return bf, nil
 }
 
-var errHeartbeatPayload = errors.New("Heartbeats should not have a payload")
+var errHeartbeatPayload = errors.New("heartbeats should not have a payload")
 
 func (r *AmqpReader) parseHeartbeatFrame(channel uint16, size uint32) (frame frame, err error) {
 	hf := &HeartbeatFrame{

tap/extensions/amqp/spec091.go

@@ -70,10 +70,10 @@ func (msg *ConnectionStart) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Mechanisms, err = readLongstr(r); err != nil {
+	if msg.Mechanisms, err = readLongStr(r); err != nil {
 		return
 	}
-	if msg.Locales, err = readLongstr(r); err != nil {
+	if msg.Locales, err = readLongStr(r); err != nil {
 		return
 	}
 
@@ -93,15 +93,15 @@ func (msg *ConnectionStartOk) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Mechanism, err = readShortstr(r); err != nil {
+	if msg.Mechanism, err = readShortStr(r); err != nil {
 		return
 	}
 
-	if msg.Response, err = readLongstr(r); err != nil {
+	if msg.Response, err = readLongStr(r); err != nil {
 		return
 	}
 
-	if msg.Locale, err = readShortstr(r); err != nil {
+	if msg.Locale, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -114,7 +114,7 @@ type connectionSecure struct {
 
 func (msg *connectionSecure) read(r io.Reader) (err error) {
 
-	if msg.Challenge, err = readLongstr(r); err != nil {
+	if msg.Challenge, err = readLongStr(r); err != nil {
 		return
 	}
 
@@ -127,7 +127,7 @@ type connectionSecureOk struct {
 
 func (msg *connectionSecureOk) read(r io.Reader) (err error) {
 
-	if msg.Response, err = readLongstr(r); err != nil {
+	if msg.Response, err = readLongStr(r); err != nil {
 		return
 	}
 
@@ -189,17 +189,17 @@ type connectionOpen struct {
 func (msg *connectionOpen) read(r io.Reader) (err error) {
 	var bits byte
 
-	if msg.VirtualHost, err = readShortstr(r); err != nil {
+	if msg.VirtualHost, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.reserved1, err = readShortstr(r); err != nil {
+	if msg.reserved1, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.reserved2 = (bits&(1<<0) > 0)
+	msg.reserved2 = bits&(1<<0) > 0
 
 	return
 }
@@ -210,7 +210,7 @@ type connectionOpenOk struct {
 
 func (msg *connectionOpenOk) read(r io.Reader) (err error) {
 
-	if msg.reserved1, err = readShortstr(r); err != nil {
+	if msg.reserved1, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -230,7 +230,7 @@ func (msg *ConnectionClose) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.ReplyText, err = readShortstr(r); err != nil {
+	if msg.ReplyText, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -258,7 +258,7 @@ type connectionBlocked struct {
 
 func (msg *connectionBlocked) read(r io.Reader) (err error) {
 
-	if msg.Reason, err = readShortstr(r); err != nil {
+	if msg.Reason, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -279,7 +279,7 @@ type channelOpen struct {
 
 func (msg *channelOpen) read(r io.Reader) (err error) {
 
-	if msg.reserved1, err = readShortstr(r); err != nil {
+	if msg.reserved1, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -292,7 +292,7 @@ type channelOpenOk struct {
 
 func (msg *channelOpenOk) read(r io.Reader) (err error) {
 
-	if msg.reserved1, err = readLongstr(r); err != nil {
+	if msg.reserved1, err = readLongStr(r); err != nil {
 		return
 	}
 
@@ -309,7 +309,7 @@ func (msg *channelFlow) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Active = (bits&(1<<0) > 0)
+	msg.Active = bits&(1<<0) > 0
 
 	return
 }
@@ -324,7 +324,7 @@ func (msg *channelFlowOk) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Active = (bits&(1<<0) > 0)
+	msg.Active = bits&(1<<0) > 0
 
 	return
 }
@@ -342,7 +342,7 @@ func (msg *channelClose) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.ReplyText, err = readShortstr(r); err != nil {
+	if msg.ReplyText, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -383,21 +383,21 @@ func (msg *ExchangeDeclare) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.Type, err = readShortstr(r); err != nil {
+	if msg.Type, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Passive = (bits&(1<<0) > 0)
-	msg.Durable = (bits&(1<<1) > 0)
-	msg.AutoDelete = (bits&(1<<2) > 0)
-	msg.Internal = (bits&(1<<3) > 0)
-	msg.NoWait = (bits&(1<<4) > 0)
+	msg.Passive = bits&(1<<0) > 0
+	msg.Durable = bits&(1<<1) > 0
+	msg.AutoDelete = bits&(1<<2) > 0
+	msg.Internal = bits&(1<<3) > 0
+	msg.NoWait = bits&(1<<4) > 0
 
 	if msg.Arguments, err = readTable(r); err != nil {
 		return
@@ -428,15 +428,15 @@ func (msg *exchangeDelete) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.IfUnused = (bits&(1<<0) > 0)
-	msg.NoWait = (bits&(1<<1) > 0)
+	msg.IfUnused = bits&(1<<0) > 0
+	msg.NoWait = bits&(1<<1) > 0
 
 	return
 }
@@ -465,20 +465,20 @@ func (msg *exchangeBind) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Destination, err = readShortstr(r); err != nil {
+	if msg.Destination, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.Source, err = readShortstr(r); err != nil {
+	if msg.Source, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.NoWait = (bits&(1<<0) > 0)
+	msg.NoWait = bits&(1<<0) > 0
 
 	if msg.Arguments, err = readTable(r); err != nil {
 		return
@@ -511,20 +511,20 @@ func (msg *exchangeUnbind) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Destination, err = readShortstr(r); err != nil {
+	if msg.Destination, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.Source, err = readShortstr(r); err != nil {
+	if msg.Source, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.NoWait = (bits&(1<<0) > 0)
+	msg.NoWait = bits&(1<<0) > 0
 
 	if msg.Arguments, err = readTable(r); err != nil {
 		return
@@ -559,18 +559,18 @@ func (msg *QueueDeclare) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Passive = (bits&(1<<0) > 0)
-	msg.Durable = (bits&(1<<1) > 0)
-	msg.Exclusive = (bits&(1<<2) > 0)
-	msg.AutoDelete = (bits&(1<<3) > 0)
-	msg.NoWait = (bits&(1<<4) > 0)
+	msg.Passive = bits&(1<<0) > 0
+	msg.Durable = bits&(1<<1) > 0
+	msg.Exclusive = bits&(1<<2) > 0
+	msg.AutoDelete = bits&(1<<3) > 0
+	msg.NoWait = bits&(1<<4) > 0
 
 	if msg.Arguments, err = readTable(r); err != nil {
 		return
@@ -587,7 +587,7 @@ type QueueDeclareOk struct {
 
 func (msg *QueueDeclareOk) read(r io.Reader) (err error) {
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -617,20 +617,20 @@ func (msg *QueueBind) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.NoWait = (bits&(1<<0) > 0)
+	msg.NoWait = bits&(1<<0) > 0
 
 	if msg.Arguments, err = readTable(r); err != nil {
 		return
@@ -661,13 +661,13 @@ func (msg *queueUnbind) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -699,14 +699,14 @@ func (msg *queuePurge) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.NoWait = (bits&(1<<0) > 0)
+	msg.NoWait = bits&(1<<0) > 0
 
 	return
 }
@@ -739,16 +739,16 @@ func (msg *queueDelete) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.IfUnused = (bits&(1<<0) > 0)
-	msg.IfEmpty = (bits&(1<<1) > 0)
-	msg.NoWait = (bits&(1<<2) > 0)
+	msg.IfUnused = bits&(1<<0) > 0
+	msg.IfEmpty = bits&(1<<1) > 0
+	msg.NoWait = bits&(1<<2) > 0
 
 	return
 }
@@ -786,7 +786,7 @@ func (msg *basicQos) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Global = (bits&(1<<0) > 0)
+	msg.Global = bits&(1<<0) > 0
 
 	return
 }
@@ -817,20 +817,20 @@ func (msg *BasicConsume) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.ConsumerTag, err = readShortstr(r); err != nil {
+	if msg.ConsumerTag, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.NoLocal = (bits&(1<<0) > 0)
-	msg.NoAck = (bits&(1<<1) > 0)
-	msg.Exclusive = (bits&(1<<2) > 0)
-	msg.NoWait = (bits&(1<<3) > 0)
+	msg.NoLocal = bits&(1<<0) > 0
+	msg.NoAck = bits&(1<<1) > 0
+	msg.Exclusive = bits&(1<<2) > 0
+	msg.NoWait = bits&(1<<3) > 0
 
 	if msg.Arguments, err = readTable(r); err != nil {
 		return
@@ -845,7 +845,7 @@ type BasicConsumeOk struct {
 
 func (msg *BasicConsumeOk) read(r io.Reader) (err error) {
 
-	if msg.ConsumerTag, err = readShortstr(r); err != nil {
+	if msg.ConsumerTag, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -860,14 +860,14 @@ type basicCancel struct {
 func (msg *basicCancel) read(r io.Reader) (err error) {
 	var bits byte
 
-	if msg.ConsumerTag, err = readShortstr(r); err != nil {
+	if msg.ConsumerTag, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.NoWait = (bits&(1<<0) > 0)
+	msg.NoWait = bits&(1<<0) > 0
 
 	return
 }
@@ -878,7 +878,7 @@ type basicCancelOk struct {
 
 func (msg *basicCancelOk) read(r io.Reader) (err error) {
 
-	if msg.ConsumerTag, err = readShortstr(r); err != nil {
+	if msg.ConsumerTag, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -910,18 +910,18 @@ func (msg *BasicPublish) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Mandatory = (bits&(1<<0) > 0)
-	msg.Immediate = (bits&(1<<1) > 0)
+	msg.Mandatory = bits&(1<<0) > 0
+	msg.Immediate = bits&(1<<1) > 0
 
 	return
 }
@@ -949,13 +949,13 @@ func (msg *basicReturn) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.ReplyText, err = readShortstr(r); err != nil {
+	if msg.ReplyText, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -983,7 +983,7 @@ func (msg *BasicDeliver) setContent(props Properties, body []byte) {
 func (msg *BasicDeliver) read(r io.Reader) (err error) {
 	var bits byte
 
-	if msg.ConsumerTag, err = readShortstr(r); err != nil {
+	if msg.ConsumerTag, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -994,12 +994,12 @@ func (msg *BasicDeliver) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Redelivered = (bits&(1<<0) > 0)
+	msg.Redelivered = bits&(1<<0) > 0
 
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -1019,14 +1019,14 @@ func (msg *basicGet) read(r io.Reader) (err error) {
 		return
 	}
 
-	if msg.Queue, err = readShortstr(r); err != nil {
+	if msg.Queue, err = readShortStr(r); err != nil {
 		return
 	}
 
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.NoAck = (bits&(1<<0) > 0)
+	msg.NoAck = bits&(1<<0) > 0
 
 	return
 }
@@ -1059,12 +1059,12 @@ func (msg *basicGetOk) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Redelivered = (bits&(1<<0) > 0)
+	msg.Redelivered = bits&(1<<0) > 0
 
-	if msg.Exchange, err = readShortstr(r); err != nil {
+	if msg.Exchange, err = readShortStr(r); err != nil {
 		return
 	}
-	if msg.RoutingKey, err = readShortstr(r); err != nil {
+	if msg.RoutingKey, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -1081,7 +1081,7 @@ type basicGetEmpty struct {
 
 func (msg *basicGetEmpty) read(r io.Reader) (err error) {
 
-	if msg.reserved1, err = readShortstr(r); err != nil {
+	if msg.reserved1, err = readShortStr(r); err != nil {
 		return
 	}
 
@@ -1103,7 +1103,7 @@ func (msg *basicAck) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Multiple = (bits&(1<<0) > 0)
+	msg.Multiple = bits&(1<<0) > 0
 
 	return
 }
@@ -1123,7 +1123,7 @@ func (msg *basicReject) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Requeue = (bits&(1<<0) > 0)
+	msg.Requeue = bits&(1<<0) > 0
 
 	return
 }
@@ -1138,7 +1138,7 @@ func (msg *basicRecoverAsync) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Requeue = (bits&(1<<0) > 0)
+	msg.Requeue = bits&(1<<0) > 0
 
 	return
 }
@@ -1153,7 +1153,7 @@ func (msg *basicRecover) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Requeue = (bits&(1<<0) > 0)
+	msg.Requeue = bits&(1<<0) > 0
 
 	return
 }
@@ -1182,8 +1182,8 @@ func (msg *basicNack) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Multiple = (bits&(1<<0) > 0)
-	msg.Requeue = (bits&(1<<1) > 0)
+	msg.Multiple = bits&(1<<0) > 0
+	msg.Requeue = bits&(1<<1) > 0
 
 	return
 }
@@ -1246,7 +1246,7 @@ func (msg *confirmSelect) read(r io.Reader) (err error) {
 	if err = binary.Read(r, binary.BigEndian, &bits); err != nil {
 		return
 	}
-	msg.Nowait = (bits&(1<<0) > 0)
+	msg.Nowait = bits&(1<<0) > 0
 
 	return
 }

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect10/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/http/\* expect

tap/extensions/http/go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/beevik/etree v1.1.0
+	github.com/google/martian v2.1.0+incompatible
 	github.com/mertyildiran/gqlparser/v2 v2.4.6
 	github.com/stretchr/testify v1.7.0
 	github.com/up9inc/mizu/tap/api v0.0.0
@@ -12,7 +13,6 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	golang.org/x/text v0.3.7 // indirect

tap/extensions/http/handlers.go

@@ -32,7 +32,7 @@ func replaceForwardedFor(item *api.OutputChannelItem) {
 		return
 	}
 
-	request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
+	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
 
 	forwardedFor := request.Header.Get("X-Forwarded-For")
 	if forwardedFor == "" {

tap/extensions/http/http2_assembler.go

@@ -91,7 +91,7 @@ func (ga *Http2Assembler) readMessage() (streamID uint32, messageHTTP1 interface
 	// Exactly one Framer is used for each half connection.
 	// (Instead of creating a new Framer for each ReadFrame operation)
 	// This is needed in order to decompress the headers,
-	// because the compression context is updated with each requests/response.
+	// because the compression context is updated with each request/response.
 	frame, err := ga.framer.ReadFrame()
 	if err != nil {
 		return

tap/extensions/http/main.go

@@ -14,7 +14,7 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-var http10protocol api.Protocol = api.Protocol{
+var http10protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.0",
 	Abbreviation:    "HTTP",
@@ -28,7 +28,7 @@ var http10protocol api.Protocol = api.Protocol{
 	Priority:        0,
 }
 
-var http11protocol api.Protocol = api.Protocol{
+var http11protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
 	Abbreviation:    "HTTP",
@@ -42,7 +42,7 @@ var http11protocol api.Protocol = api.Protocol{
 	Priority:        0,
 }
 
-var http2Protocol api.Protocol = api.Protocol{
+var http2Protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2)",
 	Abbreviation:    "HTTP/2",
@@ -56,7 +56,7 @@ var http2Protocol api.Protocol = api.Protocol{
 	Priority:        0,
 }
 
-var grpcProtocol api.Protocol = api.Protocol{
+var grpcProtocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ gRPC over HTTP/2 ]",
 	Abbreviation:    "gRPC",
@@ -70,7 +70,7 @@ var grpcProtocol api.Protocol = api.Protocol{
 	Priority:        0,
 }
 
-var graphQL1Protocol api.Protocol = api.Protocol{
+var graphQL1Protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1 [ GraphQL over HTTP/1.1 ]",
 	Abbreviation:    "GQL",
@@ -84,7 +84,7 @@ var graphQL1Protocol api.Protocol = api.Protocol{
 	Priority:        0,
 }
 
-var graphQL2Protocol api.Protocol = api.Protocol{
+var graphQL2Protocol = api.Protocol{
 	Name:            "http",
 	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) [ GraphQL over HTTP/2 ]",
 	Abbreviation:    "GQL",
@@ -98,6 +98,15 @@ var graphQL2Protocol api.Protocol = api.Protocol{
 	Priority:        0,
 }
 
+var protocolsMap = map[string]*api.Protocol{
+	fmt.Sprintf("%s/%s/%s", http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation):       &http10protocol,
+	fmt.Sprintf("%s/%s/%s", http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation):       &http11protocol,
+	fmt.Sprintf("%s/%s/%s", http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation):          &http2Protocol,
+	fmt.Sprintf("%s/%s/%s", grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation):             &grpcProtocol,
+	fmt.Sprintf("%s/%s/%s", graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation): &graphQL1Protocol,
+	fmt.Sprintf("%s/%s/%s", graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation): &graphQL2Protocol,
+}
+
 const (
 	TypeHttpRequest = iota
 	TypeHttpResponse
@@ -109,6 +118,10 @@ func (d dissecting) Register(extension *api.Extension) {
 	extension.Protocol = &http11protocol
 }
 
+func (d dissecting) GetProtocols() map[string]*api.Protocol {
+	return protocolsMap
+}
+
 func (d dissecting) Ping() {
 	log.Printf("pong %s", http11protocol.Name)
 }
@@ -279,10 +292,10 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	if elapsedTime < 0 {
 		elapsedTime = 0
 	}
-	httpPair, _ := json.Marshal(item.Pair)
+
 	return &api.Entry{
-		Protocol: item.Protocol,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", item.Protocol.Name, item.Protocol.Version, item.Protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -302,7 +315,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		Timestamp:    item.Timestamp,
 		StartTime:    item.Pair.Request.CaptureTime,
 		ElapsedTime:  elapsedTime,
-		HTTPPair:     string(httpPair),
 	}
 }
 
@@ -315,22 +327,20 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	statusQuery := fmt.Sprintf(`response.status == %d`, status)
 
 	return &api.BaseEntry{
-		Id:             entry.Id,
-		Protocol:       entry.Protocol,
-		Capture:        entry.Capture,
-		Summary:        summary,
-		SummaryQuery:   summaryQuery,
-		Status:         status,
-		StatusQuery:    statusQuery,
-		Method:         method,
-		MethodQuery:    methodQuery,
-		Timestamp:      entry.Timestamp,
-		Source:         entry.Source,
-		Destination:    entry.Destination,
-		IsOutgoing:     entry.Outgoing,
-		Latency:        entry.ElapsedTime,
-		Rules:          entry.Rules,
-		ContractStatus: entry.ContractStatus,
+		Id:           entry.Id,
+		Protocol:     *protocolsMap[entry.ProtocolId],
+		Capture:      entry.Capture,
+		Summary:      summary,
+		SummaryQuery: summaryQuery,
+		Status:       status,
+		StatusQuery:  statusQuery,
+		Method:       method,
+		MethodQuery:  methodQuery,
+		Timestamp:    entry.Timestamp,
+		Source:       entry.Source,
+		Destination:  entry.Destination,
+		IsOutgoing:   entry.Outgoing,
+		Latency:      entry.ElapsedTime,
 	}
 }
 
@@ -505,10 +515,10 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`http`:  fmt.Sprintf(`proto.name == "%s" and proto.version.startsWith("%c")`, http11protocol.Name, http11protocol.Version[0]),
-		`http2`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s"`, http11protocol.Name, http2Protocol.Version),
-		`grpc`:  fmt.Sprintf(`proto.name == "%s" and proto.version == "%s" and proto.macro == "%s"`, http11protocol.Name, grpcProtocol.Version, grpcProtocol.Macro),
-		`gql`:   fmt.Sprintf(`proto.name == "%s" and proto.macro == "%s"`, graphQL1Protocol.Name, graphQL1Protocol.Macro),
+		`http`:  fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, http10protocol.Name, http10protocol.Version, http10protocol.Abbreviation, http11protocol.Name, http11protocol.Version, http11protocol.Abbreviation),
+		`http2`: fmt.Sprintf(`protocol == "%s/%s/%s"`, http2Protocol.Name, http2Protocol.Version, http2Protocol.Abbreviation),
+		`grpc`:  fmt.Sprintf(`protocol == "%s/%s/%s"`, grpcProtocol.Name, grpcProtocol.Version, grpcProtocol.Abbreviation),
+		`gql`:   fmt.Sprintf(`protocol == "%s/%s/%s" or protocol == "%s/%s/%s"`, graphQL1Protocol.Name, graphQL1Protocol.Version, graphQL1Protocol.Abbreviation, graphQL2Protocol.Name, graphQL2Protocol.Version, graphQL2Protocol.Abbreviation),
 	}
 }
 

tap/extensions/http/main_test.go

@@ -44,10 +44,10 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"http":  `proto.name == "http" and proto.version.startsWith("1")`,
-		"http2": `proto.name == "http" and proto.version == "2.0"`,
-		"grpc":  `proto.name == "http" and proto.version == "2.0" and proto.macro == "grpc"`,
-		"gql":   `proto.name == "http" and proto.macro == "gql"`,
+		"http":  `protocol == "http/1.0/HTTP" or protocol == "http/1.1/HTTP"`,
+		"http2": `protocol == "http/2.0/HTTP/2"`,
+		"grpc":  `protocol == "http/2.0/gRPC"`,
+		"gql":   `protocol == "http/1.1/GQL" or protocol == "http/2.0/GQL"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/http/matcher.go

@@ -29,7 +29,7 @@ func (matcher *requestResponseMatcher) registerRequest(ident string, request *ht
 		IsRequest:   true,
 		CaptureTime: captureTime,
 		CaptureSize: captureSize,
-		Payload: api.HTTPPayload{
+		Payload: HTTPPayload{
 			Type: TypeHttpRequest,
 			Data: request,
 		},
@@ -53,7 +53,7 @@ func (matcher *requestResponseMatcher) registerResponse(ident string, response *
 		IsRequest:   false,
 		CaptureTime: captureTime,
 		CaptureSize: captureSize,
-		Payload: api.HTTPPayload{
+		Payload: HTTPPayload{
 			Type: TypeHttpResponse,
 			Data: response,
 		},

tap/extensions/http/sensitive_data_cleaner.go

@@ -30,7 +30,7 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 		return false
 	}
 
-	request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
+	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
 
 	for headerKey, headerValues := range request.Header {
 		if strings.ToLower(headerKey) == userAgent {
@@ -50,8 +50,8 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 }
 
 func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
-	request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
-	response := item.Pair.Response.Payload.(api.HTTPPayload).Data.(*http.Response)
+	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
+	response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)
 
 	filterHeaders(&request.Header)
 	filterHeaders(&response.Header)
@@ -66,7 +66,7 @@ func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptio
 	if err != nil {
 		return
 	}
-	filteredBody, err := filterHttpBody([]byte(body), contenType, options)
+	filteredBody, err := filterHttpBody(body, contenType, options)
 	if err == nil {
 		request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
 	} else {
@@ -80,7 +80,7 @@ func filterResponseBody(response *http.Response, options *api.TrafficFilteringOp
 	if err != nil {
 		return
 	}
-	filteredBody, err := filterHttpBody([]byte(body), contentType, options)
+	filteredBody, err := filterHttpBody(body, contentType, options)
 	if err == nil {
 		response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
 	} else {

tap/extensions/http/structs.go

@@ -0,0 +1,100 @@
+package http
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+
+	"github.com/google/martian/har"
+)
+
+type HTTPPayload struct {
+	Type uint8
+	Data interface{}
+}
+
+type HTTPPayloader interface {
+	MarshalJSON() ([]byte, error)
+}
+
+type HTTPWrapper struct {
+	Method  string      `json:"method"`
+	Url     string      `json:"url"`
+	Details interface{} `json:"details"`
+}
+
+func (h HTTPPayload) MarshalJSON() ([]byte, error) {
+	switch h.Type {
+	case TypeHttpRequest:
+		harRequest, err := har.NewRequest(h.Data.(*http.Request), true)
+		if err != nil {
+			return nil, errors.New("Failed converting request to HAR")
+		}
+		sort.Slice(harRequest.Headers, func(i, j int) bool {
+			if harRequest.Headers[i].Name < harRequest.Headers[j].Name {
+				return true
+			}
+			if harRequest.Headers[i].Name > harRequest.Headers[j].Name {
+				return false
+			}
+			return harRequest.Headers[i].Value < harRequest.Headers[j].Value
+		})
+		sort.Slice(harRequest.QueryString, func(i, j int) bool {
+			if harRequest.QueryString[i].Name < harRequest.QueryString[j].Name {
+				return true
+			}
+			if harRequest.QueryString[i].Name > harRequest.QueryString[j].Name {
+				return false
+			}
+			return harRequest.QueryString[i].Value < harRequest.QueryString[j].Value
+		})
+		if harRequest.PostData != nil {
+			sort.Slice(harRequest.PostData.Params, func(i, j int) bool {
+				if harRequest.PostData.Params[i].Name < harRequest.PostData.Params[j].Name {
+					return true
+				}
+				if harRequest.PostData.Params[i].Name > harRequest.PostData.Params[j].Name {
+					return false
+				}
+				return harRequest.PostData.Params[i].Value < harRequest.PostData.Params[j].Value
+			})
+		}
+		return json.Marshal(&HTTPWrapper{
+			Method:  harRequest.Method,
+			Url:     "",
+			Details: harRequest,
+		})
+	case TypeHttpResponse:
+		harResponse, err := har.NewResponse(h.Data.(*http.Response), true)
+		if err != nil {
+			return nil, errors.New("Failed converting response to HAR")
+		}
+		sort.Slice(harResponse.Headers, func(i, j int) bool {
+			if harResponse.Headers[i].Name < harResponse.Headers[j].Name {
+				return true
+			}
+			if harResponse.Headers[i].Name > harResponse.Headers[j].Name {
+				return false
+			}
+			return harResponse.Headers[i].Value < harResponse.Headers[j].Value
+		})
+		sort.Slice(harResponse.Cookies, func(i, j int) bool {
+			if harResponse.Cookies[i].Name < harResponse.Cookies[j].Name {
+				return true
+			}
+			if harResponse.Cookies[i].Name > harResponse.Cookies[j].Name {
+				return false
+			}
+			return harResponse.Cookies[i].Value < harResponse.Cookies[j].Value
+		})
+		return json.Marshal(&HTTPWrapper{
+			Method:  "",
+			Url:     "",
+			Details: harResponse,
+		})
+	default:
+		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %v", h.Type))
+	}
+}

tap/extensions/kafka/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect9/kafka/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/kafka/\* expect

tap/extensions/kafka/go.mod

@@ -14,7 +14,6 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/klauspost/compress v1.14.2 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

tap/extensions/kafka/go.sum

@@ -11,8 +11,6 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
 github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=

tap/extensions/kafka/helpers.go

@@ -817,12 +817,12 @@ func representDeleteTopicsRequest(data map[string]interface{}) []interface{} {
 	repPayload, _ := json.Marshal([]api.TableData{
 		{
 			Name:     "TopicNames",
-			Value:    string(topicNames),
+			Value:    topicNames,
 			Selector: `request.payload.topicNames`,
 		},
 		{
 			Name:     "Topics",
-			Value:    string(topics),
+			Value:    topics,
 			Selector: `request.payload.topics`,
 		},
 		{

tap/extensions/kafka/main.go

@@ -10,7 +10,7 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-var _protocol api.Protocol = api.Protocol{
+var _protocol = api.Protocol{
 	Name:            "kafka",
 	LongName:        "Apache Kafka Protocol",
 	Abbreviation:    "KAFKA",
@@ -24,12 +24,20 @@ var _protocol api.Protocol = api.Protocol{
 	Priority:        2,
 }
 
+var protocolsMap = map[string]*api.Protocol{
+	fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation): &_protocol,
+}
+
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
 	extension.Protocol = &_protocol
 }
 
+func (d dissecting) GetProtocols() map[string]*api.Protocol {
+	return protocolsMap
+}
+
 func (d dissecting) Ping() {
 	log.Printf("pong %s", _protocol.Name)
 }
@@ -62,8 +70,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		Protocol: _protocol,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", _protocol.Name, _protocol.Version, _protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -186,22 +194,20 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	}
 
 	return &api.BaseEntry{
-		Id:             entry.Id,
-		Protocol:       entry.Protocol,
-		Capture:        entry.Capture,
-		Summary:        summary,
-		SummaryQuery:   summaryQuery,
-		Status:         status,
-		StatusQuery:    statusQuery,
-		Method:         method,
-		MethodQuery:    methodQuery,
-		Timestamp:      entry.Timestamp,
-		Source:         entry.Source,
-		Destination:    entry.Destination,
-		IsOutgoing:     entry.Outgoing,
-		Latency:        entry.ElapsedTime,
-		Rules:          entry.Rules,
-		ContractStatus: entry.ContractStatus,
+		Id:           entry.Id,
+		Protocol:     *protocolsMap[entry.ProtocolId],
+		Capture:      entry.Capture,
+		Summary:      summary,
+		SummaryQuery: summaryQuery,
+		Status:       status,
+		StatusQuery:  statusQuery,
+		Method:       method,
+		MethodQuery:  methodQuery,
+		Timestamp:    entry.Timestamp,
+		Source:       entry.Source,
+		Destination:  entry.Destination,
+		IsOutgoing:   entry.Outgoing,
+		Latency:      entry.ElapsedTime,
 	}
 }
 
@@ -244,7 +250,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`kafka`: fmt.Sprintf(`proto.name == "%s"`, _protocol.Name),
+		`kafka`: fmt.Sprintf(`protocol == "%s/%s/%s"`, _protocol.Name, _protocol.Version, _protocol.Abbreviation),
 	}
 }
 

tap/extensions/kafka/main_test.go

@@ -44,7 +44,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"kafka": `proto.name == "kafka"`,
+		"kafka": `protocol == "kafka/12/KAFKA"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/kafka/reflect.go

@@ -56,6 +56,6 @@ func makeArray(t reflect.Type, n int) array {
 
 func (a array) index(i int) value { return value{val: a.val.Index(i)} }
 
-func indexOf(s reflect.StructField) index { return index(s.Index) }
+func indexOf(s reflect.StructField) index { return s.Index }
 
 func bytesToString(b []byte) string { return string(b) }

tap/extensions/redis/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/redis/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect14/redis/\* expect

tap/extensions/redis/go.mod

@@ -9,7 +9,6 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect

tap/extensions/redis/go.sum

@@ -1,7 +1,5 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

tap/extensions/redis/main.go

@@ -10,7 +10,7 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-var protocol api.Protocol = api.Protocol{
+var protocol = api.Protocol{
 	Name:            "redis",
 	LongName:        "Redis Serialization Protocol",
 	Abbreviation:    "REDIS",
@@ -24,12 +24,20 @@ var protocol api.Protocol = api.Protocol{
 	Priority:        3,
 }
 
+var protocolsMap = map[string]*api.Protocol{
+	fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation): &protocol,
+}
+
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
 	extension.Protocol = &protocol
 }
 
+func (d dissecting) GetProtocols() map[string]*api.Protocol {
+	return protocolsMap
+}
+
 func (d dissecting) Ping() {
 	log.Printf("pong %s", protocol.Name)
 }
@@ -70,8 +78,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	return &api.Entry{
-		Protocol: protocol,
-		Capture:  item.Capture,
+		ProtocolId: fmt.Sprintf("%s/%s/%s", protocol.Name, protocol.Version, protocol.Abbreviation),
+		Capture:    item.Capture,
 		Source: &api.TCP{
 			Name: resolvedSource,
 			IP:   item.ConnectionInfo.ClientIP,
@@ -114,22 +122,20 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	}
 
 	return &api.BaseEntry{
-		Id:             entry.Id,
-		Protocol:       entry.Protocol,
-		Capture:        entry.Capture,
-		Summary:        summary,
-		SummaryQuery:   summaryQuery,
-		Status:         status,
-		StatusQuery:    statusQuery,
-		Method:         method,
-		MethodQuery:    methodQuery,
-		Timestamp:      entry.Timestamp,
-		Source:         entry.Source,
-		Destination:    entry.Destination,
-		IsOutgoing:     entry.Outgoing,
-		Latency:        entry.ElapsedTime,
-		Rules:          entry.Rules,
-		ContractStatus: entry.ContractStatus,
+		Id:           entry.Id,
+		Protocol:     *protocolsMap[entry.ProtocolId],
+		Capture:      entry.Capture,
+		Summary:      summary,
+		SummaryQuery: summaryQuery,
+		Status:       status,
+		StatusQuery:  statusQuery,
+		Method:       method,
+		MethodQuery:  methodQuery,
+		Timestamp:    entry.Timestamp,
+		Source:       entry.Source,
+		Destination:  entry.Destination,
+		IsOutgoing:   entry.Outgoing,
+		Latency:      entry.ElapsedTime,
 	}
 }
 
@@ -145,7 +151,7 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`redis`: fmt.Sprintf(`proto.name == "%s"`, protocol.Name),
+		`redis`: fmt.Sprintf(`protocol == "%s/%s/%s"`, protocol.Name, protocol.Version, protocol.Abbreviation),
 	}
 }
 

tap/extensions/redis/main_test.go

@@ -45,7 +45,7 @@ func TestRegister(t *testing.T) {
 
 func TestMacros(t *testing.T) {
 	expectedMacros := map[string]string{
-		"redis": `proto.name == "redis"`,
+		"redis": `protocol == "redis/3.x/REDIS"`,
 	}
 	dissector := NewDissector()
 	macros := dissector.Macros()

tap/extensions/redis/structs.go

@@ -4,7 +4,7 @@ type RedisType string
 type RedisCommand string
 type RedisKeyword string
 
-var types map[rune]RedisType = map[rune]RedisType{
+var types = map[rune]RedisType{
 	plusByte:          "Simple String",
 	dollarByte:        "Bulk String",
 	asteriskByte:      "Array",
@@ -13,7 +13,7 @@ var types map[rune]RedisType = map[rune]RedisType{
 	notApplicableByte: "N/A",
 }
 
-var commands []RedisCommand = []RedisCommand{
+var commands = []RedisCommand{
 	"PING",
 	"SET",
 	"GET",
@@ -200,7 +200,7 @@ var commands []RedisCommand = []RedisCommand{
 	"XCLAIM",
 }
 
-var keywords []RedisKeyword = []RedisKeyword{
+var keywords = []RedisKeyword{
 	"AGGREGATE",
 	"ALPHA",
 	"ASC",

tap/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/go-errors/errors v1.4.2
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/golang-lru v0.5.4
-	github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e
+	github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/struCoder/pidusage v0.2.1
 	github.com/up9inc/mizu/logger v0.0.0
@@ -24,7 +24,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

tap/go.sum

@@ -68,8 +68,6 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
@@ -83,8 +81,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e h1:6J5obSn9umEThiYzWzndcPOZR0Qj/sVCZpH6V1G7yNE=
-github.com/knightsc/gapstone v0.0.0-20211014144438-5e0e64002a6e/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
+github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 h1:1KszOoXSFt0aRQ6wxxcKm7QKgfLPI0TWO47UcY/f+vA=
+github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9/go.mod h1:1K5hEzsMBLTPdRJKEHqBFJ8Zt2VRqDhomcQ11KH0WW4=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=

tap/passive_tapper.go

@@ -292,7 +292,7 @@ func startTlsTapper(extension *api.Extension, outputItems chan *api.OutputChanne
 	// A quick way to instrument libssl.so without PID filtering - used for debuging and troubleshooting
 	//
 	if os.Getenv("MIZU_GLOBAL_SSL_LIBRARY") != "" {
-		if err := tls.GlobalSsllibTap(os.Getenv("MIZU_GLOBAL_SSL_LIBRARY")); err != nil {
+		if err := tls.GlobalSSLLibTap(os.Getenv("MIZU_GLOBAL_SSL_LIBRARY")); err != nil {
 			tlstapper.LogError(err)
 			return nil
 		}

tap/tcp_assembler.go

@@ -85,7 +85,7 @@ func NewTcpAssembler(outputItems chan *api.OutputChannelItem, streamsMap api.Tcp
 
 	maxBufferedPagesTotal := GetMaxBufferedPagesPerConnection()
 	maxBufferedPagesPerConnection := GetMaxBufferedPagesTotal()
-	logger.Log.Infof("Assembler options: maxBufferedPagesTotal=%d, maxBufferedPagesPerConnection=%d, opts=%v",
+	logger.Log.Infof("Assembler options: maxBufferedPagesTotal=%d, maxBufferedPagesPerConnection=%d, opts=%+v",
 		maxBufferedPagesTotal, maxBufferedPagesPerConnection, opts)
 	a.Assembler.AssemblerOptions.MaxBufferedPagesTotal = maxBufferedPagesTotal
 	a.Assembler.AssemblerOptions.MaxBufferedPagesPerConnection = maxBufferedPagesPerConnection

tap/tlstapper/bpf/go_uprobes.c

@@ -60,7 +60,18 @@ Capstone Engine: https://www.capstone-engine.org/
 
 static __always_inline __u32 go_crypto_tls_get_fd_from_tcp_conn(struct pt_regs *ctx) {
     struct go_interface conn;
-    long err = bpf_probe_read(&conn, sizeof(conn), (void*)GO_ABI_INTERNAL_PT_REGS_R1(ctx));
+    long err;
+    __u64 addr;
+#if defined(bpf_target_arm64)
+    err = bpf_probe_read(&addr, sizeof(addr), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx)+0x8);
+    if (err != 0) {
+        return invalid_fd;
+    }
+#else
+    addr = GO_ABI_INTERNAL_PT_REGS_R1(ctx);
+#endif
+
+    err = bpf_probe_read(&conn, sizeof(conn), (void*)addr);
     if (err != 0) {
         return invalid_fd;
     }
@@ -88,14 +99,23 @@ static __always_inline void go_crypto_tls_uprobe(struct pt_regs *ctx, struct bpf
     }
 
     struct ssl_info info = new_ssl_info();
+    long err;
 
+#if defined(bpf_target_arm64)
+    err = bpf_probe_read(&info.buffer_len, sizeof(__u32), (void*)GO_ABI_INTERNAL_PT_REGS_SP(ctx)+0x18);
+    if (err != 0) {
+        log_error(ctx, LOG_ERROR_READING_BYTES_COUNT, pid_tgid, err, ORIGIN_SSL_UPROBE_CODE);
+        return;
+    }
+#else
     info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R2(ctx);
+#endif
     info.buffer = (void*)GO_ABI_INTERNAL_PT_REGS_R4(ctx);
     info.fd = go_crypto_tls_get_fd_from_tcp_conn(ctx);
 
     // GO_ABI_INTERNAL_PT_REGS_GP is Goroutine address
     __u64 pid_fp = pid << 32 | GO_ABI_INTERNAL_PT_REGS_GP(ctx);
-    long err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY);
+    err = bpf_map_update_elem(go_context, &pid_fp, &info, BPF_ANY);
 
     if (err != 0) {
         log_error(ctx, LOG_ERROR_PUTTING_SSL_CONTEXT, pid_tgid, err, 0l);
@@ -130,7 +150,15 @@ static __always_inline void go_crypto_tls_ex_uprobe(struct pt_regs *ctx, struct
 
     // In case of read, the length is determined on return
     if (flags == FLAGS_IS_READ_BIT) {
+#if defined(bpf_target_arm64)
+        // On ARM64 we look at a general-purpose register as an indicator of error return
+        if (GO_ABI_INTERNAL_PT_REGS_R6(ctx) == 0x10) {
+            return;
+        }
+        info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R7(ctx); // n in return n, nil
+#else
         info.buffer_len = GO_ABI_INTERNAL_PT_REGS_R1(ctx); // n in return n, nil
+#endif
         // This check achieves ignoring 0 length reads (the reads result with an error)
         if (info.buffer_len <= 0) {
             return;

tap/tlstapper/bpf/include/go_abi_internal.h

@@ -66,11 +66,12 @@ https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-
 https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD64Ops.go#L100
 */
 #define GO_ABI_INTERNAL_PT_REGS_R1(x) ((x)->eax)
-#define GO_ABI_INTERNAL_PT_REGS_P2(x) ((x)->ecx)
-#define GO_ABI_INTERNAL_PT_REGS_P3(x) ((x)->edx)
-#define GO_ABI_INTERNAL_PT_REGS_P4(x) 0
-#define GO_ABI_INTERNAL_PT_REGS_P5(x) 0
-#define GO_ABI_INTERNAL_PT_REGS_P6(x) 0
+#define GO_ABI_INTERNAL_PT_REGS_R2(x) ((x)->ecx)
+#define GO_ABI_INTERNAL_PT_REGS_R3(x) ((x)->edx)
+#define GO_ABI_INTERNAL_PT_REGS_R4(x) 0
+#define GO_ABI_INTERNAL_PT_REGS_R5(x) 0
+#define GO_ABI_INTERNAL_PT_REGS_R6(x) 0
+#define GO_ABI_INTERNAL_PT_REGS_R7(x) 0
 #define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->esp)
 #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->ebp)
 #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->e14)
@@ -83,6 +84,7 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/AMD6
 #define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->rbx)
 #define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->rbp)
 #define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->rsi)
+#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->rdi)
 #define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->rsp)
 #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->rbp)
 #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->r14)
@@ -101,7 +103,8 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/ARM6
 #define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->uregs[3])
 #define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->uregs[4])
 #define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->uregs[5])
-#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->uregs[14])
+#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->uregs[6])
+#define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->uregs[13])
 #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->uregs[29])
 #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->uregs[28])
 
@@ -116,7 +119,8 @@ struct pt_regs;
 #define GO_ABI_INTERNAL_PT_REGS_R4(x) (((PT_REGS_ARM64 *)(x))->regs[3])
 #define GO_ABI_INTERNAL_PT_REGS_R5(x) (((PT_REGS_ARM64 *)(x))->regs[4])
 #define GO_ABI_INTERNAL_PT_REGS_R6(x) (((PT_REGS_ARM64 *)(x))->regs[5])
-#define GO_ABI_INTERNAL_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->regs[30])
+#define GO_ABI_INTERNAL_PT_REGS_R7(x) (((PT_REGS_ARM64 *)(x))->regs[6])
+#define GO_ABI_INTERNAL_PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp)
 #define GO_ABI_INTERNAL_PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29])
 #define GO_ABI_INTERNAL_PT_REGS_GP(x) (((PT_REGS_ARM64 *)(x))->regs[28])
 
@@ -132,6 +136,7 @@ https://github.com/golang/go/blob/go1.17.6/src/cmd/compile/internal/ssa/gen/PPC6
 #define GO_ABI_INTERNAL_PT_REGS_R4(x) ((x)->gpr[6])
 #define GO_ABI_INTERNAL_PT_REGS_R5(x) ((x)->gpr[7])
 #define GO_ABI_INTERNAL_PT_REGS_R6(x) ((x)->gpr[8])
+#define GO_ABI_INTERNAL_PT_REGS_R7(x) ((x)->gpr[9])
 #define GO_ABI_INTERNAL_PT_REGS_SP(x) ((x)->sp)
 #define GO_ABI_INTERNAL_PT_REGS_FP(x) ((x)->gpr[12])
 #define GO_ABI_INTERNAL_PT_REGS_GP(x) ((x)->gpr[30])

tap/tlstapper/bpf/openssl_uprobes.c

@@ -48,8 +48,7 @@ static __always_inline void ssl_uprobe(struct pt_regs *ctx, void* ssl, void* buf
 		return;
 	}
 	
-	struct ssl_info *infoPtr = bpf_map_lookup_elem(map_fd, &id);
-	struct ssl_info info = lookup_ssl_info(ctx, &openssl_write_context, id);
+	struct ssl_info info = lookup_ssl_info(ctx, map_fd, id);
 	
 	info.count_ptr = count_ptr;
 	info.buffer = buffer;

tap/tlstapper/chunk.go

@@ -9,8 +9,8 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-const FLAGS_IS_CLIENT_BIT uint32 = (1 << 0)
-const FLAGS_IS_READ_BIT uint32 = (1 << 1)
+const FlagsIsClientBit uint32 = 1 << 0
+const FlagsIsReadBit uint32 = 1 << 1
 
 func (c *tlsTapperTlsChunk) getAddress() (net.IP, uint16, error) {
 	address := bytes.NewReader(c.Address[:])
@@ -36,7 +36,7 @@ func (c *tlsTapperTlsChunk) getAddress() (net.IP, uint16, error) {
 }
 
 func (c *tlsTapperTlsChunk) isClient() bool {
-	return c.Flags&FLAGS_IS_CLIENT_BIT != 0
+	return c.Flags&FlagsIsClientBit != 0
 }
 
 func (c *tlsTapperTlsChunk) isServer() bool {
@@ -44,7 +44,7 @@ func (c *tlsTapperTlsChunk) isServer() bool {
 }
 
 func (c *tlsTapperTlsChunk) isRead() bool {
-	return c.Flags&FLAGS_IS_READ_BIT != 0
+	return c.Flags&FlagsIsReadBit != 0
 }
 
 func (c *tlsTapperTlsChunk) isWrite() bool {

tap/tlstapper/go_offsets.go

@@ -77,7 +77,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro
 	case "arm64":
 		engine, err = gapstone.New(
 			gapstone.CS_ARCH_ARM64,
-			gapstone.CS_MODE_ARM,
+			gapstone.CS_MODE_LITTLE_ENDIAN,
 		)
 	default:
 		err = fmt.Errorf("Unsupported architecture: %v", runtime.GOARCH)
@@ -86,6 +86,16 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro
 		return
 	}
 
+	engineMajor, engineMinor := engine.Version()
+	logger.Log.Infof(
+		"Disassembling %s with Capstone %d.%d (arch: %d, mode: %d)",
+		filePath,
+		engineMajor,
+		engineMinor,
+		engine.Arch(),
+		engine.Mode(),
+	)
+
 	offsets = make(map[string]*goExtendedOffset)
 	var fd *os.File
 	fd, err = os.Open(filePath)
@@ -133,7 +143,7 @@ func getOffsets(filePath string) (offsets map[string]*goExtendedOffset, err erro
 		extendedOffset := &goExtendedOffset{enter: offset}
 
 		// source: https://gist.github.com/grantseltzer/3efa8ecc5de1fb566e8091533050d608
-		// skip over any symbols that aren't functinons/methods
+		// skip over any symbols that aren't functions/methods
 		if sym.Info != byte(2) && sym.Info != byte(18) {
 			offsets[sym.Name] = extendedOffset
 			continue

tap/tlstapper/ssllib_hooks.go

@@ -107,47 +107,47 @@ func (s *sslHooks) installSslHooks(bpfObjects *tlsTapperObjects, sslLibrary *lin
 }
 
 func (s *sslHooks) close() []error {
-	errors := make([]error, 0)
+	returnValue := make([]error, 0)
 
 	if err := s.sslWriteProbe.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sslWriteRetProbe.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sslReadProbe.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sslReadRetProbe.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if s.sslWriteExProbe != nil {
 		if err := s.sslWriteExProbe.Close(); err != nil {
-			errors = append(errors, err)
+			returnValue = append(returnValue, err)
 		}
 	}
 
 	if s.sslWriteExRetProbe != nil {
 		if err := s.sslWriteExRetProbe.Close(); err != nil {
-			errors = append(errors, err)
+			returnValue = append(returnValue, err)
 		}
 	}
 
 	if s.sslReadExProbe != nil {
 		if err := s.sslReadExProbe.Close(); err != nil {
-			errors = append(errors, err)
+			returnValue = append(returnValue, err)
 		}
 	}
 
 	if s.sslReadExRetProbe != nil {
 		if err := s.sslReadExRetProbe.Close(); err != nil {
-			errors = append(errors, err)
+			returnValue = append(returnValue, err)
 		}
 	}
 
-	return errors
+	return returnValue
 }

tap/tlstapper/syscall_hooks.go

@@ -57,31 +57,31 @@ func (s *syscallHooks) installSyscallHooks(bpfObjects *tlsTapperObjects) error {
 }
 
 func (s *syscallHooks) close() []error {
-	errors := make([]error, 0)
+	returnValue := make([]error, 0)
 
 	if err := s.sysEnterRead.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sysEnterWrite.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sysEnterAccept4.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sysExitAccept4.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sysEnterConnect.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := s.sysExitConnect.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
-	return errors
+	return returnValue
 }

tap/tlstapper/tls_poller.go

@@ -273,13 +273,13 @@ func (p *tlsPoller) getNamespace(pid uint32) string {
 	namespaceIfc, ok := p.pidToNamespace.Load(pid)
 
 	if !ok {
-		return api.UNKNOWN_NAMESPACE
+		return api.UnknownNamespace
 	}
 
 	namespace, ok := namespaceIfc.(string)
 
 	if !ok {
-		return api.UNKNOWN_NAMESPACE
+		return api.UnknownNamespace
 	}
 
 	return namespace

tap/tlstapper/tls_process_discoverer.go

@@ -28,7 +28,7 @@ func UpdateTapTargets(tls *TlsTapper, pods *[]v1.Pod, procfs string) error {
 	tls.ClearPids()
 
 	for pid, pod := range containerPids {
-		if err := tls.AddSsllibPid(procfs, pid, pod.Namespace); err != nil {
+		if err := tls.AddSSLLibPid(procfs, pid, pod.Namespace); err != nil {
 			LogError(err)
 		}
 
@@ -90,14 +90,14 @@ func buildContainerIdsMap(pods *[]v1.Pod) map[string]v1.Pod {
 
 	for _, pod := range *pods {
 		for _, container := range pod.Status.ContainerStatuses {
-			url, err := url.Parse(container.ContainerID)
+			parsedUrl, err := url.Parse(container.ContainerID)
 
 			if err != nil {
 				logger.Log.Warningf("Expecting URL like container ID %v", container.ContainerID)
 				continue
 			}
 
-			result[url.Host] = pod
+			result[parsedUrl.Host] = pod
 		}
 	}
 

tap/tlstapper/tls_tapper.go

@@ -10,7 +10,7 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-const GLOABL_TAP_PID = 0
+const GlobalTapPid = 0
 
 //go:generate go run github.com/cilium/ebpf/cmd/bpf2go@0d0727ef53e2f53b1731c73f4c61e0f58693083a -target $BPF_TARGET -cflags $BPF_CFLAGS -type tls_chunk tlsTapper bpf/tls_tapper.c
 
@@ -66,8 +66,8 @@ func (t *TlsTapper) PollForLogging() {
 	t.bpfLogger.poll()
 }
 
-func (t *TlsTapper) GlobalSsllibTap(sslLibrary string) error {
-	return t.tapSsllibPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE)
+func (t *TlsTapper) GlobalSSLLibTap(sslLibrary string) error {
+	return t.tapSSLLibPid(GlobalTapPid, sslLibrary, api.UnknownNamespace)
 }
 
 func (t *TlsTapper) GlobalGoTap(procfs string, pid string) error {
@@ -76,18 +76,18 @@ func (t *TlsTapper) GlobalGoTap(procfs string, pid string) error {
 		return err
 	}
 
-	return t.tapGoPid(procfs, uint32(_pid), api.UNKNOWN_NAMESPACE)
+	return t.tapGoPid(procfs, uint32(_pid), api.UnknownNamespace)
 }
 
-func (t *TlsTapper) AddSsllibPid(procfs string, pid uint32, namespace string) error {
+func (t *TlsTapper) AddSSLLibPid(procfs string, pid uint32, namespace string) error {
 	sslLibrary, err := findSsllib(procfs, pid)
 
 	if err != nil {
 		logger.Log.Infof("PID skipped no libssl.so found (pid: %d) %v", pid, err)
-		return nil // hide the error on purpose, its OK for a process to not use libssl.so
+		return nil // hide the error on purpose, it's OK for a process to not use libssl.so
 	}
 
-	return t.tapSsllibPid(pid, sslLibrary, namespace)
+	return t.tapSSLLibPid(pid, sslLibrary, namespace)
 }
 
 func (t *TlsTapper) AddGoPid(procfs string, pid uint32, namespace string) error {
@@ -110,7 +110,7 @@ func (t *TlsTapper) ClearPids() {
 	t.poller.clearPids()
 	t.registeredPids.Range(func(key, v interface{}) bool {
 		pid := key.(uint32)
-		if pid == GLOABL_TAP_PID {
+		if pid == GlobalTapPid {
 			return true
 		}
 
@@ -123,31 +123,31 @@ func (t *TlsTapper) ClearPids() {
 }
 
 func (t *TlsTapper) Close() []error {
-	errors := make([]error, 0)
+	returnValue := make([]error, 0)
 
 	if err := t.bpfObjects.Close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
-	errors = append(errors, t.syscallHooks.close()...)
+	returnValue = append(returnValue, t.syscallHooks.close()...)
 
 	for _, sslHooks := range t.sslHooksStructs {
-		errors = append(errors, sslHooks.close()...)
+		returnValue = append(returnValue, sslHooks.close()...)
 	}
 
 	for _, goHooks := range t.goHooksStructs {
-		errors = append(errors, goHooks.close()...)
+		returnValue = append(returnValue, goHooks.close()...)
 	}
 
 	if err := t.bpfLogger.close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
 	if err := t.poller.close(); err != nil {
-		errors = append(errors, err)
+		returnValue = append(returnValue, err)
 	}
 
-	return errors
+	return returnValue
 }
 
 func setupRLimit() error {
@@ -160,7 +160,7 @@ func setupRLimit() error {
 	return nil
 }
 
-func (t *TlsTapper) tapSsllibPid(pid uint32, sslLibrary string, namespace string) error {
+func (t *TlsTapper) tapSSLLibPid(pid uint32, sslLibrary string, namespace string) error {
 	newSsl := sslHooks{}
 
 	if err := newSsl.installUprobes(&t.bpfObjects, sslLibrary); err != nil {
@@ -193,7 +193,8 @@ func (t *TlsTapper) tapGoPid(procfs string, pid uint32, namespace string) error
 	hooks := goHooks{}
 
 	if err := hooks.installUprobes(&t.bpfObjects, exePath); err != nil {
-		return err
+		logger.Log.Infof("PID skipped not a Go binary or symbol table is stripped (pid: %v) %v", pid, exePath)
+		return nil // hide the error on purpose, its OK for a process to be not a Go binary or stripped Go binary
 	}
 
 	logger.Log.Infof("Tapping TLS (pid: %v) (Go: %v)", pid, exePath)