Replace source IP with X-Forwarded-For in http, if exists (#606)

Support source IP resolving for HTTP traffic in Istio service mesh.

If X-Forwarded-For HTTP request header is present, replace the source address with the left-most address in X-Forwarded-For (earliest in Envoy implementation of this header).

This allows Mizu to resolve source IPs in Istio service meshes, if the use_remote_address option is on. Added instructions on how to turn it on.

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent cli tap docker
+.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -28,6 +28,9 @@ ui: ## Build UI.
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
+cli-debug: ## Build CLI.
+	@echo "building cli"; cd cli && $(MAKE) build-debug
+
 build-cli-ci: ## Build CLI for CI.
 	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
 
@@ -37,6 +40,12 @@ agent: ## Build agent.
 	${MAKE} extensions
 	@ls -l agent/build
 
+agent-debug: ## Build agent for debug.
+	@(echo "building mizu agent for debug.." )
+	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
+	${MAKE} extensions-debug
+	@ls -l agent/build
+
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
@@ -62,7 +71,7 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
+clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
@@ -73,9 +82,15 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
+clean-extensions:  ## Clean extensions
+	@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
+
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+extensions-debug:
+	devops/build_extensions_debug.sh
+
 extensions:
 	devops/build_extensions.sh
 

agent/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/up9inc/basenine/client/go v0.0.0-20220107003657-7c0578359920
+	github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0

agent/go.sum

@@ -472,8 +472,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20220107003657-7c0578359920 h1:QQpgRleNNpxxAG/rKmk4dwJh0jHyRaQz4QOVlPmqv1c=
-github.com/up9inc/basenine/client/go v0.0.0-20220107003657-7c0578359920/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d h1:WTz53dcfqCIWZpZLQoHbIcNc21s0ZHEZH7EqMPp99qQ=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=

agent/pkg/api/main.go

@@ -119,15 +119,12 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
-		baseEntry := extension.Dissector.Summarize(mizuEntry)
-		mizuEntry.Base = baseEntry
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
 				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				baseEntry.ContractStatus = contract.Status
 				mizuEntry.ContractStatus = contract.Status
 				mizuEntry.ContractRequestReason = contract.RequestReason
 				mizuEntry.ContractResponseReason = contract.ResponseReason
@@ -137,7 +134,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
-				baseEntry.Rules = rules
+				mizuEntry.Rules = rules
 			}
 		}
 

agent/pkg/api/socket_routes.go

@@ -16,6 +16,7 @@ import (
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EventHandlers interface {
@@ -131,18 +132,10 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 						return
 					}
 
-					var dataMap map[string]interface{}
-					err = json.Unmarshal(bytes, &dataMap)
+					var entry *tapApi.Entry
+					err = json.Unmarshal(bytes, &entry)
 
-					var base map[string]interface{}
-					switch dataMap["base"].(type) {
-					case map[string]interface{}:
-						base = dataMap["base"].(map[string]interface{})
-						base["id"] = uint(dataMap["id"].(float64))
-					default:
-						logger.Log.Debugf("Base field has an unrecognized type: %+v", dataMap)
-						continue
-					}
+					base := tapApi.Summarize(entry)
 
 					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
 					SendToSocket(socketId, baseEntryBytes)

agent/pkg/api/socket_server_handlers.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappersCount"
 	"mizuserver/pkg/up9"
 	"sync"
 
@@ -29,7 +30,7 @@ func init() {
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
-		providers.TapperAdded()
+		tappersCount.Add()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
 		socketListLock.Lock()
@@ -41,7 +42,7 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
-		providers.TapperRemoved()
+		tappersCount.Remove()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()

agent/pkg/controllers/config_controller.go

@@ -11,18 +11,20 @@ import (
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tapConfig"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappersStatus"
 	"net/http"
 	"regexp"
 	"time"
 )
 
-var globalTapConfig = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
 var cancelTapperSyncer context.CancelFunc
 
 func PostTapConfig(c *gin.Context) {
-	tapConfig := &models.TapConfig{}
+	requestTapConfig := &models.TapConfig{}
 
-	if err := c.Bind(tapConfig); err != nil {
+	if err := c.Bind(requestTapConfig); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
@@ -30,14 +32,14 @@ func PostTapConfig(c *gin.Context) {
 	if cancelTapperSyncer != nil {
 		cancelTapperSyncer()
 
-		providers.TapStatus = shared.TapStatus{}
-		providers.TappersStatus = make(map[string]shared.TapperStatus)
+		tappedPods.Set([]*shared.PodInfo{})
+		tappersStatus.Reset()
 
 		broadcastTappedPodsStatus()
 	}
 
 	var tappedNamespaces []string
-	for namespace, tapped := range tapConfig.TappedNamespaces {
+	for namespace, tapped := range requestTapConfig.TappedNamespaces {
 		if tapped {
 			tappedNamespaces = append(tappedNamespaces, namespace)
 		}
@@ -60,7 +62,7 @@ func PostTapConfig(c *gin.Context) {
 	}
 
 	cancelTapperSyncer = cancel
-	globalTapConfig = tapConfig
+	tapConfig.Save(requestTapConfig)
 
 	c.JSON(http.StatusOK, "OK")
 }
@@ -81,17 +83,19 @@ func GetTapConfig(c *gin.Context) {
 		return
 	}
 
+	savedTapConfig := tapConfig.Get()
+
 	tappedNamespaces := make(map[string]bool)
 	for _, namespace := range namespaces {
 		if namespace.Name == config.Config.MizuResourcesNamespace {
 			continue
 		}
 
-		tappedNamespaces[namespace.Name] = globalTapConfig.TappedNamespaces[namespace.Name]
+		tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
 	}
 
-	tapConfig := models.TapConfig{TappedNamespaces: tappedNamespaces}
-	c.JSON(http.StatusOK, tapConfig)
+	tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
+	c.JSON(http.StatusOK, tapConfigToReturn)
 }
 
 func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
@@ -129,7 +133,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 					return
 				}
 
-				providers.TapStatus = shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)}
+				tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
 				broadcastTappedPodsStatus()
 			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
 				if !ok {
@@ -137,7 +141,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 					return
 				}
 
-				addTapperStatus(tapperStatus)
+				tappersStatus.Set(&tapperStatus)
 				broadcastTappedPodsStatus()
 			case <-ctx.Done():
 				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")

agent/pkg/controllers/entries_controller.go

@@ -64,8 +64,8 @@ func GetEntries(c *gin.Context) {
 	var dataSlice []interface{}
 
 	for _, row := range data {
-		var dataMap map[string]interface{}
-		err = json.Unmarshal(row, &dataMap)
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{
 				"error":     true,
@@ -76,8 +76,7 @@ func GetEntries(c *gin.Context) {
 			return // exit
 		}
 
-		base := dataMap["base"].(map[string]interface{})
-		base["id"] = uint(dataMap["id"].(float64))
+		base := tapApi.Summarize(entry)
 
 		dataSlice = append(dataSlice, base)
 	}
@@ -95,9 +94,19 @@ func GetEntries(c *gin.Context) {
 }
 
 func GetEntry(c *gin.Context) {
+	singleEntryRequest := &models.SingleEntryRequest{}
+
+	if err := c.BindQuery(singleEntryRequest); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	validationError := validation.Validate(singleEntryRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
+	}
+
 	id, _ := strconv.Atoi(c.Param("id"))
-	var entry tapApi.MizuEntry
-	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id)
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
 	if Error(c, err) {
 		return // exit
 	}
@@ -125,7 +134,7 @@ func GetEntry(c *gin.Context) {
 		json.Unmarshal(inrec, &rules)
 	}
 
-	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
+	c.JSON(http.StatusOK, tapApi.EntryWrapper{
 		Protocol:       entry.Protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,

agent/pkg/controllers/status_controller.go

@@ -8,43 +8,42 @@ import (
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappersCount"
+	"mizuserver/pkg/providers/tappersStatus"
 	"mizuserver/pkg/up9"
-	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {
-	tappers := make([]shared.TapperStatus, 0)
-	for _, value := range providers.TappersStatus {
+	tappers := make([]*shared.TapperStatus, 0)
+	for _, value := range tappersStatus.Get() {
 		tappers = append(tappers, value)
 	}
 
 	response := shared.HealthResponse{
-		TapStatus:     providers.TapStatus,
-		TappersCount:  providers.TappersCount,
+		TappedPods:    tappedPods.Get(),
+		TappersCount:  tappersCount.Get(),
 		TappersStatus: tappers,
 	}
 	c.JSON(http.StatusOK, response)
 }
 
 func PostTappedPods(c *gin.Context) {
-	tapStatus := &shared.TapStatus{}
-	if err := c.Bind(tapStatus); err != nil {
+	var requestTappedPods []*shared.PodInfo
+	if err := c.Bind(&requestTappedPods); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	if err := validation.Validate(tapStatus); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-	logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
-	providers.TapStatus.Pods = tapStatus.Pods
+
+	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
+	tappedPods.Set(requestTappedPods)
 	broadcastTappedPodsStatus()
 }
 
 func broadcastTappedPodsStatus() {
-	tappedPodsStatus := utils.GetTappedPodsStatus()
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
 
 	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
 	if jsonBytes, err := json.Marshal(message); err != nil {
@@ -54,14 +53,6 @@ func broadcastTappedPodsStatus() {
 	}
 }
 
-func addTapperStatus(tapperStatus shared.TapperStatus) {
-	if providers.TappersStatus == nil {
-		providers.TappersStatus = make(map[string]shared.TapperStatus)
-	}
-
-	providers.TappersStatus[tapperStatus.NodeName] = tapperStatus
-}
-
 func PostTapperStatus(c *gin.Context) {
 	tapperStatus := &shared.TapperStatus{}
 	if err := c.Bind(tapperStatus); err != nil {
@@ -75,12 +66,12 @@ func PostTapperStatus(c *gin.Context) {
 	}
 
 	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
-	addTapperStatus(*tapperStatus)
+	tappersStatus.Set(tapperStatus)
 	broadcastTappedPodsStatus()
 }
 
 func GetTappersCount(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TappersCount)
+	c.JSON(http.StatusOK, tappersCount.Get())
 }
 
 func GetAuthStatus(c *gin.Context) {
@@ -94,7 +85,7 @@ func GetAuthStatus(c *gin.Context) {
 }
 
 func GetTappingStatus(c *gin.Context) {
-	tappedPodsStatus := utils.GetTappedPodsStatus()
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
 	c.JSON(http.StatusOK, tappedPodsStatus)
 }
 

agent/pkg/models/models.go

@@ -12,7 +12,7 @@ import (
 	"github.com/up9inc/mizu/tap"
 )
 
-func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
+func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
@@ -28,6 +28,10 @@ type EntriesRequest struct {
 	TimeoutMs int    `form:"timeoutMs" validate:"min=1"`
 }
 
+type SingleEntryRequest struct {
+	Query string `form:"query"`
+}
+
 type EntriesResponse struct {
 	Data []interface{}      `json:"data"`
 	Meta *basenine.Metadata `json:"meta"`
@@ -35,7 +39,7 @@ type EntriesResponse struct {
 
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data map[string]interface{} `json:"data,omitempty"`
+	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
@@ -74,7 +78,7 @@ type WebSocketStartTimeMessage struct {
 	Data int64 `json:"data"`
 }
 
-func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) {
+func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,

agent/pkg/providers/status_provider.go

@@ -8,19 +8,14 @@ import (
 	"github.com/up9inc/mizu/tap"
 	"mizuserver/pkg/models"
 	"os"
-	"sync"
 	"time"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15
 
 var (
-	TappersCount         int
-	TapStatus            shared.TapStatus
-	TappersStatus        map[string]shared.TapperStatus
-	authStatus           *models.AuthStatus
-	RecentTLSLinks       = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
-	tappersCountLock     = sync.Mutex{}
+	authStatus     *models.AuthStatus
+	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
 )
 
 func GetAuthStatus() (*models.AuthStatus, error) {
@@ -68,15 +63,3 @@ func GetAllRecentTLSAddresses() []string {
 
 	return recentTLSLinks
 }
-
-func TapperAdded() {
-	tappersCountLock.Lock()
-	TappersCount++
-	tappersCountLock.Unlock()
-}
-
-func TapperRemoved() {
-	tappersCountLock.Lock()
-	TappersCount--
-	tappersCountLock.Unlock()
-}

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -0,0 +1,54 @@
+package tapConfig
+
+import (
+	"encoding/json"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"io/ioutil"
+	"mizuserver/pkg/models"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tap-config.json"
+
+var lock = &sync.Mutex{}
+
+var config *models.TapConfig
+
+func Get() *models.TapConfig {
+	if config == nil {
+		lock.Lock()
+		defer lock.Unlock()
+
+		if config == nil {
+			if content, err := ioutil.ReadFile(FilePath); err != nil {
+				config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
+				if !os.IsNotExist(err) {
+					logger.Log.Errorf("Error loading tap config from file, err: %v", err)
+				}
+			} else {
+				if err = json.Unmarshal(content, &config); err != nil {
+					config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
+					logger.Log.Errorf("Error while unmarshal tap config, err: %v", err)
+				}
+			}
+		}
+	}
+
+	return config
+}
+
+func Save(tapConfigToSave *models.TapConfig) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	config = tapConfigToSave
+	if data, err := json.Marshal(config); err != nil {
+		logger.Log.Errorf("Error while marshal tap config, err: %v", err)
+	} else {
+		if err := ioutil.WriteFile(FilePath, data, 0644); err != nil {
+			logger.Log.Errorf("Error writing tap config to file, err: %v", err)
+		}
+	}
+}

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -0,0 +1,32 @@
+package tappedPods
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"mizuserver/pkg/providers/tappersStatus"
+	"strings"
+)
+
+var tappedPods []*shared.PodInfo
+
+func Get() []*shared.PodInfo {
+	return tappedPods
+}
+
+func Set(tappedPodsToSet []*shared.PodInfo) {
+	tappedPods = tappedPodsToSet
+}
+
+func GetTappedPodsStatus() []shared.TappedPodStatus {
+	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
+	for _, pod := range Get() {
+		var status string
+		if tapperStatus, ok := tappersStatus.Get()[pod.NodeName]; ok {
+			status = strings.ToLower(tapperStatus.Status)
+		}
+
+		isTapped := status == "running"
+		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
+	}
+
+	return tappedPodsStatus
+}

agent/pkg/providers/tappersCount/tappers_count_provider.go

@@ -0,0 +1,25 @@
+package tappersCount
+
+import "sync"
+
+var lock = &sync.Mutex{}
+
+var tappersCount int
+
+func Add() {
+	lock.Lock()
+	defer lock.Unlock()
+
+	tappersCount++
+}
+
+func Remove() {
+	lock.Lock()
+	defer lock.Unlock()
+
+	tappersCount--
+}
+
+func Get() int {
+	return tappersCount
+}

agent/pkg/providers/tappersStatus/tappers_status_provider.go

@@ -0,0 +1,25 @@
+package tappersStatus
+
+import "github.com/up9inc/mizu/shared"
+
+var tappersStatus map[string]*shared.TapperStatus
+
+func Get() map[string]*shared.TapperStatus {
+	if tappersStatus == nil {
+		tappersStatus = make(map[string]*shared.TapperStatus)
+	}
+
+	return tappersStatus
+}
+
+func Set(tapperStatus *shared.TapperStatus) {
+	if tappersStatus == nil {
+		tappersStatus = make(map[string]*shared.TapperStatus)
+	}
+
+	tappersStatus[tapperStatus.NodeName] = tapperStatus
+}
+
+func Reset() {
+	tappersStatus = make(map[string]*shared.TapperStatus)
+}

agent/pkg/up9/main.go

@@ -243,7 +243,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			var dataMap map[string]interface{}
 			err = json.Unmarshal(dataBytes, &dataMap)
 
-			var entry tapApi.MizuEntry
+			var entry tapApi.Entry
 			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
 				continue
 			}

agent/pkg/utils/utils.go

@@ -3,12 +3,10 @@ package utils
 import (
 	"context"
 	"fmt"
-	"mizuserver/pkg/providers"
 	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 	"time"
 
@@ -45,16 +43,6 @@ func StartServer(app *gin.Engine) {
 	}
 }
 
-func GetTappedPodsStatus() []shared.TappedPodStatus {
-	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
-	for _, pod := range providers.TapStatus.Pods {
-		status := strings.ToLower(providers.TappersStatus[pod.NodeName].Status)
-		isTapped := status == "running"
-		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
-	}
-	return tappedPodsStatus
-}
-
 func CheckErr(e error) {
 	if e != nil {
 		logger.Log.Errorf("%v", e)

cli/Makefile

@@ -14,8 +14,12 @@ help: ## This help.
 install:
 	go install mizu.go
 
+build-debug:
+	export GCLFAGS='-gcflags="all=-N -l"'
+	${MAKE} build
+
 build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
-	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
+	go build ${GCLFAGS} -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \

cli/apiserver/provider.go

@@ -87,9 +87,8 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 	tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url)
 
 	podInfos := kubernetes.GetPodInfosForPods(pods)
-	tapStatus := shared.TapStatus{Pods: podInfos}
 
-	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+	if jsonValue, err := json.Marshal(podInfos); err != nil {
 		return fmt.Errorf("failed Marshal the tapped pods %w", err)
 	} else {
 		if response, err := provider.client.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {

cli/cmd/installRunner.go

@@ -89,6 +89,8 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
 		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
 		AgentDatabasePath:      shared.DataDirPath,
 		StandaloneMode:         true,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
 	}
 
 	return &mizuAgentConfig
@@ -99,7 +101,7 @@ func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes.
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
 	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 
-	timeAfter := time.After(30 * time.Second)
+	timeAfter := time.After(1 * time.Minute)
 	for {
 		select {
 		case wEvent, ok := <-eventChan:

cli/cmd/tapRunner.go

@@ -156,6 +156,8 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		TapperResources:        config.Config.Tap.TapperResources,
 		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
 		AgentDatabasePath:      shared.DataDirPath,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
 	}
 
 	return &mizuAgentConfig

cli/config/configStruct.go

@@ -34,9 +34,11 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map" default:"false" readonly:""`
+	OAS                    bool                        `yaml:"oas" default:"false" readonly:""`
 }
 
-func(config *ConfigStruct) validate() error {
+func (config *ConfigStruct) validate() error {
 	if _, err := logging.LogLevel(config.LogLevelStr); err != nil {
 		return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err)
 	}

deploy/kubernetes/helm-chart/values.yaml

@@ -32,7 +32,7 @@ container:
     port: 9099
     image:
       repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/basenine"
-      tag: "v0.2.26"
+      tag: "v0.3.0"
   kratos:
     name: "kratos"
     port: 4433

devops/build_extensions.sh

@@ -2,6 +2,7 @@
 
 for f in tap/extensions/*; do
     if [ -d "$f" ]; then
+        echo Building extension: $f
         extension=$(basename $f) && \
         cd tap/extensions/${extension} && \
         go build -buildmode=plugin -o ../${extension}.so .  && \

docs/SERVICE_MESH.md

@@ -1,5 +1,7 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
+
 # Service mesh mutual tls (mtls) with Mizu
+
 This document describe how Mizu tapper handles workloads configured with mtls, making the internal traffic between services in a cluster to be encrypted.
 
 The list of service meshes supported by Mizu include:
@@ -7,38 +9,67 @@ The list of service meshes supported by Mizu include:
 - Istio
 - Linkerd
 
-In order to create a service mesh setup for development, follow those steps:
-1. Deploy a sample application to a Kubernetes cluster, the sample application needs to make internal service to service calls
-2. SSH to one of the nodes, and run `tcpdump`
-3. Make sure you see the internal service to service calls in a plain text
-4. Deploy a service mesh (Istio, Linkerd) to the cluster - make sure it is attached to all pods of the sample application, and that it is configured with mtls (default)
-5. Run `tcpdump` again, make sure you don't see the internal service to service calls in a plain text 
+## Installation
+
+### Optional: Allow source IP resolving in Istio
+
+When using Istio, in order to enable Mizu to reslove source IPs to names, turn on the [use_remote_address](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for) option in Istio sidecar Envoys.
+This setting causes the Envoys to append to `X-Forwarded-For` request header. Mizu in turn uses the `X-Forwarded-For` header to determine the true source IPs.
+One way to turn on the `use_remote_address` HTTP connection manager option is by applying an `EnvoyFilter`:
+
+```yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name: mizu-xff
+  namespace: istio-system # as defined in meshConfig resource.
+spec:
+  configPatches:
+  - applyTo: NETWORK_FILTER
+    match:
+      context: SIDECAR_OUTBOUND # will match outbound listeners in all sidecars
+    patch:
+      operation: MERGE
+      value:
+        typed_config:
+          "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"
+          use_remote_address: true
+```
+
+Save the above text to `mizu-xff-envoyfilter.yaml` and run `kubectl apply -f mizu-xff-envoyfilter.yaml`.
+
+With Istio, mizu does not resolve source IPs for non-HTTP traffic.
 
 ## Implementation
 
 ### Istio support
 
 #### The connection between Istio and Envoy
+
 In order to implement its service mesh capabilities, [Istio](https://istio.io) uses an [Envoy](https://www.envoyproxy.io) sidecar in front of every pod in the cluster. The Envoy is responsible for the mtls communication, and that's why we are focusing on Envoy proxy.
 
 In the future we might see more players in that field, then we'll have to either add support for each of them or go with a unified eBPF solution.
 
 #### Network namespaces
+
 A [linux network namespace](https://man7.org/linux/man-pages/man7/network_namespaces.7.html) is an isolation that limit the process view of the network. In the container world it used to isolate one container from another. In the Kubernetes world it used to isolate a pod from another. That means that two containers running on the same pod share the same network namespace. A container can reach a container in the same pod by accessing `localhost`.
 
 An Envoy proxy configured with mtls receives the inbound traffic directed to the pod, decrypts it and sends it via `localhost` to the target container.
 
 #### Tapping mtls traffic
-In order for Mizu to be able to see the decrypted traffic it needs to listen on the same network namespace of the target pod. Multiple threads of the same process can have different network namespaces. 
+
+In order for Mizu to be able to see the decrypted traffic it needs to listen on the same network namespace of the target pod. Multiple threads of the same process can have different network namespaces.
 
 [gopacket](https://github.com/google/gopacket) uses [libpacp](https://github.com/the-tcpdump-group/libpcap) by default for capturing the traffic. Libpacap doesn't support network namespaces and we can't ask it to listen to traffic on a different namespace. However, we can change the network namespace of the calling thread and then start libpcap to see the traffic on a different namespace.
 
 #### Finding the network namespace of a running process
+
 The network namespace of a running process can be found in `/proc/PID/ns/net` link. Once we have this link, we can ask Linux to change the network namespace of a thread to this one.
 
 This mean that Mizu needs to have access to the `/proc` (procfs) of the running node.
 
 #### Finding the network namespace of a running pod
+
 In order for Mizu to be able to listen to mtls traffic, it needs to get the PIDs of the the running pods, filter them according to the user filters and then start listen to their internal network namespace traffic.
 
 There is no official way in Kubernetes to get from pod to PID. The CRI implementation purposefully doesn't force a pod to be a processes on the host. It can be a Virtual Machine as well like [Kata containers](https://katacontainers.io)
@@ -50,4 +81,15 @@ Once Mizu detects an Envoy process, it need to check whether this specific Envoy
 Istio sends an `INSTANCE_IP` environment variable to every Envoy proxy process. By examining the Envoy process's environment variables we can see whether it's relevant or not. Examining a process environment variables is done by reading the `/proc/PID/envion` file.
 
 #### Edge cases
+
 The method we use to find Envoy processes and correlate them to the cluster IPs may be inaccurate in certain situations. If, for example, a user runs an Envoy process manually, and set its `INSTANCE_IP` environment variable to one of the `CLUSTER_IPS` the tapper gets, then Mizu will capture traffic for it.
+
+## Development
+
+In order to create a service mesh setup for development, follow those steps:
+
+1. Deploy a sample application to a Kubernetes cluster, the sample application needs to make internal service to service calls
+2. SSH to one of the nodes, and run `tcpdump`
+3. Make sure you see the internal service to service calls in a plain text
+4. Deploy a service mesh (Istio, Linkerd) to the cluster - make sure it is attached to all pods of the sample application, and that it is configured with mtls (default)
+5. Run `tcpdump` again, make sure you don't see the internal service to service calls in a plain text

shared/consts.go

@@ -17,5 +17,5 @@ const (
 	BasenineHost                     = "127.0.0.1"
 	BaseninePort                     = "9099"
 	BasenineImageRepo                = "ghcr.io/up9inc/basenine"
-	BasenineImageTag                 = "v0.2.26"
+	BasenineImageTag                 = "v0.3.0"
 )

shared/kubernetes/utils.go

@@ -73,10 +73,10 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
 	return missingPods
 }
 
-func GetPodInfosForPods(pods []core.Pod) []shared.PodInfo {
-	podInfos := make([]shared.PodInfo, 0)
+func GetPodInfosForPods(pods []core.Pod) []*shared.PodInfo {
+	podInfos := make([]*shared.PodInfo, 0)
 	for _, pod := range pods {
-		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace, NodeName: pod.Spec.NodeName})
+		podInfos = append(podInfos, &shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace, NodeName: pod.Spec.NodeName})
 	}
 	return podInfos
 }

shared/models.go

@@ -41,6 +41,8 @@ type MizuAgentConfig struct {
 	MizuResourcesNamespace string        `json:"mizuResourceNamespace"`
 	AgentDatabasePath      string        `json:"agentDatabasePath"`
 	StandaloneMode         bool          `json:"standaloneMode"`
+	ServiceMap             bool          `json:"serviceMap"`
+	OAS                    bool          `json:"oas"`
 }
 
 type WebSocketMessageMetadata struct {
@@ -81,10 +83,6 @@ type TappedPodStatus struct {
 	IsTapped  bool   `json:"isTapped"`
 }
 
-type TapStatus struct {
-	Pods []PodInfo `json:"pods"`
-}
-
 type PodInfo struct {
 	Namespace string `json:"namespace"`
 	Name      string `json:"name"`
@@ -124,9 +122,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 }
 
 type HealthResponse struct {
-	TapStatus     TapStatus      `json:"tapStatus"`
-	TappersCount  int            `json:"tappersCount"`
-	TappersStatus []TapperStatus `json:"tappersStatus"`
+	TappedPods    []*PodInfo      `json:"tappedPods"`
+	TappersCount  int             `json:"tappersCount"`
+	TappersStatus []*TapperStatus `json:"tappersStatus"`
 }
 
 type VersionResponse struct {

tap/api/api.go

@@ -81,7 +81,7 @@ type OutputChannelItem struct {
 	Timestamp      int64
 	ConnectionInfo *ConnectionInfo
 	Pair           *RequestResponsePair
-	Summary        *BaseEntryDetails
+	Summary        *BaseEntry
 }
 
 type SuperTimer struct {
@@ -97,8 +97,7 @@ type Dissector interface {
 	Register(*Extension)
 	Ping()
 	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error
-	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *MizuEntry
-	Summarize(entry *MizuEntry) *BaseEntryDetails
+	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *Entry
 	Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error)
 	Macros() map[string]string
 }
@@ -117,7 +116,7 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 	e.AppStats.IncMatchedPairs()
 }
 
-type MizuEntry struct {
+type Entry struct {
 	Id                     uint                   `json:"id"`
 	Protocol               Protocol               `json:"proto"`
 	Source                 *TCP                   `json:"src"`
@@ -127,13 +126,13 @@ type MizuEntry struct {
 	StartTime              time.Time              `json:"startTime"`
 	Request                map[string]interface{} `json:"request"`
 	Response               map[string]interface{} `json:"response"`
-	Base                   *BaseEntryDetails      `json:"base"`
 	Summary                string                 `json:"summary"`
 	Method                 string                 `json:"method"`
 	Status                 int                    `json:"status"`
 	ElapsedTime            int64                  `json:"elapsedTime"`
 	Path                   string                 `json:"path"`
 	IsOutgoing             bool                   `json:"isOutgoing,omitempty"`
+	Rules                  ApplicableRules        `json:"rules,omitempty"`
 	ContractStatus         ContractStatus         `json:"contractStatus,omitempty"`
 	ContractRequestReason  string                 `json:"contractRequestReason,omitempty"`
 	ContractResponseReason string                 `json:"contractResponseReason,omitempty"`
@@ -141,22 +140,22 @@ type MizuEntry struct {
 	HTTPPair               string                 `json:"httpPair,omitempty"`
 }
 
-type MizuEntryWrapper struct {
+type EntryWrapper struct {
 	Protocol       Protocol                 `json:"protocol"`
 	Representation string                   `json:"representation"`
 	BodySize       int64                    `json:"bodySize"`
-	Data           MizuEntry                `json:"data"`
+	Data           *Entry                   `json:"data"`
 	Rules          []map[string]interface{} `json:"rulesMatched,omitempty"`
 	IsRulesEnabled bool                     `json:"isRulesEnabled"`
 }
 
-type BaseEntryDetails struct {
+type BaseEntry struct {
 	Id             uint            `json:"id"`
-	Protocol       Protocol        `json:"protocol,omitempty"`
+	Protocol       Protocol        `json:"proto,omitempty"`
 	Url            string          `json:"url,omitempty"`
 	Path           string          `json:"path,omitempty"`
 	Summary        string          `json:"summary,omitempty"`
-	StatusCode     int             `json:"statusCode"`
+	StatusCode     int             `json:"status"`
 	Method         string          `json:"method,omitempty"`
 	Timestamp      int64           `json:"timestamp,omitempty"`
 	Source         *TCP            `json:"src"`
@@ -182,11 +181,29 @@ type Contract struct {
 	Content        string         `json:"content"`
 }
 
-type DataUnmarshaler interface {
-	UnmarshalData(*MizuEntry) error
+func Summarize(entry *Entry) *BaseEntry {
+	return &BaseEntry{
+		Id:             entry.Id,
+		Protocol:       entry.Protocol,
+		Path:           entry.Path,
+		Summary:        entry.Summary,
+		StatusCode:     entry.Status,
+		Method:         entry.Method,
+		Timestamp:      entry.Timestamp,
+		Source:         entry.Source,
+		Destination:    entry.Destination,
+		IsOutgoing:     entry.IsOutgoing,
+		Latency:        entry.ElapsedTime,
+		Rules:          entry.Rules,
+		ContractStatus: entry.ContractStatus,
+	}
 }
 
-func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
+type DataUnmarshaler interface {
+	UnmarshalData(*Entry) error
+}
+
+func (bed *BaseEntry) UnmarshalData(entry *Entry) error {
 	bed.Protocol = entry.Protocol
 	bed.Id = entry.Id
 	bed.Path = entry.Path

tap/extensions/amqp/main.go

@@ -223,7 +223,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 
@@ -261,7 +261,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 	request["url"] = summary
 	reqDetails["method"] = request["method"]
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -286,25 +286,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    protocol,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
 	representation := make(map[string]interface{}, 0)

tap/extensions/http/handlers.go

@@ -21,9 +21,32 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
 		FilterSensitiveData(item, options)
 	}
 
+	replaceForwardedFor(item)
+
 	emitter.Emit(item)
 }
 
+func replaceForwardedFor(item *api.OutputChannelItem) {
+	if item.Protocol.Name != "http" {
+		return
+	}
+
+	request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
+
+	forwardedFor := request.Header.Get("X-Forwarded-For")
+	if forwardedFor == "" {
+		return
+	}
+
+	ips := strings.Split(forwardedFor, ",")
+	lastIP := strings.TrimSpace(ips[0])
+
+	item.ConnectionInfo.ClientIP = lastIP
+	// Erase the port field. Because the proxy terminates the connection from the client, the port that we see here
+	// is not the source port on the client side.
+	item.ConnectionInfo.ClientPort = ""
+}
+
 func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	streamID, messageHTTP1, isGrpc, err := http2Assembler.readMessage()
 	if err != nil {

tap/extensions/http/main.go

@@ -157,7 +157,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	return nil
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	var host, authority, path string
 
 	request := item.Pair.Request.Payload.(map[string]interface{})
@@ -241,7 +241,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	httpPair, _ := json.Marshal(item.Pair)
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: item.Protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -267,26 +267,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    entry.Protocol,
-		Path:        entry.Path,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func representRequest(request map[string]interface{}) (repRequest []interface{}) {
 	details, _ := json.Marshal([]api.TableData{
 		{

tap/extensions/kafka/main.go

@@ -62,7 +62,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 	apiKey := ApiKey(reqDetails["apiKey"].(float64))
@@ -146,7 +146,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	if elapsedTime < 0 {
 		elapsedTime = 0
 	}
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: _protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -171,25 +171,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    _protocol,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
 	representation := make(map[string]interface{}, 0)

tap/extensions/redis/main.go

@@ -59,7 +59,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	response := item.Pair.Response.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
@@ -80,7 +80,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	if elapsedTime < 0 {
 		elapsedTime = 0
 	}
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -106,25 +106,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    protocol,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
 	representation := make(map[string]interface{}, 0)

ui/src/components/AuthPageBase.tsx

@@ -7,7 +7,7 @@ import "./style/AuthBasePage.sass";
 export const AuthPageBase: React.FC = ({children}) => {
     return <div className="authContainer" style={{background: `url(${background})`, backgroundSize: "cover"}}>
             <div className="authHeader">
-                <img src={logo}/>
+                <img alt="logo" src={logo}/>
             </div>
             {children}
     </div>;

ui/src/components/EntriesList.tsx

@@ -1,4 +1,4 @@
-import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
+import React, {useCallback, useEffect, useMemo, useState} from "react";
 import styles from './style/EntriesList.module.sass';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
 import Moment from 'moment';
@@ -33,14 +33,14 @@ interface EntriesListProps {
     leftOffBottom: number;
     truncatedTimestamp: number;
     setTruncatedTimestamp: any;
+    scrollableRef: any;
 }
 
 const api = Api.getInstance();
 
-export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp}) => {
+export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp, scrollableRef}) => {
     const [loadMoreTop, setLoadMoreTop] = useState(false);
     const [isLoadingTop, setIsLoadingTop] = useState(false);
-    const scrollableRef = useRef(null);
 
     useEffect(() => {
         const list = document.getElementById('list').firstElementChild;
@@ -92,7 +92,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, qu
         if (scrollTo) {
             scrollableRef.current.scrollToIndex(data.data.length - 1);
         }
-    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent, setQueriedTotal, setTruncatedTimestamp]);
+    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent, setQueriedTotal, setTruncatedTimestamp, scrollableRef]);
 
     useEffect(() => {
         if(!isWebSocketConnectionClosed || !loadMoreTop || noMoreDataTop) return;

ui/src/components/EntryDetailed.tsx

@@ -69,9 +69,7 @@ const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updat
     </div>;
 };
 
-const EntrySummary: React.FC<any> = ({data, updateQuery}) => {
-    const entry = data.base;
-
+const EntrySummary: React.FC<any> = ({entry, updateQuery}) => {
     return <EntryItem
         key={`entry-${entry.id}`}
         entry={entry}
@@ -92,7 +90,7 @@ export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData, updateQu
             elapsedTime={entryData.data.elapsedTime}
             updateQuery={updateQuery}
         />
-        {entryData.data && <EntrySummary data={entryData.data} updateQuery={updateQuery}/>}
+        {entryData.data && <EntrySummary entry={entryData.data} updateQuery={updateQuery}/>}
         <>
             {entryData.data && <EntryViewer
                 representation={entryData.representation}

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -20,11 +20,11 @@ interface TCPInterface {
 }
 
 interface Entry {
-    protocol: ProtocolInterface,
+    proto: ProtocolInterface,
     method?: string,
     summary: string,
     id: number,
-    statusCode?: number;
+    status?: number;
     timestamp: Date;
     src: TCPInterface,
     dst: TCPInterface,
@@ -53,7 +53,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
 
     const isSelected = focusedEntryId === entry.id.toString();
 
-    const classification = getClassification(entry.statusCode)
+    const classification = getClassification(entry.status)
     const numberOfRules = entry.rules.numberOfRules
     let ingoingIcon;
     let outgoingIcon;
@@ -123,7 +123,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             break;
     }
 
-    const isStatusCodeEnabled = ((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0);
+    const isStatusCodeEnabled = ((entry.proto.name === "http" && "status" in entry) || entry.status !== 0);
     var endpointServiceContainer = "10px";
     if (!isStatusCodeEnabled) endpointServiceContainer = "20px";
 
@@ -137,7 +137,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                 setFocusedEntryId(entry.id.toString());
             }}
             style={{
-                border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid",
+                border: isSelected ? `1px ${entry.proto.backgroundColor} solid` : "1px transparent solid",
                 position: !headingMode ? "absolute" : "unset",
                 top: style['top'],
                 marginTop: !headingMode ? style['marginTop'] : "10px",
@@ -145,12 +145,12 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             }}
         >
             {!headingMode ? <Protocol
-                protocol={entry.protocol}
+                protocol={entry.proto}
                 horizontal={false}
                 updateQuery={updateQuery}
             /> : null}
             {isStatusCodeEnabled && <div>
-                <StatusCode statusCode={entry.statusCode} updateQuery={updateQuery}/>
+                <StatusCode statusCode={entry.status} updateQuery={updateQuery}/>
             </div>}
             <div className={styles.endpointServiceContainer} style={{paddingLeft: endpointServiceContainer}}>
                 <Summary method={entry.method} summary={entry.summary} updateQuery={updateQuery}/>
@@ -162,7 +162,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                         flipped={true}
                         style={{marginTop: "-4px", overflow: "visible"}}
                         iconStyle={!headingMode ? {marginTop: "4px", left: "68px", position: "absolute"} :
-                        entry.protocol.name === "http" ? {marginTop: "4px", left: "calc(50vw + 41px)", position: "absolute"} :
+                        entry.proto.name === "http" ? {marginTop: "4px", left: "calc(50vw + 41px)", position: "absolute"} :
                         {marginTop: "4px", left: "calc(50vw - 9px)", position: "absolute"}}
                     >
                         <span
@@ -171,7 +171,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                             {entry.src.name ? entry.src.name : "[Unresolved]"}
                         </span>
                     </Queryable>
-                    <SwapHorizIcon style={{color: entry.protocol.backgroundColor, marginTop: "-2px"}}></SwapHorizIcon>
+                    <SwapHorizIcon style={{color: entry.proto.backgroundColor, marginTop: "-2px"}}></SwapHorizIcon>
                     <Queryable
                         query={`dst.name == "${entry.dst.name}"`}
                         updateQuery={updateQuery}
@@ -216,7 +216,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                         {entry.src.ip}
                     </span>
                 </Queryable>
-                <span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>:</span>
+				<span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>{entry.src.port ? ":" : ""}</span>
                 <Queryable
                         query={`src.port == "${entry.src.port}"`}
                         updateQuery={updateQuery}

ui/src/components/Filters.tsx

@@ -265,7 +265,7 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
                             </Typography>
                             <br></br>
                             <Typography id="modal-modal-description">
-                                true if the given selector's value starts with the string:
+                                true if the given selector's value starts with (similarly <code style={{fontSize: "14px"}}>endsWith</code>, <code style={{fontSize: "14px"}}>contains</code>) the string:
                             </Typography>
                             <SyntaxHighlighter
                                 showLineNumbers={false}
@@ -273,19 +273,19 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
                                 language="python"
                             />
                             <Typography id="modal-modal-description">
-                                true if the given selector's value ends with the string:
+                                a field that contains a JSON encoded string can be filtered based a JSONPath:
                             </Typography>
                             <SyntaxHighlighter
                                 showLineNumbers={false}
-                                code={`request.path.endsWith("something")`}
+                                code={`response.content.text.json().some.path == "somevalue"`}
                                 language="python"
                             />
                             <Typography id="modal-modal-description">
-                                true if the given selector's value contains the string:
+                                fields that contain sensitive information can be redacted:
                             </Typography>
                             <SyntaxHighlighter
                                 showLineNumbers={false}
-                                code={`request.path.contains("something")`}
+                                code={`and redact("request.path", "src.name")`}
                                 language="python"
                             />
                             <Typography id="modal-modal-description">

ui/src/components/InstallPage.tsx

@@ -1,4 +1,4 @@
-import { Button, TextField } from "@material-ui/core";
+import { Button } from "@material-ui/core";
 import React, { useContext, useState } from "react";
 import { MizuContext, Page } from "../EntApp";
 import { adminUsername } from "../consts";

ui/src/components/LoginPage.tsx

@@ -1,4 +1,4 @@
-import { Button, TextField } from "@material-ui/core";
+import { Button } from "@material-ui/core";
 import React, { useContext, useState } from "react";
 import { toast } from "react-toastify";
 import { MizuContext, Page } from "../EntApp";

ui/src/components/TrafficPage.tsx

@@ -72,6 +72,8 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
 
     const [startTime, setStartTime] = useState(0);
 
+    const scrollableRef = useRef(null);
+
     const handleQueryChange = useMemo(() => debounce(async (query: string) => {
         if (!query) {
             setQueryBackgroundColor("#f5f5f5")
@@ -210,7 +212,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
         setSelectedEntryData(null);
         (async () => {
             try {
-                const entryData = await api.getEntry(focusedEntryId);
+                const entryData = await api.getEntry(focusedEntryId, query);
                 setSelectedEntryData(entryData);
             } catch (error) {
                 if (error.response?.data?.type) {
@@ -239,6 +241,8 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
             } else {
                 openWebSocket(`leftOff(-1)`, true);
             }
+            scrollableRef.current.jumpToBottom();
+            setIsSnappedToBottom(true);
         }
     }
 
@@ -318,6 +322,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
                             leftOffBottom={leftOffBottom}
                             truncatedTimestamp={truncatedTimestamp}
                             setTruncatedTimestamp={setTruncatedTimestamp}
+                            scrollableRef={scrollableRef}
                         />
                     </div>
                 </div>

ui/src/components/style/Filters.module.sass

@@ -32,9 +32,8 @@
   fieldset
     border: none
 
-$divider-breakpoint-1: 1474px
-$divider-breakpoint-2: 1366px
-$divider-breakpoint-3: 1980px
+$divider-breakpoint-1: 1055px
+$divider-breakpoint-2: 1453px
 
 @media (max-width: $divider-breakpoint-1)
   .divider1
@@ -43,7 +42,3 @@ $divider-breakpoint-3: 1980px
 @media (max-width: $divider-breakpoint-2)
   .divider2
     display: none
-
-@media (min-width: $divider-breakpoint-1) and (max-width: $divider-breakpoint-3)
-  .divider2
-    display: none

ui/src/helpers/api.js

@@ -38,8 +38,8 @@ export default class Api {
         return response.data;
     }
 
-    getEntry = async (id) => {
-        const response = await this.client.get(`/entries/${id}`);
+    getEntry = async (id, query) => {
+        const response = await this.client.get(`/entries/${id}?query=${query}`);
         return response.data;
     }