TRA-4017 Bring back getOldEntries method using fetch API and always start streaming from now (#518)

* Bring back `getOldEntries` method using fetch API

* Determine no more data on top based on `leftOff` value

* Remove `entriesBuffer` state

* Always open WebSocket with some `leftOff` value

* Rename `leftOff` state to `leftOffBottom`

* Don't set the `focusedEntryId` through WebSocket if the WebSocket is closed

* Call `setQueriedCurrent` with addition

* Close WebSocket upon reaching to top

* Open WebSocket upon snapping to bottom

* Close the WebSocket on snap broken event instead

* Set queried current value to zero upon filter submit

* Upgrade `react-scrollable-feed-virtualized` version and use `scrollToIndex` function

* Change the footer text format

* Improve no more data top logic

* Fix `closeWebSocket()` call logic in `onSnapBrokenEvent` and handle `data.meta` being `null` in `getOldEntries`

* Fix the issues around fetching old records

* Clean up `EntriesList.module.sass`

* Decrement initial `leftOffTop` value by `2`

* Fix the order of `incomingEntries` in `getOldEntries`

* Request `leftOffTop - 1` from `fetchEntries`

* Limit the front-end total entries fetched through WebSocket count to `10000`

* Lose the UI performance gain that's provided by #452

* Revert "Fix the selected entry behavior by propagating the `focusedEntryId` through WebSocket (before #452) TRA-3983 (#513)"

This reverts commit 873f252544.

* Fix the issues caused by 09371f141f

* Upgrade Basenine version from `0.2.13` to `0.2.14`

* Upgrade Basenine version from `0.2.14` to `0.2.15`

* Fix the condition of "Fetch old records" button visibility

* Upgrade Basenine version from `0.2.15` to `0.2.16` and fix the UI code related to fetching old records

* Make `newEntries` constant

Dockerfile

@@ -42,8 +42,8 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.11/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.11/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
 RUN shasum -a 256 -c basenine_linux_amd64.sha256
 RUN chmod +x ./basenine_linux_amd64
 

README.md

@@ -185,15 +185,3 @@ Mizu can be run detached from the cli using the daemon flag: `mizu tap --daemon`
 indefinitely in the cluster.
 
 For more information please refer to [DAEMON MODE](docs/DAEMON_MODE.md)
-
-## How to Run local UI
-
-- run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`
-
-- copy Har files into the folder from last command
-
-- change `MizuWebsocketURL` and `apiURL` in `api.js` file
-
-- run from mizu/ui - `npm run start`
-
-- open browser on `localhost:3000`

agent/go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/up9inc/basenine/client/go v0.0.0-20211121072216-04366911881c
+	github.com/up9inc/basenine/client/go v0.0.0-20211207165834-2ced7577f9e6
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0

agent/go.sum

@@ -450,8 +450,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20211121072216-04366911881c h1:GJsCVhDKjV/k3mNG255VN7hAQ7fxyNgX5T+VJyzoOQ0=
-github.com/up9inc/basenine/client/go v0.0.0-20211121072216-04366911881c/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20211207165834-2ced7577f9e6 h1:8JOkoaZHhUPi4r7vSL/xo83foSz8BHPSabTDpxmtHFU=
+github.com/up9inc/basenine/client/go v0.0.0-20211207165834-2ced7577f9e6/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=

agent/pkg/controllers/entries_controller.go

@@ -4,8 +4,10 @@ import (
 	"encoding/json"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/utils"
+	"mizuserver/pkg/validation"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/gin-gonic/gin"
 
@@ -36,6 +38,62 @@ func Error(c *gin.Context, err error) bool {
 	return false // no error, can continue
 }
 
+func GetEntries(c *gin.Context) {
+	entriesRequest := &models.EntriesRequest{}
+
+	if err := c.BindQuery(entriesRequest); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	validationError := validation.Validate(entriesRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
+	}
+
+	if entriesRequest.TimeoutMs == 0 {
+		entriesRequest.TimeoutMs = 3000
+	}
+
+	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
+		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, validationError)
+	}
+
+	response := &models.EntriesResponse{}
+	var dataSlice []interface{}
+
+	for _, row := range data {
+		var dataMap map[string]interface{}
+		err = json.Unmarshal(row, &dataMap)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"error":     true,
+				"type":      "error",
+				"autoClose": "5000",
+				"msg":       string(row),
+			})
+			return // exit
+		}
+
+		base := dataMap["base"].(map[string]interface{})
+		base["id"] = uint(dataMap["id"].(float64))
+
+		dataSlice = append(dataSlice, base)
+	}
+
+	var metadata *basenine.Metadata
+	err = json.Unmarshal(meta, &metadata)
+	if err != nil {
+		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+	}
+
+	response.Data = dataSlice
+	response.Meta = metadata
+
+	c.JSON(http.StatusOK, response)
+}
+
 func GetEntry(c *gin.Context) {
 	id, _ := strconv.Atoi(c.Param("id"))
 	var entry tapApi.MizuEntry

agent/pkg/models/models.go

@@ -16,6 +16,19 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
+type EntriesRequest struct {
+	LeftOff   int    `form:"leftOff" validate:"required,min=-1"`
+	Direction int    `form:"direction" validate:"required,oneof='1' '-1'"`
+	Query     string `form:"query"`
+	Limit     int    `form:"limit" validate:"required,min=1"`
+	TimeoutMs int    `form:"timeoutMs" validate:"min=1"`
+}
+
+type EntriesResponse struct {
+	Data []interface{}      `json:"data"`
+	Meta *basenine.Metadata `json:"meta"`
+}
+
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
 	Data map[string]interface{} `json:"data,omitempty"`

agent/pkg/routes/entries_routes.go

@@ -10,5 +10,6 @@ import (
 func EntriesRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/entries")
 
+	routeGroup.GET("/", controllers.GetEntries)  // get entries (base/thin entries) and metadata
 	routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry
 }

cli/apiserver/provider.go

@@ -3,11 +3,12 @@ package apiserver
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -61,7 +62,14 @@ func (provider *Provider) GetHealthStatus() (*shared.HealthResponse, error) {
 	if response, err := provider.client.Get(healthUrl); err != nil {
 		return nil, err
 	} else if response.StatusCode > 299 {
-		return nil, errors.New(fmt.Sprintf("status code: %d", response.StatusCode))
+		responseBody := new(strings.Builder)
+
+		if _, err := io.Copy(responseBody, response.Body); err != nil {
+			return nil, fmt.Errorf("status code: %d - (bad response - %v)", response.StatusCode, err)
+		} else {
+			singleLineResponse := strings.ReplaceAll(responseBody.String(), "\n", "")
+			return nil, fmt.Errorf("status code: %d - (response - %v)", response.StatusCode, singleLineResponse)
+		}
 	} else {
 		defer response.Body.Close()
 

cli/telemetry/telemetry.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
 
 	"github.com/denisbrodbeck/machineid"
 	"github.com/up9inc/mizu/cli/apiserver"
@@ -62,6 +63,9 @@ func ReportAPICalls(apiProvider *apiserver.Provider) {
 }
 
 func shouldRunTelemetry() bool {
+	if _, present := os.LookupEnv("MIZU_DISABLE_TELEMTRY"); present {
+		return false
+	}
 	if !config.Config.Telemetry {
 		return false
 	}

debug.Dockerfile

@@ -37,8 +37,8 @@ COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.11/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.11/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
 RUN shasum -a 256 -c basenine_linux_amd64.sha256
 RUN chmod +x ./basenine_linux_amd64
 

shared/kubernetes/provider.go

@@ -651,7 +651,17 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithName(tapperPodName)
 	agentContainer.WithImage(podImage)
 	agentContainer.WithImagePullPolicy(imagePullPolicy)
-	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
+
+	caps := applyconfcore.Capabilities().WithDrop("ALL").WithAdd("NET_RAW").WithAdd("NET_ADMIN")
+	
+	if istio {
+		caps = caps.WithAdd("SYS_ADMIN") // for reading /proc/PID/net/ns
+		caps = caps.WithAdd("SYS_PTRACE") // for setting netns to other process
+		caps = caps.WithAdd("DAC_OVERRIDE") // for reading /proc/PID/environ
+	}
+	
+	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithCapabilities(caps))
+
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.LogLevelEnvVar).WithValue(logLevel.String()),

tap/extensions/http/handlers.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strings"
 
 	"github.com/up9inc/mizu/tap/api"
 )
@@ -34,12 +35,13 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 	switch messageHTTP1 := messageHTTP1.(type) {
 	case http.Request:
 		ident := fmt.Sprintf(
-			"%s->%s %s->%s %d",
+			"%s->%s %s->%s %d %s",
 			tcpID.SrcIP,
 			tcpID.DstIP,
 			tcpID.SrcPort,
 			tcpID.DstPort,
 			streamID,
+			"HTTP2",
 		)
 		item = reqResMatcher.registerRequest(ident, &messageHTTP1, superTimer.CaptureTime)
 		if item != nil {
@@ -53,12 +55,13 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 		}
 	case http.Response:
 		ident := fmt.Sprintf(
-			"%s->%s %s->%s %d",
+			"%s->%s %s->%s %d %s",
 			tcpID.DstIP,
 			tcpID.SrcIP,
 			tcpID.DstPort,
 			tcpID.SrcPort,
 			streamID,
+			"HTTP2",
 		)
 		item = reqResMatcher.registerResponse(ident, &messageHTTP1, superTimer.CaptureTime)
 		if item != nil {
@@ -84,23 +87,30 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 	return nil
 }
 
-func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
-	req, err := http.ReadRequest(b)
+func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, req *http.Request, err error) {
+	req, err = http.ReadRequest(b)
 	if err != nil {
-		return err
+		return
 	}
 	counterPair.Request++
 
-	body, err := ioutil.ReadAll(req.Body)
+	// Check HTTP2 upgrade - HTTP2 Over Cleartext (H2C)
+	if strings.Contains(strings.ToLower(req.Header.Get("Connection")), "upgrade") && strings.ToLower(req.Header.Get("Upgrade")) == "h2c" {
+		switchingProtocolsHTTP2 = true
+	}
+
+	var body []byte
+	body, err = ioutil.ReadAll(req.Body)
 	req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
 
 	ident := fmt.Sprintf(
-		"%s->%s %s->%s %d",
+		"%s->%s %s->%s %d %s",
 		tcpID.SrcIP,
 		tcpID.DstIP,
 		tcpID.SrcPort,
 		tcpID.DstPort,
 		counterPair.Request,
+		"HTTP1",
 	)
 	item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime)
 	if item != nil {
@@ -113,26 +123,34 @@ func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 		}
 		filterAndEmit(item, emitter, options)
 	}
-	return nil
+	return
 }
 
-func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
-	res, err := http.ReadResponse(b, nil)
+func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, err error) {
+	var res *http.Response
+	res, err = http.ReadResponse(b, nil)
 	if err != nil {
-		return err
+		return
 	}
 	counterPair.Response++
 
-	body, err := ioutil.ReadAll(res.Body)
+	// Check HTTP2 upgrade - HTTP2 Over Cleartext (H2C)
+	if res.StatusCode == 101 && strings.Contains(strings.ToLower(res.Header.Get("Connection")), "upgrade") && strings.ToLower(res.Header.Get("Upgrade")) == "h2c" {
+		switchingProtocolsHTTP2 = true
+	}
+
+	var body []byte
+	body, err = ioutil.ReadAll(res.Body)
 	res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
 
 	ident := fmt.Sprintf(
-		"%s->%s %s->%s %d",
+		"%s->%s %s->%s %d %s",
 		tcpID.DstIP,
 		tcpID.SrcIP,
 		tcpID.DstPort,
 		tcpID.SrcPort,
 		counterPair.Response,
+		"HTTP1",
 	)
 	item := reqResMatcher.registerResponse(ident, res, superTimer.CaptureTime)
 	if item != nil {
@@ -145,5 +163,5 @@ func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 		}
 		filterAndEmit(item, emitter, options)
 	}
-	return nil
+	return
 }

tap/extensions/http/main.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"net/url"
 	"time"
 
@@ -85,7 +86,15 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 
 	dissected := false
+	switchingProtocolsHTTP2 := false
 	for {
+		if switchingProtocolsHTTP2 {
+			switchingProtocolsHTTP2 = false
+			isHTTP2, err = checkIsHTTP2Connection(b, isClient)
+			prepareHTTP2Connection(b, isClient)
+			http2Assembler = createHTTP2Assembler(b)
+		}
+
 		if superIdentifier.Protocol != nil && superIdentifier.Protocol != &protocol {
 			return errors.New("Identified by another protocol")
 		}
@@ -99,15 +108,39 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			}
 			dissected = true
 		} else if isClient {
-			err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
+			var req *http.Request
+			switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
 				continue
 			}
 			dissected = true
+
+			// In case of an HTTP2 upgrade, duplicate the HTTP1 request into HTTP2 with stream ID 1
+			if switchingProtocolsHTTP2 {
+				ident := fmt.Sprintf(
+					"%s->%s %s->%s 1 %s",
+					tcpID.SrcIP,
+					tcpID.DstIP,
+					tcpID.SrcPort,
+					tcpID.DstPort,
+					"HTTP2",
+				)
+				item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime)
+				if item != nil {
+					item.ConnectionInfo = &api.ConnectionInfo{
+						ClientIP:   tcpID.SrcIP,
+						ClientPort: tcpID.SrcPort,
+						ServerIP:   tcpID.DstIP,
+						ServerPort: tcpID.DstPort,
+						IsOutgoing: true,
+					}
+					filterAndEmit(item, emitter, options)
+				}
+			}
 		} else {
-			err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options)
+			switchingProtocolsHTTP2, err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
@@ -132,6 +165,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	reqDetails := request["details"].(map[string]interface{})
 	resDetails := response["details"].(map[string]interface{})
 
+	isRequestUpgradedH2C := false
+
 	for _, header := range reqDetails["headers"].([]interface{}) {
 		h := header.(map[string]interface{})
 		if h["name"] == "Host" {
@@ -143,13 +178,19 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		if h["name"] == ":path" {
 			path = h["value"].(string)
 		}
+
+		if h["name"] == "Upgrade" {
+			if h["value"].(string) == "h2c" {
+				isRequestUpgradedH2C = true
+			}
+		}
 	}
 
 	if resDetails["bodySize"].(float64) < 0 {
 		resDetails["bodySize"] = 0
 	}
 
-	if item.Protocol.Version == "2.0" {
+	if item.Protocol.Version == "2.0" && !isRequestUpgradedH2C {
 		service = authority
 	} else {
 		service = host
@@ -162,6 +203,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 
 	request["url"] = reqDetails["url"].(string)
+	reqDetails["targetUri"] = reqDetails["url"]
 	reqDetails["path"] = path
 	reqDetails["summary"] = path
 
@@ -191,7 +233,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		resDetails["statusText"] = grpcStatusCodes[statusCode]
 	}
 
-	if item.Protocol.Version == "2.0" {
+	if item.Protocol.Version == "2.0" && !isRequestUpgradedH2C {
 		reqDetails["url"] = path
 		request["url"] = path
 	}
@@ -269,9 +311,9 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 			Selector: `request.method`,
 		},
 		{
-			Name:     "URL",
-			Value:    request["url"].(string),
-			Selector: `request.url`,
+			Name:     "Target URI",
+			Value:    request["targetUri"].(string),
+			Selector: `request.targetUri`,
 		},
 		{
 			Name:     "Path",

tap/extensions/http/matcher.go

@@ -92,6 +92,6 @@ func splitIdent(ident string) []string {
 }
 
 func genKey(split []string) string {
-	key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
+	key := fmt.Sprintf("%s:%s->%s:%s,%s%s", split[0], split[2], split[1], split[3], split[4], split[5])
 	return key
 }

ui/package-lock.json

@@ -13644,9 +13644,9 @@
       }
     },
     "react-scrollable-feed-virtualized": {
-      "version": "1.4.8",
-      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.8.tgz",
-      "integrity": "sha512-zsSO/9QB+4V6HEk39lxeMEUA6JFSZjfV4stw7RF17+vZdlVhyATsTBCzsj8hZywY4F29cBfH+3/GKrMhwmhAsw=="
+      "version": "1.4.9",
+      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.9.tgz",
+      "integrity": "sha512-YkFkPjdIXDUsaCNYhZ+Blpp3LF+CsJWscwn/0fGSjF5QBKCtPURO9AEUA362Qnjr4S8LF2IjSAOCCFedIEnVNw=="
     },
     "react-syntax-highlighter": {
       "version": "15.4.3",

ui/package.json

@@ -23,7 +23,7 @@
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
-    "react-scrollable-feed-virtualized": "^1.4.8",
+    "react-scrollable-feed-virtualized": "^1.4.9",
     "react-syntax-highlighter": "^15.4.3",
     "react-toastify": "^8.0.3",
     "typescript": "^4.2.4",

ui/src/components/EntriesList.tsx

@@ -1,33 +1,132 @@
-import React, {useRef} from "react";
+import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
 import styles from './style/EntriesList.module.sass';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
+import {EntryItem} from "./EntryListItem/EntryListItem";
 import down from "./assets/downImg.svg";
+import spinner from './assets/spinner.svg';
+import Api from "../helpers/api";
 
 interface EntriesListProps {
     entries: any[];
+    setEntries: any;
+    query: string;
     listEntryREF: any;
     onSnapBrokenEvent: () => void;
     isSnappedToBottom: boolean;
     setIsSnappedToBottom: any;
     queriedCurrent: number;
+    setQueriedCurrent: any;
     queriedTotal: number;
     startTime: number;
+    noMoreDataTop: boolean;
+    setNoMoreDataTop: (flag: boolean) => void;
+    focusedEntryId: string;
+    setFocusedEntryId: (id: string) => void;
+    updateQuery: any;
+    leftOffTop: number;
+    setLeftOffTop: (leftOffTop: number) => void;
+    reconnectWebSocket: any;
+    isWebSocketConnectionClosed: boolean;
+    closeWebSocket: any;
 }
 
-export const EntriesList: React.FC<EntriesListProps> = ({entries, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, queriedTotal, startTime}) => {
+const api = new Api();
 
+export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, reconnectWebSocket, isWebSocketConnectionClosed, closeWebSocket}) => {
+    const [loadMoreTop, setLoadMoreTop] = useState(false);
+    const [isLoadingTop, setIsLoadingTop] = useState(false);
     const scrollableRef = useRef(null);
 
+    useEffect(() => {
+        const list = document.getElementById('list').firstElementChild;
+        list.addEventListener('scroll', (e) => {
+            const el: any = e.target;
+            if(el.scrollTop === 0) {
+                setLoadMoreTop(true);
+            } else {
+                setNoMoreDataTop(false);
+                setLoadMoreTop(false);
+            }
+        });
+    }, [setLoadMoreTop, setNoMoreDataTop]);
+
+    const memoizedEntries = useMemo(() => {
+        return entries;
+    },[entries]);
+
+    const getOldEntries = useCallback(async () => {
+        setLoadMoreTop(false);
+        if (leftOffTop === null || leftOffTop <= 0) {
+            return;
+        }
+        setIsLoadingTop(true);
+        const data = await api.fetchEntries(leftOffTop, -1, query, 100, 3000);
+        if (!data || !data.meta) {
+            setNoMoreDataTop(true);
+            setIsLoadingTop(false);
+            return;
+        }
+        setLeftOffTop(data.meta.leftOff);
+
+        let scrollTo: boolean;
+        if (data.meta.leftOff === 0) {
+            setNoMoreDataTop(true);
+            scrollTo = false;
+        } else {
+            scrollTo = true;
+        }
+        setIsLoadingTop(false);
+
+        const newEntries = [...data.data.reverse(), ...entries];
+        setEntries(newEntries);
+
+        setQueriedCurrent(queriedCurrent + data.meta.current);
+
+        if (scrollTo) {
+            scrollableRef.current.scrollToIndex(data.data.length - 1);
+        }
+    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent]);
+
+    useEffect(() => {
+        if(!isWebSocketConnectionClosed || !loadMoreTop || noMoreDataTop) return;
+        getOldEntries();
+    }, [loadMoreTop, noMoreDataTop, getOldEntries, isWebSocketConnectionClosed]);
+
+    const scrollbarVisible = scrollableRef.current?.childWrapperRef.current.clientHeight > scrollableRef.current?.wrapperRef.current.clientHeight;
+
     return <>
             <div className={styles.list}>
                 <div id="list" ref={listEntryREF} className={styles.list}>
+                    {isLoadingTop && <div className={styles.spinnerContainer}>
+                        <img alt="spinner" src={spinner} style={{height: 25}}/>
+                    </div>}
+                    {noMoreDataTop && <div id="noMoreDataTop" className={styles.noMoreDataAvailable}>No more data available</div>}
                     <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onSnapBroken={onSnapBrokenEvent}>
-                        {false /* TODO: why there is a need for something here (not necessarily false)? */}
-                        {entries}
+                        {false /* It's because the first child is ignored by ScrollableFeedVirtualized */}
+                        {memoizedEntries.map(entry => <EntryItem
+                            key={`entry-${entry.id}`}
+                            entry={entry}
+                            focusedEntryId={focusedEntryId}
+                            setFocusedEntryId={setFocusedEntryId}
+                            style={{}}
+                            updateQuery={updateQuery}
+                            headingMode={false}
+                        />)}
                     </ScrollableFeedVirtualized>
                     <button type="button"
-                        className={`${styles.btnLive} ${isSnappedToBottom ? styles.hideButton : styles.showButton}`}
+                        title="Fetch old records"
+                        className={`${styles.btnOld} ${!scrollbarVisible && leftOffTop > 0 ? styles.showButton : styles.hideButton}`}
                         onClick={(_) => {
+                            closeWebSocket();
+                            getOldEntries();
+                        }}>
+                        <img alt="down" src={down} />
+                    </button>
+                    <button type="button"
+                        title="Snap to bottom"
+                        className={`${styles.btnLive} ${isSnappedToBottom && !isWebSocketConnectionClosed ? styles.hideButton : styles.showButton}`}
+                        onClick={(_) => {
+                            reconnectWebSocket();
                             scrollableRef.current.jumpToBottom();
                             setIsSnappedToBottom(true);
                         }}>
@@ -36,7 +135,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, listEntryREF,
                 </div>
 
                 <div className={styles.footer}>
-                    <div>Displaying <b>{entries?.length}</b> results (queried <b>{queriedCurrent}</b>/<b>{queriedTotal}</b>)</div>
+                    <div>Displaying <b>{entries?.length}</b> results out of <b>{queriedTotal}</b> total</div>
                     {startTime !== 0 && <div>Started listening at <span style={{marginRight: 5, fontWeight: 600, fontSize: 13}}>{new Date(startTime).toLocaleString()}</span></div>}
                 </div>
             </div>

ui/src/components/EntryDetailed.tsx

@@ -1,9 +1,9 @@
 import React from "react";
 import EntryViewer from "./EntryDetailed/EntryViewer";
+import {EntryItem} from "./EntryListItem/EntryListItem";
 import {makeStyles} from "@material-ui/core";
 import Protocol from "./UI/Protocol"
-import StatusCode from "./UI/StatusCode";
-import {Summary} from "./UI/Summary";
+import Queryable from "./UI/Queryable";
 
 const useStyles = makeStyles(() => ({
     entryTitle: {
@@ -12,6 +12,7 @@ const useStyles = makeStyles(() => ({
         maxHeight: 46,
         alignItems: 'center',
         marginBottom: 4,
+        marginLeft: 6,
         padding: 2,
         paddingBottom: 0
     },
@@ -37,45 +38,49 @@ const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updat
     const classes = useStyles();
     const response = data.response;
 
-
     return <div className={classes.entryTitle}>
-        <Protocol protocol={protocol} horizontal={true} updateQuery={null}/>
+        <Protocol protocol={protocol} horizontal={true} updateQuery={updateQuery}/>
         <div style={{right: "30px", position: "absolute", display: "flex"}}>
-            {response && <div
-                className="queryable"
-                style={{margin: "0 18px", opacity: 0.5}}
-                onClick={() => {
-                    updateQuery(`response.bodySize == ${bodySize}`)
-                }}
+            {response && <Queryable
+                query={`response.bodySize == ${bodySize}`}
+                updateQuery={updateQuery}
+                style={{margin: "0 18px"}}
+                displayIconOnMouseOver={true}
             >
-                {formatSize(bodySize)}
-            </div>}
-            {response && <div
-                className="queryable"
-                style={{marginRight: 18, opacity: 0.5}}
-                onClick={() => {
-                    updateQuery(`elapsedTime >= ${elapsedTime}`)
-                }}
+                <div
+                    style={{opacity: 0.5}}
+                >
+                    {formatSize(bodySize)}
+                </div>
+            </Queryable>}
+            {response && <Queryable
+                query={`elapsedTime >= ${elapsedTime}`}
+                updateQuery={updateQuery}
+                style={{marginRight: 18}}
+                displayIconOnMouseOver={true}
             >
-                {Math.round(elapsedTime)}ms
-            </div>}
+                <div
+                    style={{opacity: 0.5}}
+                >
+                    {Math.round(elapsedTime)}ms
+                </div>
+            </Queryable>}
         </div>
     </div>;
 };
 
 const EntrySummary: React.FC<any> = ({data, updateQuery}) => {
-    const classes = useStyles();
+    const entry = data.base;
 
-    const response = data.response;
-
-    return <div className={classes.entrySummary}>
-        {response && "status" in response && <div style={{marginRight: 8}}>
-            <StatusCode statusCode={response.status} updateQuery={updateQuery}/>
-        </div>}
-        <div style={{flexGrow: 1, overflow: 'hidden'}}>
-            <Summary method={data.method} summary={data.summary} updateQuery={updateQuery}/>
-        </div>
-    </div>;
+    return <EntryItem
+        key={`entry-${entry.id}`}
+        entry={entry}
+        focusedEntryId={null}
+        setFocusedEntryId={null}
+        style={{}}
+        updateQuery={updateQuery}
+        headingMode={true}
+    />;
 };
 
 export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData, updateQuery}) => {

ui/src/components/EntryDetailed/EntrySections.module.sass

@@ -27,7 +27,7 @@
     font-weight: 600
     font-size: .75rem
     line-height: 1.2
-    margin: .3rem 0
+    margin-bottom: -2px
 
     .dataKey
         color: $blue-gray

ui/src/components/EntryDetailed/EntrySections.tsx

@@ -3,6 +3,7 @@ import React, {useState} from "react";
 import {SyntaxHighlighter} from "../UI/SyntaxHighlighter/index";
 import CollapsibleContainer from "../UI/CollapsibleContainer";
 import FancyTextDisplay from "../UI/FancyTextDisplay";
+import Queryable from "../UI/Queryable";
 import Checkbox from "../UI/Checkbox";
 import ProtobufDecoder from "protobuf-decoder";
 
@@ -15,23 +16,29 @@ interface EntryViewLineProps {
 }
 
 const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value, updateQuery, selector, overrideQueryValue}) => {
+    let query: string;
+    if (!selector) {
+        query = "";
+    } else if (overrideQueryValue) {
+        query = `${selector} == ${overrideQueryValue}`;
+    } else if (typeof(value) == "string") {
+        query = `${selector} == "${JSON.stringify(value).slice(1, -1)}"`;
+    } else {
+        query = `${selector} == ${value}`;
+    }
     return (label && <tr className={styles.dataLine}>
-                <td
-                    className={`queryable ${styles.dataKey}`}
-                    onClick={() => {
-                        if (!selector) {
-                            return
-                        } else if (overrideQueryValue) {
-                            updateQuery(`${selector} == ${overrideQueryValue}`)
-                        } else if (typeof(value) === "string") {
-                            updateQuery(`${selector} == "${JSON.stringify(value).slice(1, -1)}"`)
-                        } else {
-                            updateQuery(`${selector} == ${value}`)
-                        }
-                    }}
-                >
-                    {label}
-                </td>
+                    <td className={`${styles.dataKey}`}>
+                        <Queryable
+                            query={query}
+                            updateQuery={updateQuery}
+                            style={{float: "right", height: "18px"}}
+                            iconStyle={{marginRight: "20px"}}
+                            flipped={true}
+                            displayIconOnMouseOver={true}
+                        >
+                            {label}
+                        </Queryable>
+                    </td>
                 <td>
                     <FancyTextDisplay
                         className={styles.dataValue}
@@ -53,9 +60,9 @@ interface EntrySectionCollapsibleTitleProps {
 
 const EntrySectionCollapsibleTitle: React.FC<EntrySectionCollapsibleTitleProps> = ({title, color, isExpanded}) => {
     return <div className={styles.title}>
-        <span className={`${styles.button} ${isExpanded ? styles.expanded : ''}`} style={{backgroundColor: color}}>
+        <div className={`${styles.button} ${isExpanded ? styles.expanded : ''}`} style={{backgroundColor: color}}>
             {isExpanded ? '-' : '+'}
-        </span>
+        </div>
         <span>{title}</span>
     </div>
 }
@@ -130,7 +137,7 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
             <table>
                 <tbody>
                     <EntryViewLine label={'Mime type'} value={contentType} updateQuery={updateQuery} selector={selector} overrideQueryValue={`r".*"`}/>
-                    <EntryViewLine label={'Encoding'} value={encoding} updateQuery={updateQuery} selector={selector} overrideQueryValue={`r".*"`}/>
+                    {encoding && <EntryViewLine label={'Encoding'} value={encoding} updateQuery={updateQuery} selector={selector} overrideQueryValue={`r".*"`}/>}
                 </tbody>
             </table>
 

ui/src/components/EntryDetailed/EntryViewer.module.sass

@@ -4,6 +4,7 @@
     font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif
     height: calc(100% - 70px)
     width: 100%
+    margin-top: 10px
 
     h3,
     h4

ui/src/components/EntryListItem/EntryListItem.module.sass

@@ -19,7 +19,6 @@
 
 .rowSelected
   border: 1px $blue-color solid
-  margin-right: 3px
 
 .ruleSuccessRow
   background: #E8FFF1
@@ -52,7 +51,6 @@
   white-space: nowrap
   color: $secondary-font-color
   padding-left: 4px
-  padding-top: 3px
   padding-right: 10px
   display: flex
   font-size: 12px

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -1,8 +1,9 @@
-import React, {useState} from "react";
+import React from "react";
 import styles from './EntryListItem.module.sass';
 import StatusCode, {getClassification, StatusCodeClassification} from "../UI/StatusCode";
 import Protocol, {ProtocolInterface} from "../UI/Protocol"
 import {Summary} from "../UI/Summary";
+import Queryable from "../UI/Queryable";
 import ingoingIconSuccess from "../assets/ingoing-traffic-success.svg"
 import ingoingIconFailure from "../assets/ingoing-traffic-failure.svg"
 import ingoingIconNeutral from "../assets/ingoing-traffic-neutral.svg"
@@ -37,14 +38,16 @@ interface Rules {
 
 interface EntryProps {
     entry: Entry;
+    focusedEntryId: string;
     setFocusedEntryId: (id: string) => void;
     style: object;
     updateQuery: any;
+    headingMode: boolean;
 }
 
-export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, style, updateQuery}) => {
+export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocusedEntryId, style, updateQuery, headingMode}) => {
 
-    const [isSelected, setIsSelected] = useState(false);
+    const isSelected = focusedEntryId === entry.id.toString();
 
     const classification = getClassification(entry.statusCode)
     const numberOfRules = entry.rules.numberOfRules
@@ -118,41 +121,44 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, style
 
     return <>
         <div
-            id={entry.id.toString()}
+            id={`entry-${entry.id.toString()}`}
             className={`${styles.row}
             ${isSelected && !rule && !contractEnabled ? styles.rowSelected : additionalRulesProperties}`}
             onClick={() => {
-                setIsSelected(!isSelected);
+                if (!setFocusedEntryId) return;
                 setFocusedEntryId(entry.id.toString());
             }}
             style={{
                 border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid",
-                position: "absolute",
+                position: !headingMode ? "absolute" : "unset",
                 top: style['top'],
-                marginTop: style['marginTop'],
-                width: "calc(100% - 25px)",
+                marginTop: !headingMode ? style['marginTop'] : "10px",
+                width: !headingMode ? "calc(100% - 25px)" : "calc(100% - 18px)",
             }}
         >
-            <Protocol
+            {!headingMode ? <Protocol
                 protocol={entry.protocol}
                 horizontal={false}
                 updateQuery={updateQuery}
-            />
+            /> : null}
             {((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0) && <div>
                 <StatusCode statusCode={entry.statusCode} updateQuery={updateQuery}/>
             </div>}
             <div className={styles.endpointServiceContainer}>
                 <Summary method={entry.method} summary={entry.summary} updateQuery={updateQuery}/>
                 <div className={styles.service}>
-                    <span
-                        title="Service Name"
-                        className="queryable"
-                        onClick={() => {
-                            updateQuery(`service == "${entry.service}"`)
-                        }}
+                    <Queryable
+                        query={`service == "${entry.service}"`}
+                        updateQuery={updateQuery}
+                        displayIconOnMouseOver={true}
+                        style={{marginTop: "-4px"}}
                     >
-                        {entry.service}
-                    </span>
+                        <span
+                            title="Service Name"
+                        >
+                            {entry.service}
+                        </span>
+                    </Queryable>
                 </div>
             </div>
             {
@@ -170,74 +176,109 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, style
                 : ""
             }
             <div className={styles.separatorRight}>
-                <span
-                    className={`queryable ${styles.tcpInfo} ${styles.ip}`}
-                    title="Source IP"
-                    onClick={() => {
-                        updateQuery(`src.ip == "${entry.sourceIp}"`)
-                    }}
+                <Queryable
+                        query={`src.ip == "${entry.sourceIp}"`}
+                        updateQuery={updateQuery}
+                        displayIconOnMouseOver={true}
+                        flipped={true}
+                        iconStyle={{marginRight: "16px"}}
                 >
-                    {entry.sourceIp}
-                </span>
-                <span className={`${styles.tcpInfo}`}>:</span>
-                <span
-                    className={`queryable ${styles.tcpInfo} ${styles.port}`}
-                    title="Source Port"
-                    onClick={() => {
-                        updateQuery(`src.port == "${entry.sourcePort}"`)
-                    }}
+                    <span
+                        className={`${styles.tcpInfo} ${styles.ip}`}
+                        title="Source IP"
+                    >
+                        {entry.sourceIp}
+                    </span>
+                </Queryable>
+                <span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>:</span>
+                <Queryable
+                        query={`src.port == "${entry.sourcePort}"`}
+                        updateQuery={updateQuery}
+                        displayIconOnMouseOver={true}
+                        flipped={true}
+                        iconStyle={{marginTop: "28px"}}
                 >
-                    {entry.sourcePort}
-                </span>
+                    <span
+                        className={`${styles.tcpInfo} ${styles.port}`}
+                        title="Source Port"
+                    >
+                        {entry.sourcePort}
+                    </span>
+                </Queryable>
                 {entry.isOutgoing ?
-                    <img
-                        src={outgoingIcon}
-                        alt="Ingoing traffic"
-                        title="Ingoing"
-                        onClick={() => {
-                            updateQuery(`outgoing == true`)
-                        }}
-                    />
+                    <Queryable
+                            query={`outgoing == true`}
+                            updateQuery={updateQuery}
+                            displayIconOnMouseOver={true}
+                            flipped={true}
+                            iconStyle={{marginTop: "28px"}}
+                    >
+                        <img
+                            src={outgoingIcon}
+                            alt="Ingoing traffic"
+                            title="Ingoing"
+                        />
+                    </Queryable>
                     :
-                    <img
-                        src={ingoingIcon}
-                        alt="Outgoing traffic"
-                        title="Outgoing"
-                        onClick={() => {
-                            updateQuery(`outgoing == false`)
-                        }}
-                    />
+                    <Queryable
+                            query={`outgoing == true`}
+                            updateQuery={updateQuery}
+                            displayIconOnMouseOver={true}
+                            flipped={true}
+                            iconStyle={{marginTop: "28px"}}
+                    >
+                        <img
+                            src={ingoingIcon}
+                            alt="Outgoing traffic"
+                            title="Outgoing"
+                            onClick={() => {
+                                updateQuery(`outgoing == false`)
+                            }}
+                        />
+                    </Queryable>
                 }
-                <span
-                    className={`queryable ${styles.tcpInfo} ${styles.ip}`}
-                    title="Destination IP"
-                    onClick={() => {
-                        updateQuery(`dst.ip == "${entry.destinationIp}"`)
-                    }}
+                <Queryable
+                        query={`dst.ip == "${entry.destinationIp}"`}
+                        updateQuery={updateQuery}
+                        displayIconOnMouseOver={true}
+                        flipped={false}
+                        iconStyle={{marginTop: "28px"}}
                 >
-                    {entry.destinationIp}
-                </span>
-                <span className={`${styles.tcpInfo}`}>:</span>
-                <span
-                    className={`queryable ${styles.tcpInfo} ${styles.port}`}
-                    title="Destination Port"
-                    onClick={() => {
-                        updateQuery(`dst.port == "${entry.destinationPort}"`)
-                    }}
+                    <span
+                        className={`${styles.tcpInfo} ${styles.ip}`}
+                        title="Destination IP"
+                    >
+                        {entry.destinationIp}
+                    </span>
+                </Queryable>
+                <span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>:</span>
+                <Queryable
+                        query={`dst.port == "${entry.destinationPort}"`}
+                        updateQuery={updateQuery}
+                        displayIconOnMouseOver={true}
+                        flipped={false}
                 >
-                    {entry.destinationPort}
-                </span>
+                    <span
+                        className={`${styles.tcpInfo} ${styles.port}`}
+                        title="Destination Port"
+                    >
+                        {entry.destinationPort}
+                    </span>
+                </Queryable>
             </div>
             <div className={styles.timestamp}>
-                <span
-                    title="Timestamp"
-                    className="queryable"
-                    onClick={() => {
-                        updateQuery(`timestamp >= datetime("${new Date(+entry.timestamp)?.toLocaleString("en-US", {timeZone: 'UTC' })}")`)
-                    }}
+                <Queryable
+                        query={`timestamp >= datetime("${new Date(+entry.timestamp)?.toLocaleString("en-US", {timeZone: 'UTC' })}")`}
+                        updateQuery={updateQuery}
+                        displayIconOnMouseOver={true}
+                        flipped={false}
                 >
-                    {new Date(+entry.timestamp)?.toLocaleString("en-US")}
-                </span>
+                    <span
+                        title="Timestamp"
+                    >
+                        {new Date(+entry.timestamp)?.toLocaleString("en-US")}
+                    </span>
+                </Queryable>
             </div>
         </div>
     </>

ui/src/components/Filters.tsx

@@ -13,7 +13,7 @@ interface FiltersProps {
     setQuery: any
     backgroundColor: string
     ws: any
-    openWebSocket: (query: string) => void;
+    openWebSocket: (query: string, resetEntries: boolean) => void;
 }
 
 export const Filters: React.FC<FiltersProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
@@ -33,7 +33,7 @@ interface QueryFormProps {
     setQuery: any
     backgroundColor: string
     ws: any
-    openWebSocket: (query: string) => void;
+    openWebSocket: (query: string, resetEntries: boolean) => void;
 }
 
 const style = {
@@ -63,8 +63,12 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
     }
 
     const handleSubmit = (e) => {
-        ws.close()
-        openWebSocket(query)
+        ws.close();
+        if (query) {
+            openWebSocket(`(${query}) and leftOff(-1)`, true);
+        } else {
+            openWebSocket(`leftOff(-1)`, true);
+        }
         e.preventDefault();
     }
 
@@ -226,7 +230,7 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
                                 language="python"
                             />
                             <Typography id="modal-modal-description">
-                                By clicking the UI elements in both left-pane and right-pane, you can automatically select a field and update the query:
+                                By clicking the plus icon that appears beside the queryable UI elements on hovering in both left-pane and right-pane, you can automatically select a field and update the query:
                             </Typography>
                             <img
                                 src={filterUIExample1}
@@ -235,12 +239,12 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
                                 title="Clicking to UI elements (left-pane)"
                             />
                             <Typography id="modal-modal-description">
-                                Such that; clicking this in left-pane, would append the query below:
+                                Such that; clicking this icon in left-pane, would append the query below:
                             </Typography>
                             <SyntaxHighlighter
                                 isWrapped={false}
                                 showLineNumbers={false}
-                                code={`and service == "http://carts.sock-shop"`}
+                                code={`and service == "carts.sock-shop"`}
                                 language="python"
                             />
                             <Typography id="modal-modal-description">

ui/src/components/TrafficPage.tsx

@@ -1,7 +1,6 @@
 import React, {useEffect, useRef, useState} from "react";
 import {Filters} from "./Filters";
 import {EntriesList} from "./EntriesList";
-import {EntryItem} from "./EntryListItem/EntryListItem";
 import {makeStyles} from "@material-ui/core";
 import "./style/TrafficPage.sass";
 import styles from './style/EntriesList.module.sass';
@@ -21,7 +20,7 @@ const useLayoutStyles = makeStyles(() => ({
         padding: "12px 24px",
         borderRadius: 4,
         marginTop: 15,
-        background: variables.headerBackgoundColor,
+        background: variables.headerBackgroundColor,
     },
 
     viewer: {
@@ -51,11 +50,12 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     const classes = useLayoutStyles();
 
     const [entries, setEntries] = useState([] as any);
-    const [entriesBuffer, setEntriesBuffer] = useState([] as any);
     const [focusedEntryId, setFocusedEntryId] = useState(null);
     const [selectedEntryData, setSelectedEntryData] = useState(null);
     const [connection, setConnection] = useState(ConnectionStatus.Closed);
 
+    const [noMoreDataTop, setNoMoreDataTop] = useState(false);
+
     const [tappingStatus, setTappingStatus] = useState(null);
 
     const [isSnappedToBottom, setIsSnappedToBottom] = useState(true);
@@ -66,6 +66,8 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     const [queriedCurrent, setQueriedCurrent] = useState(0);
     const [queriedTotal, setQueriedTotal] = useState(0);
+    const [leftOffBottom, setLeftOffBottom] = useState(0);
+    const [leftOffTop, setLeftOffTop] = useState(null);
 
     const [startTime, setStartTime] = useState(0);
 
@@ -100,16 +102,30 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     const listEntry = useRef(null);
 
-    const openWebSocket = (query) => {
-        setFocusedEntryId(null);
-        setEntries([]);
-        setEntriesBuffer([]);
+    const openWebSocket = (query: string, resetEntries: boolean) => {
+        if (resetEntries) {
+            setFocusedEntryId(null);
+            setEntries([]);
+            setQueriedCurrent(0);
+            setLeftOffTop(null);
+            setNoMoreDataTop(false);
+        }
         ws.current = new WebSocket(MizuWebsocketURL);
         ws.current.onopen = () => {
-            ws.current.send(query)
             setConnection(ConnectionStatus.Connected);
+            ws.current.send(query);
+        }
+        ws.current.onclose = () => {
+            setConnection(ConnectionStatus.Closed);
+        }
+        ws.current.onerror = (event) => {
+            console.error("WebSocket error:", event);
+            if (query) {
+                openWebSocket(`(${query}) and leftOff(${leftOffBottom})`, false);
+            } else {
+                openWebSocket(`leftOff(${leftOffBottom})`, false);
+            }
         }
-        ws.current.onclose = () => setConnection(ConnectionStatus.Closed);
     }
 
     if (ws.current) {
@@ -119,17 +135,14 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
             switch (message.messageType) {
                 case "entry":
                     const entry = message.data;
-                    if (!focusedEntryId) setFocusedEntryId(entry.id.toString());
-                    setEntriesBuffer([
-                        ...entriesBuffer,
-                        <EntryItem
-                            key={entry.id}
-                            entry={entry}
-                            setFocusedEntryId={setFocusedEntryId}
-                            style={{}}
-                            updateQuery={updateQuery}
-                        />
-                    ]);
+                    if (!focusedEntryId) setFocusedEntryId(entry.id.toString())
+                    const newEntries = [...entries, entry];
+                    if (newEntries.length === 10001) {
+                        setLeftOffTop(newEntries[0].entry.id);
+                        newEntries.shift();
+                        setNoMoreDataTop(false);
+                    }
+                    setEntries(newEntries);
                     break
                 case "status":
                     setTappingStatus(message.tappingStatus);
@@ -153,9 +166,12 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     });
                     break;
                 case "queryMetadata":
-                    setQueriedCurrent(message.data.current);
+                    setQueriedCurrent(queriedCurrent + message.data.current);
                     setQueriedTotal(message.data.total);
-                    setEntries(entriesBuffer);
+                    setLeftOffBottom(message.data.leftOff);
+                    if (leftOffTop === null) {
+                        setLeftOffTop(message.data.leftOff - 1);
+                    }
                     break;
                 case "startTime":
                     setStartTime(message.data);
@@ -168,7 +184,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     useEffect(() => {
         (async () => {
-            openWebSocket("rlimit(100)");
+            openWebSocket("leftOff(-1)", true);
             try{
                 const tapStatusResponse = await api.tapStatus();
                 setTappingStatus(tapStatusResponse);
@@ -190,27 +206,41 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                 const entryData = await api.getEntry(focusedEntryId);
                 setSelectedEntryData(entryData);
             } catch (error) {
-                toast[error.response.data.type](`Entry[${focusedEntryId}]: ${error.response.data.msg}`, {
-                    position: "bottom-right",
-                    theme: "colored",
-                    autoClose: error.response.data.autoClose,
-                    hideProgressBar: false,
-                    closeOnClick: true,
-                    pauseOnHover: true,
-                    draggable: true,
-                    progress: undefined,
-                });
+                if (error.response) {
+                    toast[error.response.data.type](`Entry[${focusedEntryId}]: ${error.response.data.msg}`, {
+                        position: "bottom-right",
+                        theme: "colored",
+                        autoClose: error.response.data.autoClose,
+                        hideProgressBar: false,
+                        closeOnClick: true,
+                        pauseOnHover: true,
+                        draggable: true,
+                        progress: undefined,
+                    });
+                }
                 console.error(error);
             }
-        })()
-    }, [focusedEntryId])
+        })();
+        // eslint-disable-next-line
+    }, [focusedEntryId]);
+
+    const closeWebSocket = () => {
+        ws.current.close();
+    }
+
+    const reconnectWebSocket = () => {
+        if (query) {
+            openWebSocket(`(${query}) and leftOff(${leftOffBottom})`, false);
+        } else {
+            openWebSocket(`leftOff(${leftOffBottom})`, false);
+        }
+    }
 
     const toggleConnection = () => {
         if (connection === ConnectionStatus.Connected) {
-            ws.current.close();
+            closeWebSocket();
         } else {
-            openWebSocket(query);
-            setConnection(ConnectionStatus.Connected);
+            reconnectWebSocket();
         }
     }
 
@@ -234,7 +264,10 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     }
 
     const onSnapBrokenEvent = () => {
-        setIsSnappedToBottom(false)
+        setIsSnappedToBottom(false);
+        if (connection === ConnectionStatus.Connected) {
+            closeWebSocket();
+        }
     }
 
     return (
@@ -263,13 +296,26 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     <div className={styles.container}>
                         <EntriesList
                             entries={entries}
+                            setEntries={setEntries}
+                            query={query}
                             listEntryREF={listEntry}
                             onSnapBrokenEvent={onSnapBrokenEvent}
                             isSnappedToBottom={isSnappedToBottom}
                             setIsSnappedToBottom={setIsSnappedToBottom}
                             queriedCurrent={queriedCurrent}
+                            setQueriedCurrent={setQueriedCurrent}
                             queriedTotal={queriedTotal}
                             startTime={startTime}
+                            noMoreDataTop={noMoreDataTop}
+                            setNoMoreDataTop={setNoMoreDataTop}
+                            focusedEntryId={focusedEntryId}
+                            setFocusedEntryId={setFocusedEntryId}
+                            updateQuery={updateQuery}
+                            leftOffTop={leftOffTop}
+                            setLeftOffTop={setLeftOffTop}
+                            reconnectWebSocket={reconnectWebSocket}
+                            isWebSocketConnectionClosed={connection === ConnectionStatus.Closed}
+                            closeWebSocket={closeWebSocket}
                         />
                     </div>
                 </div>

ui/src/components/UI/Protocol.tsx

@@ -1,5 +1,6 @@
 import React from "react";
 import styles from './style/Protocol.module.sass';
+import Queryable from "./Queryable";
 
 export interface ProtocolInterface {
     name: string
@@ -22,34 +23,45 @@ interface ProtocolProps {
 
 const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal, updateQuery}) => {
     if (horizontal) {
-        return <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
+        return <Queryable
+            query={protocol.macro}
+            updateQuery={updateQuery}
+            displayIconOnMouseOver={true}
+        >
+            <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
+                <span
+                    className={`${styles.base} ${styles.horizontal}`}
+                    style={{
+                        backgroundColor: protocol.backgroundColor,
+                        color: protocol.foregroundColor,
+                        fontSize: 13,
+                    }}
+                    title={protocol.abbr}
+                >
+                    {protocol.longName}
+                </span>
+            </a>
+        </Queryable>
+    } else {
+        return <Queryable
+            query={protocol.macro}
+            updateQuery={updateQuery}
+            displayIconOnMouseOver={true}
+            flipped={false}
+            iconStyle={{marginTop: "48px"}}
+        >
             <span
-                className={`${styles.base} ${styles.horizontal}`}
+                className={`${styles.base} ${styles.vertical}`}
                 style={{
                     backgroundColor: protocol.backgroundColor,
                     color: protocol.foregroundColor,
-                    fontSize: 13,
+                    fontSize: protocol.fontSize,
                 }}
-                title={protocol.abbr}
+                title={protocol.longName}
             >
-                {protocol.longName}
+                {protocol.abbr}
             </span>
-        </a>
-    } else {
-        return <span
-            className={`${styles.base} ${styles.vertical}`}
-            style={{
-                backgroundColor: protocol.backgroundColor,
-                color: protocol.foregroundColor,
-                fontSize: protocol.fontSize,
-            }}
-            title={protocol.longName}
-            onClick={() => {
-                updateQuery(protocol.macro)
-            }}
-        >
-            {protocol.abbr}
-        </span>
+        </Queryable>
     }
 };
 

ui/src/components/UI/Queryable.tsx

@@ -0,0 +1,62 @@
+import React, { useEffect, useState } from 'react';
+import { CopyToClipboard } from 'react-copy-to-clipboard';
+import AddCircleIcon from '@material-ui/icons/AddCircle';
+import './style/Queryable.sass';
+
+interface Props {
+    query: string,
+    updateQuery: any,
+    style?: object,
+    iconStyle?: object,
+    className?: string,
+    useTooltip?: boolean,
+    displayIconOnMouseOver?: boolean,
+    flipped?: boolean,
+}
+
+const Queryable: React.FC<Props> = ({query, updateQuery, style, iconStyle, className, useTooltip= true, displayIconOnMouseOver = false, flipped = false, children}) => {
+    const [showAddedNotification, setAdded] = useState(false);
+    const [showTooltip, setShowTooltip] = useState(false);
+
+    const onCopy = () => {
+        setAdded(true)
+    };
+
+    useEffect(() => {
+        let timer;
+        if (showAddedNotification) {
+            updateQuery(query);
+            timer = setTimeout(() => {
+                setAdded(false);
+            }, 1000);
+        }
+        return () => clearTimeout(timer);
+    }, [showAddedNotification, query, updateQuery]);
+
+    const addButton = query ? <CopyToClipboard text={query} onCopy={onCopy}>
+                    <span
+                        className={`Queryable-Icon`}
+                        title={`Add "${query}" to the filter`}
+                        style={iconStyle}
+                    >
+                        <AddCircleIcon fontSize="small" color="inherit"></AddCircleIcon>
+                        {showAddedNotification && <span className={'Queryable-AddNotifier'}>Added</span>}
+                    </span>
+				</CopyToClipboard> : null;
+
+    return (
+        <div
+            className={`Queryable-Container displayIconOnMouseOver ${className ? className : ''} ${displayIconOnMouseOver ? 'displayIconOnMouseOver ' : ''}`}
+            style={style}
+            onMouseOver={ e => setShowTooltip(true)}
+            onMouseLeave={ e => setShowTooltip(false)}
+        >
+            {flipped && addButton}
+            {children}
+            {!flipped && addButton}
+            {useTooltip && showTooltip && <span className={'Queryable-Tooltip'}>{query}</span>}
+        </div>
+    );
+};
+
+export default Queryable;

ui/src/components/UI/StatusCode.tsx

@@ -1,5 +1,6 @@
 import React from "react";
 import styles from './style/StatusCode.module.sass';
+import Queryable from "./Queryable";
 
 export enum StatusCodeClassification {
     SUCCESS = "success",
@@ -16,15 +17,20 @@ const StatusCode: React.FC<EntryProps> = ({statusCode, updateQuery}) => {
 
     const classification = getClassification(statusCode)
 
-    return <span
-        title="Status Code"
-        className={`queryable ${styles[classification]} ${styles.base}`}
-        onClick={() => {
-            updateQuery(`response.status == ${statusCode}`)
-        }}
+    return <Queryable
+        query={`response.status == ${statusCode}`}
+        updateQuery={updateQuery}
+        displayIconOnMouseOver={true}
+        flipped={true}
+        iconStyle={{marginTop: "40px"}}
     >
-        {statusCode}
-    </span>
+        <span
+            title="Status Code"
+            className={`${styles[classification]} ${styles.base}`}
+        >
+            {statusCode}
+        </span>
+    </Queryable>
 };
 
 export function getClassification(statusCode: number): string {

ui/src/components/UI/Summary.tsx

@@ -1,6 +1,7 @@
 import miscStyles from "./style/misc.module.sass";
 import React from "react";
 import styles from './style/Summary.module.sass';
+import Queryable from "./Queryable";
 
 interface SummaryProps {
     method: string
@@ -9,24 +10,29 @@ interface SummaryProps {
 }
 
 export const Summary: React.FC<SummaryProps> = ({method, summary, updateQuery}) => {
+
     return <div className={styles.container}>
-        {method && <span
-            title="Method"
-            className={`queryable ${miscStyles.protocol} ${miscStyles.method}`}
-            onClick={() => {
-                updateQuery(`method == "${method}"`)
-            }}
+        {method && <Queryable
+            query={`method == "${method}"`}
+            className={`${miscStyles.protocol} ${miscStyles.method}`}
+            updateQuery={updateQuery}
+            displayIconOnMouseOver={true}
+            style={{whiteSpace: "nowrap"}}
         >
-            {method}
-        </span>}
-        {summary && <div
-            title="Summary"
-            className={`queryable ${styles.summary}`}
-            onClick={() => {
-                updateQuery(`summary == "${summary}"`)
-            }}
+            <span>
+                {method}
+            </span>
+        </Queryable>}
+        {summary && <Queryable
+            query={`summary == "${summary}"`}
+            updateQuery={updateQuery}
+            displayIconOnMouseOver={true}
         >
-            {summary}
-        </div>}
+            <div
+                className={`${styles.summary}`}
+            >
+                {summary}
+            </div>
+        </Queryable>}
     </div>
 };

ui/src/components/UI/style/Queryable.sass

@@ -0,0 +1,48 @@
+.Queryable-Container
+    display: flex
+    align-items: center
+
+    &.displayIconOnMouseOver
+        .Queryable-Icon
+            opacity: 0
+            width: 0px
+            pointer-events: none
+        &:hover
+            .Queryable-Icon
+                opacity: 1
+                pointer-events: all
+
+
+    .Queryable-Icon
+        height: 22px
+        width: 22px
+        cursor: pointer
+        color: #27AE60
+
+        &:hover
+            background-color: rgba(255, 255, 255, 0.06)
+            border-radius: 4px
+            color: #1E884B
+
+    .Queryable-AddNotifier
+        background-color: #1E884B
+        font-weight: normal
+        padding: 2px 5px
+        border-radius: 4px
+        position: absolute
+        transform: translate(0, 10%)
+        color: white
+        z-index: 1000
+        font-size: 11px
+
+    .Queryable-Tooltip
+        background-color: #1E884B
+        font-weight: normal
+        padding: 2px 5px
+        border-radius: 4px
+        position: absolute
+        transform: translate(0, -80%)
+        color: white
+        z-index: 1000
+        font-size: 11px
+

ui/src/components/UI/style/Summary.module.sass

@@ -3,6 +3,4 @@
     align-items: center
 
 .summary
-    text-overflow: ellipsis
-    overflow: hidden
     white-space: nowrap

ui/src/components/UI/style/misc.module.sass

@@ -11,13 +11,14 @@
 
     &.method
         margin-right: 10px
+        height: 12px
 
     &.filterPlate
         border-color: #bcc6dd20
         color: #a0b2ff
         font-size: 10px
 
-.noSelect 
+.noSelect
     -webkit-touch-callout: none
     -webkit-user-select: none
     -khtml-user-select: none

ui/src/components/style/EntriesList.module.sass

@@ -38,6 +38,31 @@
   border: 1px solid #627ef7
   background-color: rgba(255, 255, 255, 0.06)
 
+.spinnerContainer
+  display: flex
+  justify-content: center
+  margin-bottom: 10px
+
+.noMoreDataAvailable
+  text-align: center
+  font-weight: 600
+  color: $secondary-font-color
+
+.btnOld
+  position: absolute
+  top: 20px
+  right: 10px
+  background: #205CF5
+  border-radius: 50%
+  height: 35px
+  width: 35px
+  border: none
+  cursor: pointer
+  z-index: 1
+  img
+    height: 10px
+    transform: scaleY(-1)
+
 .btnLive
   position: absolute
   bottom: 10px

ui/src/components/style/TrafficPage.sass

@@ -114,4 +114,4 @@
 .playPauseIcon
     cursor: pointer
     margin-right: 15px
-    height: 30px
+    height: 30px

ui/src/helpers/api.js

@@ -38,6 +38,14 @@ export default class Api {
         return response.data;
     }
 
+    fetchEntries = async (leftOff, direction, query, limit, timeoutMs) => {
+        const response = await this.client.get(`/entries/?leftOff=${leftOff}&direction=${direction}&query=${query}&limit=${limit}&timeoutMs=${timeoutMs}`).catch(function (thrown) {
+            console.error(thrown.message);
+            return {};
+        });
+        return response.data;
+    }
+
     getRecentTLSLinks = async () => {
         const response = await this.client.get("/status/recentTLSLinks");
         return response.data;

ui/src/index.sass

@@ -22,11 +22,6 @@ code
 .uppercase
   text-transform: uppercase
 
-.queryable
-  cursor: pointer
-  &:hover
-    text-decoration: underline
-
 /****
  * Button
  ***/

ui/src/variables.module.scss

@@ -11,7 +11,7 @@ $blue-gray: #494677;
 
 :export {
   mainBackgroundColor: $main-background-color;
-  headerBackgoundColor: $header-background-color;
+  headerBackgroundColor: $header-background-color;
   fontColor: $font-color;
   secondaryFontColor: $secondary-font-color;
   blueColor: $blue-color;