Added send to socket error validation (#1085)

* selectList sticky header

* selectList changed & servicemap adapted

* ignore eslint

Co-authored-by: Leon <>
Co-authored-by: AmitUp9 <96980485+AmitUp9@users.noreply.github.com>

.gitignore

@@ -55,4 +55,4 @@ tap/extensions/*/expect
 *.editorconfig
 
 # Ignore *.log files
-*.log
+*.log

Dockerfile

@@ -94,8 +94,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
 RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
     chmod +x ./basenine_linux_"${GOARCH}" && \

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -57,13 +57,6 @@ export function rightOnHoverCheck(path, expectedText) {
     cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
 }
 
-export function checkThatAllEntriesShown() {
-    cy.get('#entries-length').then(number => {
-        if (number.text() === '1')
-            cy.get('[title="Fetch old records"]').click();
-    });
-}
-
 export function checkFilterByMethod(funcDict) {
     const {protocol, method, methodQuery, summary, summaryQuery} = funcDict;
     const summaryDict = getSummaryDict(summary, summaryQuery);
@@ -76,12 +69,7 @@ export function checkFilterByMethod(funcDict) {
         cy.get('[type="submit"]').click();
         cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
 
-        cy.get('#entries-length').then(number => {
-            // if the entries list isn't expanded it expands here
-            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
-                cy.get('[title="Fetch old records"]').click();
-
-            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
+            cy.get('#entries-length').should('not.have.text', '0').then(() => {
                 cy.get(`#list [id]`).then(elements => {
                     const listElmWithIdAttr = Object.values(elements);
                     let doneCheckOnFirst = false;
@@ -108,7 +96,6 @@ export function checkFilterByMethod(funcDict) {
                 });
             });
         });
-    });
 }
 
 export function getEntryId(id) {

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,5 +1,4 @@
 import {
-    checkThatAllEntriesShown,
     isValueExistsInElement,
     resizeToHugeMizu,
 } from "../testHelpers/TrafficHelper";
@@ -12,13 +11,14 @@ checkEntries();
 
 function checkEntries() {
     it('checking all entries', function () {
-        checkThatAllEntriesShown();
-        resizeToHugeMizu();
+        cy.get('#entries-length').should('not.have.text', '0').then(() => {
+            resizeToHugeMizu();
 
-        cy.get('#list [id^=entry]').each(entryElement => {
-            entryElement.click();
-            cy.get('#tbody-Headers').should('be.visible');
-            isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+            cy.get('#list [id^=entry]').each(entryElement => {
+                entryElement.click();
+                cy.get('#tbody-Headers').should('be.visible');
+                isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+            });
         });
     });
 }

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -65,70 +65,70 @@ it('right side sanity test', function () {
 checkIllegalFilter('invalid filter');
 
 checkFilter({
-    name: 'http',
+    filter: 'http',
     leftSidePath: '> :nth-child(1) > :nth-child(1)',
     leftSideExpectedText: 'HTTP',
     rightSidePath: '[title=HTTP]',
     rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
-    applyByEnter: true
+    applyByCtrlEnter: true
 });
 
 checkFilter({
-    name: 'response.status == 200',
+    filter: 'response.status == 200',
     leftSidePath: '[title="Status Code"]',
     leftSideExpectedText: '200',
     rightSidePath: '> :nth-child(2) [title="Status Code"]',
     rightSideExpectedText: '200',
-    applyByEnter: false
+    applyByCtrlEnter: false
 });
 
 if (Cypress.env('shouldCheckSrcAndDest')) {
     serviceMapCheck();
 
     checkFilter({
-        name: 'src.name == ""',
+        filter: 'src.name == ""',
         leftSidePath: '[title="Source Name"]',
         leftSideExpectedText: '[Unresolved]',
         rightSidePath: '> :nth-child(2) [title="Source Name"]',
         rightSideExpectedText: '[Unresolved]',
-        applyByEnter: false
+        applyByCtrlEnter: false
     });
 
     checkFilter({
-        name: `dst.name == "httpbin.mizu-tests"`,
+        filter: `dst.name == "httpbin.mizu-tests"`,
         leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
         leftSideExpectedText: 'httpbin.mizu-tests',
         rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
         rightSideExpectedText: 'httpbin.mizu-tests',
-        applyByEnter: false
+        applyByCtrlEnter: false
     });
 }
 
 checkFilter({
-    name: 'request.method == "GET"',
+    filter: 'request.method == "GET"',
     leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
     leftSideExpectedText: 'GET',
     rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
     rightSideExpectedText: 'GET',
-    applyByEnter: true
+    applyByCtrlEnter: true
 });
 
 checkFilter({
-    name: 'request.path == "/get"',
+    filter: 'request.path == "/get"',
     leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
     leftSideExpectedText: '/get',
     rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
     rightSideExpectedText: '/get',
-    applyByEnter: false
+    applyByCtrlEnter: false
 });
 
 checkFilter({
-    name: 'src.ip == "127.0.0.1"',
+    filter: 'src.ip == "127.0.0.1"',
     leftSidePath: '[title="Source IP"]',
     leftSideExpectedText: '127.0.0.1',
     rightSidePath: '> :nth-child(2) [title="Source IP"]',
     rightSideExpectedText: '127.0.0.1',
-    applyByEnter: false
+    applyByCtrlEnter: false
 });
 
 checkFilterNoResults('request.method == "POST"');
@@ -182,17 +182,19 @@ function checkIllegalFilter(illegalFilterName) {
 
 function checkFilter(filterDetails) {
     const {
-        name,
+        filter,
         leftSidePath,
         rightSidePath,
         rightSideExpectedText,
         leftSideExpectedText,
-        applyByEnter
+        applyByCtrlEnter
     } = filterDetails;
 
     const entriesForDeeperCheck = 5;
 
-    it(`checking the filter: ${name}`, function () {
+    it(`checking the filter: ${filter}`, function () {
+        waitForFetch50AndPause();
+
         cy.get('#total-entries').should('not.have.text', '0').then(number => {
             const totalEntries = number.text();
 
@@ -200,30 +202,28 @@ function checkFilter(filterDetails) {
                 const element = elem[0];
                 const entryId = getEntryId(element.id);
                 // checks the hover on the last entry (the only one in DOM at the beginning)
-                leftOnHoverCheck(entryId, leftSidePath, name);
+                leftOnHoverCheck(entryId, leftSidePath, filter);
 
                 cy.get('.w-tc-editor-text').clear();
                 // applying the filter with alt+enter or with the button
-                cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
+                cy.get('.w-tc-editor-text').type(`${filter}${applyByCtrlEnter ? '{ctrl+enter}' : ''}`);
                 cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-                if (!applyByEnter)
+                if (!applyByCtrlEnter)
                     cy.get('[type="submit"]').click();
 
+                waitForFetch50AndPause();
+
                 // only one entry in DOM after filtering, checking all checks on it
                 leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
-                leftOnHoverCheck(entryId, leftSidePath, name);
+                leftOnHoverCheck(entryId, leftSidePath, filter);
 
                 rightTextCheck(rightSidePath, rightSideExpectedText);
-                rightOnHoverCheck(rightSidePath, name);
+                rightOnHoverCheck(rightSidePath, filter);
                 checkRightSideResponseBody();
             });
 
-            cy.get('[title="Fetch old records"]').click();
             resizeToHugeMizu();
 
-            // waiting for the entries number to load
-            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
             // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
             cy.get(`#list [id^=entry]`).each(elem => {
                 const element = elem[0];
@@ -232,7 +232,7 @@ function checkFilter(filterDetails) {
             });
 
             // making the other 3 checks on the first X entries (longer time for each check)
-            deeperCheck(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
+            deeperCheck(leftSidePath, rightSidePath, filter, rightSideExpectedText, entriesForDeeperCheck);
 
             // reloading then waiting for the entries number to load
             resizeToNormalMizu();
@@ -242,7 +242,14 @@ function checkFilter(filterDetails) {
     });
 }
 
-function deeperCheck(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
+function waitForFetch50AndPause() {
+    // wait half a second and pause the stream to preserve the DOM
+    cy.wait(500);
+    cy.get('#pause-icon').click();
+    cy.get('#pause-icon').should('not.be.visible');
+}
+
+function deeperCheck(leftSidePath, rightSidePath, filterName, rightSideExpectedText, entriesNumToCheck) {
     cy.get(`#list [id^=entry]`).each((element, index) => {
         if (index < entriesNumToCheck) {
             const entryId = getEntryId(element[0].id);

agent/go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v1.0.0
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607
+	github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4
 	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0

agent/go.sum

@@ -683,8 +683,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk=
-github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4 h1:nNOrU1HVH0fnaG7GNhxCc8kNPVL035Iix7ihUF6lZT8=
+github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=

agent/main.go

@@ -373,6 +373,7 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 func initializeDependencies() {
 	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
 	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance() })
+	dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
 	dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
 	dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
 	dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })

agent/pkg/api/entry_streamer_socket_connector.go

@@ -5,20 +5,19 @@ import (
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EntryStreamerSocketConnector interface {
-	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams)
-	SendMetadata(socketId int, metadata *basenine.Metadata)
-	SendToastError(socketId int, err error)
+	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
+	SendMetadata(socketId int, metadata *basenine.Metadata) error
+	SendToastError(socketId int, err error) error
 	CleanupSocket(socketId int)
 }
 
 type DefaultEntryStreamerSocketConnector struct{}
 
-func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) {
+func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
 	var message []byte
 	if params.EnableFullEntries {
 		message, _ = models.CreateFullEntryWebSocketMessage(entry)
@@ -29,26 +28,32 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
 	}
 
 	if err := SendToSocket(socketId, message); err != nil {
-		logger.Log.Error(err)
+		return err
 	}
+
+	return nil
 }
 
-func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) {
+func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
 	metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
 	if err := SendToSocket(socketId, metadataBytes); err != nil {
-		logger.Log.Error(err)
+		return err
 	}
+
+	return nil
 }
 
-func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) {
+func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
 	toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
 		Type:      "error",
 		AutoClose: 5000,
 		Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
 	})
 	if err := SendToSocket(socketId, toastBytes); err != nil {
-		logger.Log.Error(err)
+		return err
 	}
+
+	return nil
 }
 
 func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {

agent/pkg/api/main.go

@@ -25,10 +25,7 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
 	"github.com/up9inc/mizu/logger"
-	"github.com/up9inc/mizu/shared"
 	tapApi "github.com/up9inc/mizu/tap/api"
-
-	basenine "github.com/up9inc/basenine/client/go"
 )
 
 var k8sResolver *resolver.Resolver
@@ -103,20 +100,6 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
-BasenineReconnect:
-	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-	if err != nil {
-		logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
-		time.Sleep(shared.BasenineReconnectInterval * time.Second)
-		goto BasenineReconnect
-	}
-	if err = connection.InsertMode(); err != nil {
-		logger.Log.Errorf("Insert mode call failed: %v", err)
-		connection.Close()
-		time.Sleep(shared.BasenineReconnectInterval * time.Second)
-		goto BasenineReconnect
-	}
-
 	disableOASValidation := false
 	ctx := context.Background()
 	doc, contractContent, router, err := loadOAS(ctx)
@@ -163,11 +146,9 @@ BasenineReconnect:
 
 		providers.EntryAdded(len(data))
 
-		if err = connection.SendText(string(data)); err != nil {
-			logger.Log.Errorf("An error occured while inserting a new record to database: %v", err)
-			connection.Close()
-			time.Sleep(shared.BasenineReconnectInterval * time.Second)
-			goto BasenineReconnect
+		entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
+		if err := entryInserter.Insert(mizuEntry); err != nil {
+			logger.Log.Errorf("Error inserting entry, err: %v", err)
 		}
 
 		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)

agent/pkg/api/socket_data_inserter.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap/api"
+	"sync"
+	"time"
+)
+
+type EntryInserter interface {
+	Insert(entry *api.Entry) error
+}
+
+type BasenineEntryInserter struct {
+	connection *basenine.Connection
+}
+
+var instance *BasenineEntryInserter
+var once sync.Once
+
+func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
+	once.Do(func() {
+		instance = &BasenineEntryInserter{}
+	})
+
+	return instance
+}
+
+func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
+	if e.connection == nil {
+		e.connection = initializeConnection()
+	}
+
+	data, err := json.Marshal(entry)
+	if err != nil {
+		return fmt.Errorf("error marshling entry, err: %v", err)
+	}
+
+	if err := e.connection.SendText(string(data)); err != nil {
+		e.connection.Close()
+		e.connection = nil
+
+		return fmt.Errorf("error sending text to database, err: %v", err)
+	}
+
+	return nil
+}
+
+func initializeConnection() *basenine.Connection{
+	for {
+		connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		if err = connection.InsertMode(); err != nil {
+			logger.Log.Errorf("Insert mode call failed: %v", err)
+			connection.Close()
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		return connection
+	}
+}

agent/pkg/api/socket_data_streamer.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
@@ -33,9 +34,18 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 	meta := make(chan []byte)
 
 	query := params.Query
-	err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+	if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
+		if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
+			return err
+		}
+
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
 	if err != nil {
-		entryStreamerSocketConnector.SendToastError(socketId, err)
+		logger.Log.Errorf("Fetch error: %v", err)
 	}
 
 	handleDataChannel := func(c *basenine.Connection, data chan []byte) {
@@ -47,13 +57,15 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 			}
 
 			var entry *tapApi.Entry
-			err = json.Unmarshal(bytes, &entry)
-			if err != nil {
-				logger.Log.Debugf("Error unmarshalling entry: %v", err.Error())
+			if err = json.Unmarshal(bytes, &entry); err != nil {
+				logger.Log.Debugf("Error unmarshalling entry: %v", err)
 				continue
 			}
 
-			entryStreamerSocketConnector.SendEntry(socketId, entry, params)
+			if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
+				logger.Log.Errorf("Error sending entry to socket, err: %v", err)
+				return
+			}
 		}
 	}
 
@@ -66,20 +78,22 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 			}
 
 			var metadata *basenine.Metadata
-			err = json.Unmarshal(bytes, &metadata)
-			if err != nil {
-				logger.Log.Debugf("Error unmarshalling metadata: %v", err.Error())
+			if err = json.Unmarshal(bytes, &metadata); err != nil {
+				logger.Log.Debugf("Error unmarshalling metadata: %v", err)
 				continue
 			}
 
-			entryStreamerSocketConnector.SendMetadata(socketId, metadata)
+			if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
+				logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
+				return
+			}
 		}
 	}
 
 	go handleDataChannel(connection, data)
 	go handleMetaChannel(connection, meta)
 
-	if err = connection.Query(query, data, meta); err != nil {
+	if err = connection.Query(leftOff, query, data, meta); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 		entryStreamerSocketConnector.CleanupSocket(socketId)
 		return err
@@ -94,3 +108,64 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 
 	return nil
 }
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
+	if params.Fetch <= 0 {
+		leftOff = params.LeftOff
+		return
+	}
+
+	var data [][]byte
+	var firstMeta []byte
+	var lastMeta []byte
+	data, firstMeta, lastMeta, err = basenine.Fetch(
+		shared.BasenineHost,
+		shared.BaseninePort,
+		params.LeftOff,
+		-1,
+		params.Query,
+		params.Fetch,
+		time.Duration(params.TimeoutMs)*time.Millisecond,
+	)
+	if err != nil {
+		return
+	}
+
+	var firstMetadata *basenine.Metadata
+	if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
+		return
+	}
+
+	leftOff = firstMetadata.LeftOff
+
+	var lastMetadata *basenine.Metadata
+	if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
+		return
+	}
+
+	if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
+		return
+	}
+
+	data = e.reverseBytesSlice(data)
+	for _, row := range data {
+		var entry *tapApi.Entry
+		if err = json.Unmarshal(row, &entry); err != nil {
+			break
+		}
+
+		if err = connector.SendEntry(socketId, entry, params); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
+	for i := len(arr) - 1; i >= 0; i-- {
+		newArr = append(newArr, arr[i])
+	}
+	return newArr
+}

agent/pkg/api/socket_routes.go

@@ -34,8 +34,11 @@ type SocketConnection struct {
 }
 
 type WebSocketParams struct {
+	LeftOff           string `json:"leftOff"`
 	Query             string `json:"query"`
 	EnableFullEntries bool   `json:"enableFullEntries"`
+	Fetch             int    `json:"fetch"`
+	TimeoutMs         int    `json:"timeoutMs"`
 }
 
 var (

agent/pkg/dependency/type_names.go

@@ -5,6 +5,7 @@ type DependencyContainerType string
 const (
 	ServiceMapGeneratorDependency = "ServiceMapGeneratorDependency"
 	OasGeneratorDependency        = "OasGeneratorDependency"
+	EntriesInserter               = "EntriesInserter"
 	EntriesProvider               = "EntriesProvider"
 	EntriesSocketStreamer         = "EntriesSocketStreamer"
 	EntryStreamerSocketConnector  = "EntryStreamerSocketConnector"

agent/pkg/entries/entries_provider.go

@@ -22,7 +22,7 @@ type EntriesProvider interface {
 type BasenineEntriesProvider struct{}
 
 func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
-	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+	data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
 		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
 		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
 	if err != nil {
@@ -49,7 +49,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 	}
 
 	var metadata *basenine.Metadata
-	err = json.Unmarshal(meta, &metadata)
+	err = json.Unmarshal(lastMeta, &metadata)
 	if err != nil {
 		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
 	}

agent/pkg/har/utils.go

@@ -124,8 +124,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	text := postData["text"]
 	postDataText := ""
@@ -177,8 +177,8 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	encoding := content["encoding"]
 	text := content["text"]

agent/pkg/oas/oas_generator.go

@@ -104,7 +104,7 @@ func (g *defaultOasGenerator) runGenerator() {
 	g.dbMutex.Lock()
 	defer g.dbMutex.Unlock()
 	logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
-	if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil {
+	if err := g.dbConn.Query("latest", g.entriesQuery, dataChan, metaChan); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 	}
 

agent/pkg/up9/main.go

@@ -327,7 +327,7 @@ BasenineReconnect:
 	go handleMetaChannel(&wg, connection, meta)
 	wg.Add(2)
 
-	if err = connection.Query(query, data, meta); err != nil {
+	if err = connection.Query("latest", query, data, meta); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 		connection.Close()
 		time.Sleep(shared.BasenineReconnectInterval * time.Second)

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect9/http/\* expect

tap/extensions/http/main.go

@@ -66,7 +66,7 @@ var grpcProtocol api.Protocol = api.Protocol{
 	BackgroundColor: "#244c5a",
 	ForegroundColor: "#ffffff",
 	FontSize:        11,
-	ReferenceLink:   "https://grpc.github.io/grpc/core/md_doc_statuscodes.html",
+	ReferenceLink:   "https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html",
 	Ports:           []string{"80", "443", "8080", "50051"},
 	Priority:        0,
 }
@@ -401,8 +401,8 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	text := postData["text"]
 	if text != nil {
@@ -483,8 +483,8 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	encoding := content["encoding"]
 	text := content["text"]

tap/extensions/kafka/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/kafka/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect9/kafka/\* expect

tap/extensions/kafka/helpers.go

@@ -3,13 +3,14 @@ package kafka
 import (
 	"encoding/json"
 	"fmt"
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
 	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+
 	"github.com/fatih/camelcase"
 	"github.com/ohler55/ojg/jp"
 	"github.com/ohler55/ojg/oj"
@@ -36,9 +37,14 @@ type KafkaWrapper struct {
 
 func representRequestHeader(data map[string]interface{}, rep []interface{}) []interface{} {
 	requestHeader, _ := json.Marshal([]api.TableData{
+		{
+			Name:     "ApiKeyName",
+			Value:    data["apiKeyName"].(string),
+			Selector: `request.apiKeyName`,
+		},
 		{
 			Name:     "ApiKey",
-			Value:    apiNames[int(data["apiKey"].(float64))],
+			Value:    int(data["apiKey"].(float64)),
 			Selector: `request.apiKey`,
 		},
 		{

tap/extensions/kafka/main.go

@@ -96,8 +96,8 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
 	statusQuery := ""
 
 	apiKey := ApiKey(entry.Request["apiKey"].(float64))
-	method := apiNames[apiKey]
-	methodQuery := fmt.Sprintf("request.apiKey == %d", int(entry.Request["apiKey"].(float64)))
+	method := entry.Request["apiKeyName"].(string)
+	methodQuery := fmt.Sprintf(`request.apiKeyName == "%s"`, method)
 
 	summary := ""
 	summaryQuery := ""

tap/extensions/kafka/request.go

@@ -11,6 +11,7 @@ import (
 
 type Request struct {
 	Size          int32       `json:"size"`
+	ApiKeyName    string      `json:"apiKeyName"`
 	ApiKey        ApiKey      `json:"apiKey"`
 	ApiVersion    int16       `json:"apiVersion"`
 	CorrelationID int32       `json:"correlationID"`
@@ -202,6 +203,7 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, ca
 
 	request := &Request{
 		Size:          size,
+		ApiKeyName:    apiNames[apiKey],
 		ApiKey:        apiKey,
 		ApiVersion:    apiVersion,
 		CorrelationID: correlationID,

tap/tcp_reassembly_stream.go

@@ -10,12 +10,6 @@ import (
 	"github.com/up9inc/mizu/tap/diagnose"
 )
 
-type ReassemblyStream interface {
-	Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool
-	ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext)
-	ReassemblyComplete(ac reassembly.AssemblerContext) bool
-}
-
 type tcpReassemblyStream struct {
 	ident      string
 	tcpState   *reassembly.TCPSimpleFSM
@@ -25,7 +19,7 @@ type tcpReassemblyStream struct {
 	tcpStream  api.TcpStream
 }
 
-func NewTcpReassemblyStream(ident string, tcp *layers.TCP, fsmOptions reassembly.TCPSimpleFSMOptions, stream api.TcpStream) ReassemblyStream {
+func NewTcpReassemblyStream(ident string, tcp *layers.TCP, fsmOptions reassembly.TCPSimpleFSMOptions, stream api.TcpStream) reassembly.Stream {
 	return &tcpReassemblyStream{
 		ident:      ident,
 		tcpState:   reassembly.NewTCPSimpleFSM(fsmOptions),

ui-common/src/components.scss

@@ -0,0 +1,22 @@
+.subSectionHeader{
+    position: relative;
+    font-style: normal;
+    font-weight: bold;
+    font-size: 12px;
+    line-height: 15px;
+    color: $font-color;
+    &::after{
+      content: "";
+      border: 1px solid #E9EBF8;
+      transform: rotate(180deg);
+      position: absolute;
+      left: 0px;
+      right: -100%;
+      top: 100%;
+      bottom: 0%;
+      width: 100%;
+      width: -moz-available;
+      width: -webkit-fill-available;
+      width: strech;
+    }
+}

ui-common/src/components/ServiceMapModal/ServiceMapModal.module.sass

@@ -1,64 +1,98 @@
 @import "../../variables.module"
+@import "../../components"
+
+.closeIcon
+  position: absolute
+  right: 20px
+  top: 20px
 
 .modalContainer
   display: flex
+  height: calc(100% - 110px)
+  background: #F0F5FF
+  padding: 0 15px
+  padding-bottom: 25px
+
+.headerContainer
   width: 100%
-  height: 100%
+  background: white
+  display: flex
+  align-items: center
+  margin-bottom: 15px
+
+.headerSection
+  display: flex
+  align-content: center
+  align-items: center
+  margin-left: 35px
+  margin-bottom: 25px
+  margin-top: 25px
+
+  & .title
+    font-size: 28px
+    color: $blue-gray
+    font-weight: 600
+    margin-right: 35px
+
+  & .actions
 
 .graphSection
   flex: 85%
 
 .filterSection
-  flex: 15%
   height: 100%
+  display: none
+
+  &.show
+    display: inline-block
 
   .filters table
     margin-top: 0px
 
-    tr
-      border-style: none
-
     td
-      color: #8f9bb2
+      color: $light-gray
       font-size: 11px
       font-weight: 600
-      padding-top: 2px
-      padding-bottom: 2px
+      padding-top: 5px
+      padding-bottom: 5px
+    th
+      font-size: 12px
 
 .colorBlock
   display: inline-block
-  height: 15px
-  width: 50px
+  height: 12px
+  width: 22px
 
 .filterWrapper
   height: 100%
   display: flex
   flex-direction: column
-  margin-right: 10px
   width: 100%
+  border-radius: 4px
 
 .servicesFilterSearch
-  width: calc(100% - 10px)
-  max-width: 300px
+  width: -moz-available
+  width: -webkit-fill-available
+  width: fill-available
+  max-width: 200px
   box-shadow: 0px 1px 5px #979797
   margin-left: 10px
   margin-bottom: 5px
+  margin-top: 10px
+  margin-right: 10px
 
-.servicesFilter
-  margin-top: 15px
+.card
+  background: white
+  padding: 10px
+  border-radius: 4px
+  user-select: none
+  box-shadow: 0px 1px 5px #979797
+
+.servicesFilterWrapper
+  margin-top: 20px
+  margin-bottom: 3px
   height: 100%
   overflow: hidden
 
-  & .servicesFilterList
-    overflow-y: auto
-    height: calc(100% - 30px - 5px)
-
-.separtorLine
-  margin-top: 10px
-  border: 1px solid #E9EBF8
-
-.closeIcon
-  cursor: pointer
-  user-select: none
-  margin-top: -15px
-  margin-right: 5px
+.servicesFilterList
+  height: calc(100% - 30px - 52px)

ui-common/src/components/ServiceMapModal/ServiceMapModal.tsx

@@ -8,6 +8,8 @@ import debounce from 'lodash/debounce';
 import ServiceMapOptions from './ServiceMapOptions'
 import { useCommonStyles } from "../../helpers/commonStyle";
 import refreshIcon from "assets/refresh.svg";
+import filterIcon from "assets/filter-icon.svg";
+import filterIconClicked from "assets/filter-icon-clicked.svg";
 import closeIcon from "assets/close.svg"
 import styles from './ServiceMapModal.module.sass'
 import SelectList from "../UI/SelectList";
@@ -23,14 +25,14 @@ const modalStyle = {
     transform: 'translate(-50%, 0%)',
     width: '89vw',
     height: '82vh',
-    bgcolor: 'background.paper',
+    bgcolor: '#F0F5FF',
     borderRadius: '5px',
     boxShadow: 24,
     p: 4,
     color: '#000',
-    padding: "25px 15px"
+    padding: "1px 1px",
+    paddingBottom: "15px"
 };
-
 interface LegentLabelProps {
     color: string,
     name: string
@@ -45,14 +47,11 @@ const LegentLabel: React.FC<LegentLabelProps> = ({ color, name }) => {
     </React.Fragment>
 }
 
-const protocols = [
-    { key: "HTTP", value: "HTTP", component: <LegentLabel color="#205cf5" name="HTTP" /> },
-    { key: "HTTP/2", value: "HTTP/2", component: <LegentLabel color='#244c5a' name="HTTP/2" /> },
-    { key: "gRPC", value: "gRPC", component: <LegentLabel color='#244c5a' name="gRPC" /> },
-    { key: "GQL", value: "GQL", component: <LegentLabel color='#e10098' name="GQL" /> },
-    { key: "AMQP", value: "AMQP", component: <LegentLabel color='#ff6600' name="AMQP" /> },
-    { key: "KAFKA", value: "KAFKA", component: <LegentLabel color='#000000' name="KAFKA" /> },
-    { key: "REDIS", value: "REDIS", component: <LegentLabel color='#a41e11' name="REDIS" /> },]
+type ProtocolType = {
+    key: string;
+    value: string;
+    component: JSX.Element;
+};
 
 
 interface ServiceMapModalProps {
@@ -66,11 +65,11 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
     const commonClasses = useCommonStyles();
     const [isLoading, setIsLoading] = useState<boolean>(true);
     const [graphData, setGraphData] = useState<GraphData>({ nodes: [], edges: [] });
-    const [checkedProtocols, setCheckedProtocols] = useState(protocols.map(x => x.key))
+    const [checkedProtocols, setCheckedProtocols] = useState([])
     const [checkedServices, setCheckedServices] = useState([])
     const [serviceMapApiData, setServiceMapApiData] = useState<ServiceMapGraph>({ edges: [], nodes: [] })
-    const [servicesSearchVal, setServicesSearchVal] = useState("")
     const [graphOptions, setGraphOptions] = useState(ServiceMapOptions);
+    const [isFilterClicked, setIsFilterClicked] = useState(true)
 
     const getServiceMapData = useCallback(async () => {
         try {
@@ -113,15 +112,23 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
             },
         }
     }
-    const mapToKeyValForFilter = (arr) => arr.map(mapNodesDatatoGraph)
+    const mapToKeyValForFilter = useCallback((arr) => arr.map(mapNodesDatatoGraph)
         .map((edge) => { return { key: edge.label, value: edge.label } })
-        .sort((a, b) => { return a.key.localeCompare(b.key) });
+        .sort((a, b) => { return a.key.localeCompare(b.key) }), [])
+
+    const getProtocolsForFilter = useMemo(() => {
+        return serviceMapApiData.edges.reduce<ProtocolType[]>((returnArr, currentValue, currentIndex, array) => {
+            if (!returnArr.find(prot => prot.key === currentValue.protocol.abbr))
+                returnArr.push({ key: currentValue.protocol.abbr, value: currentValue.protocol.abbr, component: <LegentLabel color={currentValue.protocol.backgroundColor} name={currentValue.protocol.abbr} /> })
+            return returnArr
+        }, new Array<ProtocolType>())
+    }, [serviceMapApiData])
 
     const getServicesForFilter = useMemo(() => {
         const resolved = mapToKeyValForFilter(serviceMapApiData.nodes?.filter(x => x.resolved))
         const unResolved = mapToKeyValForFilter(serviceMapApiData.nodes?.filter(x => !x.resolved))
         return [...resolved, ...unResolved]
-    }, [serviceMapApiData])
+    }, [mapToKeyValForFilter, serviceMapApiData.nodes])
 
     useEffect(() => {
         const newGraphData: GraphData = {
@@ -142,10 +149,18 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
     }
 
     useEffect(() => {
-        if (checkedServices.length == 0)
+        if (checkedServices.length === 0)
             setCheckedServices(getServicesForFilter.map(x => x.key).filter(serviceName => !Utils.isIpAddress(serviceName)))
+        // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [getServicesForFilter])
 
+    useEffect(() => {
+        if (checkedProtocols.length === 0) {
+            setCheckedProtocols(getProtocolsForFilter.map(x => x.key))
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getProtocolsForFilter])
+
     useEffect(() => {
         getServiceMapData()
     }, [getServiceMapData])
@@ -173,20 +188,45 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
             BackdropProps={{ timeout: 500 }}>
             <Fade in={isOpen}>
                 <Box sx={modalStyle}>
+                    <div className={styles.closeIcon}>
+                        <img src={closeIcon} alt="close" onClick={() => onClose()} style={{ cursor: "pointer", userSelect: "none" }}></img>
+                    </div>
+                    <div className={styles.headerContainer}>
+                        <div className={styles.headerSection}>
+                            <span className={styles.title}>Services</span>
+                            <Button size="medium"
+                                variant="contained"
+                                startIcon={<img src={isFilterClicked ? filterIconClicked : filterIcon} className="custom" alt="refresh" style={{ height: "26px", width: "26px" }}></img>}
+                                className={commonClasses.outlinedButton + " " + commonClasses.imagedButton + ` ${isFilterClicked ? commonClasses.clickedButton : ""}`}
+                                onClick={() => setIsFilterClicked(prevState => !prevState)}
+                                style={{ textTransform: 'unset' }}>
+                                Filter
+                            </Button >
+                            <Button style={{ marginLeft: "2%", textTransform: 'unset' }}
+                                startIcon={<img src={refreshIcon} className="custom" alt="refresh"></img>}
+                                size="medium"
+                                variant="contained"
+                                className={commonClasses.outlinedButton + " " + commonClasses.imagedButton}
+                                onClick={refreshServiceMap}
+                            >
+                                Refresh
+                            </Button>
+                        </div>
+                    </div>
+
                     <div className={styles.modalContainer}>
-                        <div className={styles.filterSection}>
+                        <div className={styles.filterSection + ` ${isFilterClicked ? styles.show : ""}`}>
                             <Resizeable minWidth={170} maxWidth={320}>
                                 <div className={styles.filterWrapper}>
-                                    <div className={styles.protocolsFilterList}>
-                                        <SelectList items={protocols} checkBoxWidth="5%" tableName={"Protocols"} multiSelect={true}
-                                            checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters} />
+                                    <div className={styles.card}>
+                                        <SelectList items={getProtocolsForFilter} checkBoxWidth="5%" tableName={"PROTOCOLS"} multiSelect={true}
+                                            checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters}
+                                            inputSearchClass={styles.servicesFilterSearch} isFilterable={false}/>
                                     </div>
-                                    <div className={styles.separtorLine}></div>
-                                    <div className={styles.servicesFilter}>
-                                        <input className={commonClasses.textField + ` ${styles.servicesFilterSearch}`} placeholder="search service" value={servicesSearchVal} onChange={(event) => setServicesSearchVal(event.target.value)} />
+                                    <div className={styles.servicesFilterWrapper + ` ${styles.card}`}>
                                         <div className={styles.servicesFilterList}>
-                                            <SelectList items={getServicesForFilter} tableName={"Services"} tableClassName={styles.filters} multiSelect={true} searchValue={servicesSearchVal}
-                                                checkBoxWidth="5%" checkedValues={checkedServices} setCheckedValues={onServiceChanges} />
+                                            <SelectList items={getServicesForFilter} tableName={"SERVICES"} tableClassName={styles.filters} multiSelect={true}
+                                                checkBoxWidth="5%" checkedValues={checkedServices} setCheckedValues={onServiceChanges} inputSearchClass={styles.servicesFilterSearch}/>
                                         </div>
                                     </div>
                                 </div>
@@ -194,16 +234,6 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
                         </div>
                         <div className={styles.graphSection}>
                             <div style={{ display: "flex", justifyContent: "space-between" }}>
-                                <Button style={{ marginLeft: "3%" }}
-                                    startIcon={<img src={refreshIcon} className="custom" alt="refresh" style={{ marginRight: "8%" }}></img>}
-                                    size="medium"
-                                    variant="contained"
-                                    className={commonClasses.outlinedButton + " " + commonClasses.imagedButton}
-                                    onClick={refreshServiceMap}
-                                >
-                                    Refresh
-                                </Button>
-                                <img src={closeIcon} alt="close" onClick={() => onClose()} className={styles.closeIcon}></img>
                             </div>
                             {isLoading && <div className={spinnerStyle.spinnerContainer}>
                                 <img alt="spinner" src={spinnerImg} style={{ height: 50 }} />

ui-common/src/components/ServiceMapModal/assets/filter-icon-clicked.svg

@@ -0,0 +1,3 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6 8L10.6667 12.6667V18H13.3333V12.6667L18 8V6H6V8Z" stroke="white"/>
+</svg>

ui-common/src/components/ServiceMapModal/assets/filter-icon.svg

@@ -0,0 +1,3 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6 8L10.6667 12.6667V18H13.3333V12.6667L18 8V6H6V8Z" stroke="#205CF5"/>
+</svg>

ui-common/src/components/TrafficViewer/EntriesList.tsx

@@ -23,7 +23,7 @@ interface EntriesListProps {
   setIsSnappedToBottom: any;
   noMoreDataTop: boolean;
   setNoMoreDataTop: (flag: boolean) => void;
-  openWebSocket: (query: string, resetEntries: boolean) => void;
+  openWebSocket: (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => void;
   scrollableRef: any;
   ws: any;
 }
@@ -195,11 +195,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({
                 className={`${styles.btnLive} ${isSnappedToBottom && !isWsConnectionClosed ? styles.hideButton : styles.showButton}`}
                 onClick={(_) => {
                   if (isWsConnectionClosed) {
-                    if (query) {
-                      openWebSocket(`(${query}) and leftOff("${leftOffBottom}")`, false);
-                    } else {
-                      openWebSocket(`leftOff("${leftOffBottom}")`, false);
-                    }
+                    openWebSocket(leftOffBottom, query, false, 0, 0);
                   }
                   scrollableRef.current.jumpToBottom();
                   setIsSnappedToBottom(true);

ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.tsx

@@ -289,10 +289,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, style, headingMode, name
                             src={ingoingIcon}
                             alt="Ingoing traffic"
                             title="Ingoing"
-                            onClick={() => {
-                                const query = `outgoing == false`;
-                                setQuery(queryState ? `${queryState} and ${query}` : query);
-                            }}
                         />
                     </Queryable>
                 }

ui-common/src/components/TrafficViewer/TrafficViewer.tsx

@@ -20,7 +20,7 @@ import {StatusBar} from "../UI/StatusBar";
 import tappingStatusAtom from "../../recoil/tappingStatus/atom";
 import {TOAST_CONTAINER_ID} from "../../configs/Consts";
 import leftOffTopAtom from "../../recoil/leftOffTop";
-import { DEFAULT_QUERY } from '../../hooks/useWS';
+import { DEFAULT_LEFTOFF, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS } from '../../hooks/useWS';
 
 const useLayoutStyles = makeStyles(() => ({
   details: {
@@ -114,11 +114,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
   const ws = useRef(null);
 
   const openEmptyWebSocket = () => {
-    if (query) {
-      openWebSocket(`(${query}) and ${DEFAULT_QUERY}`, true);
-    } else {
-      openWebSocket(DEFAULT_QUERY, true);
-    }
+    openWebSocket(DEFAULT_LEFTOFF, query, true, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS);
   }
 
   const closeWebSocket = () => {
@@ -129,7 +125,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
   }
 
   const listEntry = useRef(null);
-  const openWebSocket = (query: string, resetEntries: boolean) => {
+  const openWebSocket = (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => {
     if (resetEntries) {
       setFocusedEntryId(null);
       setEntries([]);
@@ -138,7 +134,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
     }
     try {
       ws.current = new WebSocket(webSocketUrl);
-      sendQueryWhenWsOpen(query);
+      sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs);
 
       ws.current.onopen = () => {
         setWsReadyState(ws?.current?.readyState);
@@ -157,12 +153,18 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
     }
   }
 
-  const sendQueryWhenWsOpen = (query) => {
+  const sendQueryWhenWsOpen = (leftOff: string, query: string, fetch: number, fetchTimeoutMs: number) => {
     setTimeout(() => {
       if (ws?.current?.readyState === WebSocket.OPEN) {
-        ws.current.send(JSON.stringify({"query": query, "enableFullEntries": false}));
+        ws.current.send(JSON.stringify({
+          "leftOff": leftOff,
+          "query": query,
+          "enableFullEntries": false,
+          "fetch": fetch,
+          "timeoutMs": fetchTimeoutMs
+        }));
       } else {
-        sendQueryWhenWsOpen(query);
+        sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs);
       }
     }, 500)
   }
@@ -240,16 +242,21 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
 
   return (
     <div className={TrafficViewerStyles.TrafficPage}>
-      {tappingStatus && isShowStatusBar && <StatusBar isDemoBannerView={isDemoBannerView}/>}
+      {tappingStatus && isShowStatusBar && <StatusBar disabled={ws?.current?.readyState !== WebSocket.OPEN} isDemoBannerView={isDemoBannerView}/>}
       <div className={TrafficViewerStyles.TrafficPageHeader}>
         <div className={TrafficViewerStyles.TrafficPageStreamStatus}>
-          <img className={TrafficViewerStyles.playPauseIcon}
-               style={{visibility: wsReadyState === WebSocket.OPEN ? "visible" : "hidden"}} alt="pause"
-               src={pauseIcon} onClick={toggleConnection}/>
-          <img className={TrafficViewerStyles.playPauseIcon}
+          <img id="pause-icon"
+               className={TrafficViewerStyles.playPauseIcon}
+               style={{visibility: wsReadyState === WebSocket.OPEN ? "visible" : "hidden"}}
+               alt="pause"
+               src={pauseIcon}
+               onClick={toggleConnection}/>
+          <img id="play-icon"
+               className={TrafficViewerStyles.playPauseIcon}
                style={{position: "absolute", visibility: wsReadyState === WebSocket.OPEN ? "hidden" : "visible"}}
                alt="play"
-               src={playIcon} onClick={toggleConnection}/>
+               src={playIcon}
+               onClick={toggleConnection}/>
           <div className={TrafficViewerStyles.connectionText}>
             {getConnectionTitle()}
             {getConnectionIndicator()}

ui-common/src/components/UI/SelectList.tsx

@@ -3,6 +3,7 @@ import Radio from "./Radio";
 import styles from './style/SelectList.module.sass'
 import NoDataMessage from "./NoDataMessage";
 import Checkbox from "./Checkbox";
+import { useCommonStyles } from "../../helpers/commonStyle";
 
 
 export interface Props {
@@ -10,14 +11,17 @@ export interface Props {
     tableName: string;
     checkedValues?: string[];
     multiSelect: boolean;
-    searchValue?: string;
     setCheckedValues: (newValues) => void;
-    tableClassName?
-    checkBoxWidth?: string
+    tableClassName?;
+    checkBoxWidth?: string;
+    inputSearchClass? : string
+    isFilterable? : boolean
 }
 
-const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], multiSelect = true, searchValue = "", setCheckedValues, tableClassName,
-    checkBoxWidth = 50 }) => {
+const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], multiSelect = true, setCheckedValues, tableClassName,
+    checkBoxWidth = 50 ,inputSearchClass,isFilterable = true}) => {
+    const commonClasses = useCommonStyles();
+    const [searchValue, setSearchValue] = useState("")  
     const noItemsMessage = "No items to show";
     const [headerChecked, setHeaderChecked] = useState(false)
 
@@ -72,15 +76,16 @@ const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], mul
     const tableHead = multiSelect ? <tr style={{ borderBottomWidth: "2px" }}>
         <th style={{ width: checkBoxWidth }}><Checkbox data-cy="checkbox-all" checked={headerChecked}
             onToggle={(isChecked) => toggleAll(isChecked)} /></th>
-        <th>{tableName}</th>
+        <th>
+            All
+        </th>
     </tr> :
-        <tr style={{ borderBottomWidth: "2px" }}>
-            <th>{tableName}</th>
+        <tr>
         </tr>
 
     const tableBody = filteredValues.length === 0 ?
         <tr>
-            <td colSpan={2}>
+            <td colSpan={2} className={styles.displayBlock}>
                 <NoDataMessage messageText={noItemsMessage} />
             </td>
         </tr>
@@ -98,7 +103,14 @@ const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], mul
         }
         )
 
-    return <div className={tableClassName ? tableClassName + ` ${styles.selectListTable}` : ` ${styles.selectListTable}`}>
+    return <React.Fragment>
+        <h3 className={styles.subSectionHeader}>
+            {tableName}
+            <span className={styles.totalSelected}>&nbsp;({checkedValues.length})</span>
+        </h3>
+        {isFilterable && <input className={commonClasses.textField + ` ${inputSearchClass}`} placeholder="Search" value={searchValue}
+                                onChange={(event) => setSearchValue(event.target.value)} data-cy="searchInput" />}
+        <div className={tableClassName ? tableClassName + ` ${styles.selectListTable}` : ` ${styles.selectListTable}`} style={{marginTop: !multiSelect ? "20px":  ""}}>
         <table cellPadding={5} style={{ borderCollapse: "collapse" }}>
             <thead>
                 {tableHead}
@@ -108,6 +120,7 @@ const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], mul
             </tbody>
         </table>
     </div>
+    </React.Fragment>
 }
 
-export default SelectList;
+export default SelectList;

ui-common/src/components/UI/StatusBar.tsx

@@ -5,6 +5,7 @@ import failIcon from 'assets/failed.svg';
 import successIcon from 'assets/success.svg';
 import {useRecoilValue} from "recoil";
 import tappingStatusAtom, {tappingStatusDetails} from "../../recoil/tappingStatus";
+import Tooltip from "./Tooltip";
 
 const pluralize = (noun: string, amount: number) => {
     return `${noun}${amount !== 1 ? 's' : ''}`
@@ -12,20 +13,22 @@ const pluralize = (noun: string, amount: number) => {
 
 interface StatusBarProps {
    isDemoBannerView: boolean;
+   disabled?: boolean;
   }
 
-export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView}) => {
+export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView, disabled}) => {
     const tappingStatus = useRecoilValue(tappingStatusAtom);
     const [expandedBar, setExpandedBar] = useState(false);
     const {uniqueNamespaces, amountOfPods, amountOfTappedPods, amountOfUntappedPods} = useRecoilValue(tappingStatusDetails);
-    return <div className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
+    return <div style={{opacity: disabled ? 0.4 : 1}} className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar && !disabled ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
         <div className={style.podsCount}>
         {tappingStatus.some(pod => !pod.isTapped) && <img src={warningIcon} alt="warning"/>}
+          {disabled &&  <Tooltip title={"Tapping status is not updated when streaming is paused"} isSimple><img src={warningIcon} alt="warning"/></Tooltip>}
             <span className={style.podsCountText} data-cy="podsCountText">
                 {`Tapping ${amountOfUntappedPods > 0 ? amountOfTappedPods + " / " + amountOfPods : amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}
             </span>
         </div>
-        {expandedBar && <div style={{marginTop: 20}}>
+        {expandedBar && !disabled && <div style={{marginTop: 20}}>
             <table>
                 <thead>
                     <tr>

ui-common/src/components/UI/style/Resizeable.module.sass

@@ -5,7 +5,6 @@
     align-items: center
     display: flex
     overflow: hidden
-    border-right: 1px solid $blue-color
     height: 100%
     width: 100%
     padding-right: 3px

ui-common/src/components/UI/style/SelectList.module.sass

@@ -1,33 +1,70 @@
 @import '../../../variables.module'
+@import '../../../components'
 
 .selectListTable
-    overflow: auto
-    height: 100%
     user-select: none // when resizble moved we get unwanted beheviour
+    height: 100%
 
     table
         width: 100%
-        margin-top: 20px
         border-collapse: collapse
+        table-layout: fixed
+        height: 100%
+        display: flex
+        flex-flow: column
+        height: 100%
 
-        th
-            color: $blue-gray
-            text-align: left
-            padding: 10px
-            position: sticky
-            top: 0
-            background: $main-background-color
+    thead
+        display: table
+        table-layout: fixed
+        flex: 0 0 auto
+        width: calc(100% - 0.9em)
 
-        tr
-            border-bottom-width: 1px
-            border-bottom-color: $data-background-color
-            border-bottom-style: solid
-            width: 100%
+    tbody
+        display: block
+        overflow: auto
+        width: 100%
+        height: 100%
+        flex: 1 1 auto
 
-        td
-            color: $light-gray
-            padding: 10px
-            font-size: 16px
+    tbody tr:hover
+        background: $header-background-color
+
+    th
+        color: $blue-gray
+        text-align: left
+        padding: 10px
+        background: $main-background-color
+        font-size: 12px
+
+    tr
+        border-bottom-width: 1px
+        border-bottom-color: $data-background-color
+        border-bottom-style: solid
+        width: 100%
+        display: block
+        position: relative
+        display: table
+        table-layout: fixed
+
+    td
+        color: $light-gray
+        padding: 10px
+        font-size: 11px
+        font-weight: 600
+        padding-top: 5px
+        padding-bottom: 5px
 
 .nowrap
     white-space: nowrap
+
+.totalSelected
+    font-size: 12px
+    color: $light-blue-color
+    font-weight: 700
+
+.displayBlock
+    display: block
+
+.filterInput
+    margin-bottom: 20px

ui-common/src/helpers/commonStyle.ts

@@ -1,4 +1,5 @@
-import {makeStyles} from "@material-ui/core";
+import { makeStyles } from "@material-ui/core";
+import variables from "../variables.module.scss"
 
 // @ts-ignore
 export const useCommonStyles = makeStyles(() => ({
@@ -9,11 +10,10 @@ export const useCommonStyles = makeStyles(() => ({
         fontSize: 12,
         padding: "9px 12px",
         borderRadius: "6px ! important",
-
         "&:hover": {
             backgroundColor: "#205cf5",
         },
-        "&:disabled":{
+        "&:disabled": {
             backgroundColor: "rgba(0, 0, 0, 0.26)"
         }
     },
@@ -25,12 +25,17 @@ export const useCommonStyles = makeStyles(() => ({
         padding: "8px 12px",
         border: "1px #205cf5 solid",
         borderRadius: "6px ! important",
-
         "&:hover": {
             backgroundColor: "transparent",
         },
     },
-
+    clickedButton: {
+        color: "white",
+        backgroundColor: "#205cf5",
+        "&:hover": {
+            backgroundColor: "#205cf5",
+        },
+    },
     imagedButton: {
         padding: "1px 14px"
     },
@@ -46,7 +51,7 @@ export const useCommonStyles = makeStyles(() => ({
         height: "30px",
         boxSizing: "border-box"
     },
-    modal :{
+    modal: {
         position: 'absolute',
         top: '40%',
         left: '50%',

ui-common/src/hooks/WindowDimensionsHook.tsx

@@ -14,11 +14,11 @@ export function useRequestTextByWidth(windowWidth){
     let responseText = "Response: "
     let elapsedTimeText = "Elapsed Time: "
 
-    if (windowWidth < 1078) {
+    if (windowWidth < 1436) {
         requestText = ""
         responseText = ""
         elapsedTimeText = ""
-    } else if (windowWidth < 1356) {
+    } else if (windowWidth < 1700) {
         requestText = "Req: "
         responseText = "Res: "
         elapsedTimeText = "ET: "
@@ -38,6 +38,6 @@ export default function useWindowDimensions() {
         window.addEventListener('resize', handleResize);
         return () => window.removeEventListener('resize', handleResize);
     }, []);
-
+    
     return windowDimensions;
 }

ui-common/src/hooks/useWS.tsx

@@ -7,7 +7,9 @@ enum WebSocketReadyState {
   CLOSED
 }
 
-export const DEFAULT_QUERY = `leftOff("latest")`;
+export const DEFAULT_LEFTOFF = `latest`;
+export const DEFAULT_FETCH = 50;
+export const DEFAULT_FETCH_TIMEOUT_MS = 3000;
 
 const useWS = (wsUrl: string) => {
   const [message, setMessage] = useState(null);

ui-common/src/index.tsx

@@ -1,11 +1,11 @@
 import TrafficViewer from './components/TrafficViewer/TrafficViewer';
 import * as UI from "./components/UI"
 import { StatusBar } from './components/UI';
-import useWS, { DEFAULT_QUERY } from './hooks/useWS';
+import useWS, { DEFAULT_LEFTOFF } from './hooks/useWS';
 import { AnalyzeButton } from "./components/AnalyzeButton/AnalyzeButton"
 import OasModal from './components/OasModal/OasModal';
 import { ServiceMapModal } from './components/ServiceMapModal/ServiceMapModal';
 
 export { UI, AnalyzeButton, StatusBar, OasModal, ServiceMapModal }
-export { useWS, DEFAULT_QUERY }
+export { useWS, DEFAULT_LEFTOFF }
 export default TrafficViewer;

ui-common/src/variables.module.scss

@@ -7,6 +7,10 @@ $blue-color: #205CF5;
 $light-blue-color: #BCCEFD;
 $success-color: #27AE60;
 $failure-color: #EB5757;
+
+$header-section-color : #fbfcfe;
+$content-section-color: #f8f9fc;
+
 $blue-gray: #494677;
 $light-gray: #8F9BB2;