TRA-4331 Implement full data streaming over WebSocket (#819)

* Implement full data streaming over WebSocket

* Fix the linting error

* Make the empty being the criteria

* Use a label to break the nested loop

acceptanceTests/cypress.json

@@ -15,7 +15,8 @@
     "tests/IgnoredUserAgents.js",
     "tests/UiTest.js",
     "tests/Redis.js",
-    "tests/Rabbit.js"
+    "tests/Rabbit.js",
+    "tests/serviceMapFunction.js"
   ],
 
   "env": {
@@ -26,6 +27,9 @@
     "minimumEntries": 25,
     "greenFilterColor": "rgb(210, 250, 210)",
     "redFilterColor": "rgb(250, 214, 220)",
-    "bodyJsonClass": ".hljs"
+    "bodyJsonClass": ".hljs",
+    "mizuWidth": 1920,
+    "normalMizuHeight": 1080,
+    "hugeMizuHeight": 3500
   }
 }

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -4,6 +4,8 @@ export const valueTabs = {
     none: null
 }
 
+const maxEntriesInDom = 13;
+
 export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
     it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
         cy.get(domPathToContainer).then(htmlText => {
@@ -15,11 +17,11 @@ export function isValueExistsInElement(shouldInclude, content, domPathToContaine
 }
 
 export function resizeToHugeMizu() {
-    cy.viewport(1920, 3500);
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('hugeMizuHeight'));
 }
 
 export function resizeToNormalMizu() {
-    cy.viewport(1920, 1080);
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('normalMizuHeight'));
 }
 
 export function verifyMinimumEntries() {
@@ -61,7 +63,7 @@ export function checkThatAllEntriesShown() {
 }
 
 export function checkFilterByMethod(funcDict) {
-    const {protocol, method, summary} = funcDict;
+    const {protocol, method, summary, hugeMizu} = funcDict;
     const summaryDict = getSummeryDict(summary);
     const methodDict = getMethodDict(method);
     const protocolDict = getProtocolDict(protocol.name, protocol.text);
@@ -82,20 +84,24 @@ export function checkFilterByMethod(funcDict) {
                     const listElmWithIdAttr = Object.values(elements);
                     let doneCheckOnFirst = false;
 
-                    listElmWithIdAttr.forEach(entry => {
-                        if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
-                            const entryNum = getEntryNumById(entry.id);
+                    cy.get('#entries-length').invoke('text').then(len => {
+                        resizeIfNeeded(len);
+                        listElmWithIdAttr.forEach(entry => {
+                            if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
+                                const entryNum = getEntryNumById(entry.id);
 
-                            leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
-                            leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
-                            if (summaryDict)
-                                leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
+                                leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
+                                leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                                if (summaryDict)
+                                    leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
 
-                            if (!doneCheckOnFirst) {
-                                deepCheck(funcDict, protocolDict, methodDict, entry);
-                                doneCheckOnFirst = true;
+                                if (!doneCheckOnFirst) {
+                                    deepCheck(funcDict, protocolDict, methodDict, entry);
+                                    doneCheckOnFirst = true;
+                                }
                             }
-                        }
+                        });
+                        resizeIfNeeded(len);
                     });
                 });
             });
@@ -103,6 +109,13 @@ export function checkFilterByMethod(funcDict) {
     });
 }
 
+function resizeIfNeeded(entriesLen) {
+    if (entriesLen > maxEntriesInDom){
+        Cypress.config().viewportHeight === Cypress.env('normalMizuHeight') ?
+            resizeToHugeMizu() : resizeToNormalMizu()
+    }
+}
+
 function deepCheck(generalDict, protocolDict, methodDict, entry) {
     const entryNum = getEntryNumById(entry.id);
     const {summary, value} = generalDict;

acceptanceTests/cypress/integration/tests/Rabbit.js

@@ -1,8 +1,8 @@
 import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
 
-
 it('opening mizu', function () {
     cy.visit(Cypress.env('testUrl'));
+    cy.get('#total-entries').invoke('text').should('match', /^[4-7][0-9]$/m)
 });
 
 const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};

agent/pkg/api/socket_routes.go

@@ -110,20 +110,31 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 		logger.Log.Error(err)
 	}
 
+out:
 	for {
-		_, msg, err := ws.ReadMessage()
-		if err != nil {
-			if _, ok := err.(*websocket.CloseError); ok {
-				logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
-			} else {
-				logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
-			}
+		// params[0]: query
+		// params[1]: enableFullEntries (empty: disable, non-empty: enable)
+		params := make([][]byte, 2)
+		for i := range params {
+			_, params[i], err = ws.ReadMessage()
+			if err != nil {
+				if _, ok := err.(*websocket.CloseError); ok {
+					logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
+				} else {
+					logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
+				}
 
-			break
+				break out
+			}
+		}
+
+		enableFullEntries := false
+		if len(params[1]) > 0 {
+			enableFullEntries = true
 		}
 
 		if !isTapper && !isQuerySet {
-			query := string(msg)
+			query := string(params[0])
 			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
 			if err != nil {
 				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
@@ -150,10 +161,15 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					var entry *tapApi.Entry
 					err = json.Unmarshal(bytes, &entry)
 
-					base := tapApi.Summarize(entry)
+					var message []byte
+					if enableFullEntries {
+						message, _ = models.CreateFullEntryWebSocketMessage(entry)
+					} else {
+						base := tapApi.Summarize(entry)
+						message, _ = models.CreateBaseEntryWebSocketMessage(base)
+					}
 
-					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
-					if err := SendToSocket(socketId, baseEntryBytes); err != nil {
+					if err := SendToSocket(socketId, message); err != nil {
 						logger.Log.Error(err)
 					}
 				}
@@ -185,7 +201,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 
 			connection.Query(query, data, meta)
 		} else {
-			eventHandlers.WebSocketMessage(socketId, msg)
+			eventHandlers.WebSocketMessage(socketId, params[0])
 		}
 	}
 }

agent/pkg/models/models.go

@@ -42,6 +42,11 @@ type WebSocketEntryMessage struct {
 	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
+type WebSocketFullEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tapApi.Entry `json:"data,omitempty"`
+}
+
 type WebSocketTappedEntryMessage struct {
 	*shared.WebSocketMessageMetadata
 	Data *tapApi.OutputChannelItem
@@ -88,6 +93,16 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	return json.Marshal(message)
 }
 
+func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
+	message := &WebSocketFullEntryMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeFullEntry,
+		},
+		Data: entry,
+	}
+	return json.Marshal(message)
+}
+
 func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
 	message := &WebSocketTappedEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{

cli/Makefile

@@ -1,3 +1,9 @@
+SHELL=/bin/bash
+
+.PHONY: help
+.DEFAULT_GOAL := help
+.ONESHELL:
+
 SUFFIX=$(GOOS)_$(GOARCH)
 COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
@@ -5,9 +11,6 @@ GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 BUILD_TIMESTAMP=$(shell date +%s)
 export VER?=0.0
 
-.PHONY: help
-.DEFAULT_GOAL := help
-
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
@@ -16,15 +19,20 @@ install:
 
 build-debug:  ## Build mizu CLI for debug
 	export GCLFAGS='-gcflags="all=-N -l"'
-	${MAKE} build
+	${MAKE} build-base
 
-build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
-	go build ${GCLFAGS} -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.Ver=$(VER)'" \
-					   -o bin/mizu_$(SUFFIX) mizu.go
+build:
+	export LDFLAGS_EXT='-s -w'
+	${MAKE} build-base
+
+build-base: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
+	go build ${GCLFAGS} -ldflags="${LDFLAGS_EXT} \
+					-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.Ver=$(VER)'" \
+					-o bin/mizu_$(SUFFIX) mizu.go
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
 build-all: ## Build for all supported platforms.

shared/models.go

@@ -15,6 +15,7 @@ type WebSocketMessageType string
 
 const (
 	WebSocketMessageTypeEntry         WebSocketMessageType = "entry"
+	WebSocketMessageTypeFullEntry     WebSocketMessageType = "fullEntry"
 	WebSocketMessageTypeTappedEntry   WebSocketMessageType = "tappedEntry"
 	WebSocketMessageTypeUpdateStatus  WebSocketMessageType = "status"
 	WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"

ui/src/components/Pages/TrafficPage/TrafficPage.tsx

@@ -122,6 +122,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus}) => {
         ws.current.onopen = () => {
             setWsConnection(WsConnectionStatus.Connected);
             ws.current.send(query);
+            ws.current.send("");
         }
         ws.current.onclose = () => {
             setWsConnection(WsConnectionStatus.Closed);