Fetch N number of records in M milliseconds timeout before streaming the records (#1056)

* Fetch N number of records in M milliseconds timeout before streaming the records

* Implement the functionality inside socket data streamer

* Reverse the `fetchData` slice

* #run_acceptance_tests

* Trying to fix the tests.
#run_acceptance_tests

* javascript compilation error.

* #run_acceptance_tests

* Name the method better

* Upgrade Basenine version to `v0.8.0`

* Fix some issues related to `Fetch`

* Upgrade the Basenine version in `Dockerfile` as well

* Remove underscore from the parameter name

* Parameterize fetch timeout ms

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
Co-authored-by: Roee Gadot <roee.gadot@up9.com>

.gitignore

@@ -52,4 +52,7 @@ tap/extensions/*/expect
 # UI folders to ignore
 **/node_modules/**
 **/dist/**
-*.editorconfig
+*.editorconfig
+
+# Ignore *.log files
+*.log

Dockerfile

@@ -94,8 +94,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
 RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
     chmod +x ./basenine_linux_"${GOARCH}" && \

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -57,13 +57,6 @@ export function rightOnHoverCheck(path, expectedText) {
     cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
 }
 
-export function checkThatAllEntriesShown() {
-    cy.get('#entries-length').then(number => {
-        if (number.text() === '1')
-            cy.get('[title="Fetch old records"]').click();
-    });
-}
-
 export function checkFilterByMethod(funcDict) {
     const {protocol, method, methodQuery, summary, summaryQuery} = funcDict;
     const summaryDict = getSummaryDict(summary, summaryQuery);
@@ -76,12 +69,7 @@ export function checkFilterByMethod(funcDict) {
         cy.get('[type="submit"]').click();
         cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
 
-        cy.get('#entries-length').then(number => {
-            // if the entries list isn't expanded it expands here
-            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
-                cy.get('[title="Fetch old records"]').click();
-
-            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
+            cy.get('#entries-length').should('not.have.text', '0').then(() => {
                 cy.get(`#list [id]`).then(elements => {
                     const listElmWithIdAttr = Object.values(elements);
                     let doneCheckOnFirst = false;
@@ -108,7 +96,6 @@ export function checkFilterByMethod(funcDict) {
                 });
             });
         });
-    });
 }
 
 export function getEntryId(id) {

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,5 +1,4 @@
 import {
-    checkThatAllEntriesShown,
     isValueExistsInElement,
     resizeToHugeMizu,
 } from "../testHelpers/TrafficHelper";
@@ -12,13 +11,14 @@ checkEntries();
 
 function checkEntries() {
     it('checking all entries', function () {
-        checkThatAllEntriesShown();
-        resizeToHugeMizu();
+        cy.get('#entries-length').should('not.have.text', '0').then(() => {
+            resizeToHugeMizu();
 
-        cy.get('#list [id^=entry]').each(entryElement => {
-            entryElement.click();
-            cy.get('#tbody-Headers').should('be.visible');
-            isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+            cy.get('#list [id^=entry]').each(entryElement => {
+                entryElement.click();
+                cy.get('#tbody-Headers').should('be.visible');
+                isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+            });
         });
     });
 }

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -218,12 +218,8 @@ function checkFilter(filterDetails) {
                 checkRightSideResponseBody();
             });
 
-            cy.get('[title="Fetch old records"]').click();
             resizeToHugeMizu();
 
-            // waiting for the entries number to load
-            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
             // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
             cy.get(`#list [id^=entry]`).each(elem => {
                 const element = elem[0];

acceptanceTests/setup.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
+set -e
 
 PREFIX=$HOME/local/bin
 VERSION=v1.22.0
+TUNNEL_LOG="tunnel.log"
+PROXY_LOG="proxy.log"
 
 echo "Attempting to install minikube and assorted tools to $PREFIX"
 
@@ -11,7 +14,7 @@ if ! [ -x "$(command -v kubectl)" ]; then
   chmod +x kubectl
   mv kubectl "$PREFIX"
 else
-  echo "kubetcl is already installed"
+  echo "kubectl is already installed"
 fi
 
 if ! [ -x "$(command -v minikube)" ]; then
@@ -27,35 +30,39 @@ echo "Starting minikube..."
 minikube start
 
 echo "Creating mizu tests namespaces"
-kubectl create namespace mizu-tests
-kubectl create namespace mizu-tests2
+kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create namespace mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating httpbin deployments"
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
-kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating redis deployment"
-kubectl create deployment redis --image=redis -n mizu-tests
+kubectl create deployment redis --image=redis -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating rabbitmq deployment"
-kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests
+kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating httpbin services"
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
-kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating redis service"
-kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
+kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating rabbitmq service"
-kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests
+kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
+# TODO: need to understand how to fail if address already in use
 echo "Starting proxy"
-kubectl proxy --port=8080 &
+rm -f ${PROXY_LOG}
+kubectl proxy --port=8080 > ${PROXY_LOG} &
+PID1=$!
+echo "kubectl proxy process id is ${PID1} and log of proxy in ${PROXY_LOG}"
 
 if [[ -z "${CI}" ]]; then
   echo "Setting env var of mizu ci image"
@@ -71,5 +78,9 @@ minikube image load "${MIZU_CI_IMAGE}"
 echo "Build cli"
 cd cli && make build GIT_BRANCH=ci SUFFIX=ci
 
+# TODO: need to understand how to fail if password is asked (sudo)
 echo "Starting tunnel"
-minikube tunnel &
+rm -f ${TUNNEL_LOG}
+minikube tunnel > ${TUNNEL_LOG} &
+PID2=$!
+echo "Minikube tunnel process id is ${PID2} and log of tunnel in ${TUNNEL_LOG}"

agent/go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v1.0.0
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607
+	github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337
 	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0

agent/go.sum

@@ -683,8 +683,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk=
-github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337 h1:eRXRZnojrZyhbiSuGHl0EPvFtWvx1ZMrsY/bSoBzYNE=
+github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=

agent/pkg/api/socket_data_streamer.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
@@ -38,6 +39,11 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 		entryStreamerSocketConnector.SendToastError(socketId, err)
 	}
 
+	leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
+	if err != nil {
+		logger.Log.Errorf("Fetch error: %v", err.Error())
+	}
+
 	handleDataChannel := func(c *basenine.Connection, data chan []byte) {
 		for {
 			bytes := <-data
@@ -79,7 +85,7 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 	go handleDataChannel(connection, data)
 	go handleMetaChannel(connection, meta)
 
-	if err = connection.Query(query, data, meta); err != nil {
+	if err = connection.Query(leftOff, query, data, meta); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 		entryStreamerSocketConnector.CleanupSocket(socketId)
 		return err
@@ -94,3 +100,61 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 
 	return nil
 }
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
+	if params.Fetch <= 0 {
+		leftOff = params.LeftOff
+		return
+	}
+
+	var data [][]byte
+	var firstMeta []byte
+	var lastMeta []byte
+	data, firstMeta, lastMeta, err = basenine.Fetch(
+		shared.BasenineHost,
+		shared.BaseninePort,
+		params.LeftOff,
+		-1,
+		params.Query,
+		params.Fetch,
+		time.Duration(params.TimeoutMs)*time.Millisecond,
+	)
+	if err != nil {
+		return
+	}
+
+	var firstMetadata *basenine.Metadata
+	err = json.Unmarshal(firstMeta, &firstMetadata)
+	if err != nil {
+		return
+	}
+	leftOff = firstMetadata.LeftOff
+
+	var lastMetadata *basenine.Metadata
+	err = json.Unmarshal(lastMeta, &lastMetadata)
+	if err != nil {
+		return
+	}
+	connector.SendMetadata(socketId, lastMetadata)
+
+	data = e.reverseBytesSlice(data)
+	for _, row := range data {
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
+		if err != nil {
+			break
+		}
+
+		connector.SendEntry(socketId, entry, params)
+	}
+	return
+}
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
+	for i := len(arr) - 1; i >= 0; i-- {
+		newArr = append(newArr, arr[i])
+	}
+	return newArr
+}

agent/pkg/api/socket_routes.go

@@ -34,8 +34,11 @@ type SocketConnection struct {
 }
 
 type WebSocketParams struct {
+	LeftOff           string `json:"leftOff"`
 	Query             string `json:"query"`
 	EnableFullEntries bool   `json:"enableFullEntries"`
+	Fetch             int    `json:"fetch"`
+	TimeoutMs         int    `json:"timeoutMs"`
 }
 
 var (

agent/pkg/entries/entries_provider.go

@@ -22,7 +22,7 @@ type EntriesProvider interface {
 type BasenineEntriesProvider struct{}
 
 func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
-	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+	data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
 		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
 		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
 	if err != nil {
@@ -49,7 +49,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 	}
 
 	var metadata *basenine.Metadata
-	err = json.Unmarshal(meta, &metadata)
+	err = json.Unmarshal(lastMeta, &metadata)
 	if err != nil {
 		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
 	}

agent/pkg/har/utils.go

@@ -124,8 +124,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	text := postData["text"]
 	postDataText := ""
@@ -177,8 +177,8 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	encoding := content["encoding"]
 	text := content["text"]

agent/pkg/oas/oas_generator.go

@@ -104,7 +104,7 @@ func (g *defaultOasGenerator) runGenerator() {
 	g.dbMutex.Lock()
 	defer g.dbMutex.Unlock()
 	logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
-	if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil {
+	if err := g.dbConn.Query("", g.entriesQuery, dataChan, metaChan); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 	}
 

agent/pkg/up9/main.go

@@ -327,7 +327,7 @@ BasenineReconnect:
 	go handleMetaChannel(&wg, connection, meta)
 	wg.Add(2)
 
-	if err = connection.Query(query, data, meta); err != nil {
+	if err = connection.Query("", query, data, meta); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 		connection.Close()
 		time.Sleep(shared.BasenineReconnectInterval * time.Second)

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect9/http/\* expect

tap/extensions/http/main.go

@@ -401,8 +401,8 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	text := postData["text"]
 	if text != nil {
@@ -483,8 +483,8 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	encoding := content["encoding"]
 	text := content["text"]

ui-common/example/README.md

@@ -1,5 +0,0 @@
-This example was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
-
-It is linked to the liraz-test package in the parent directory for development purposes.
-
-You can run `npm install` and then `npm start` to test your package.

ui-common/example/craco.config.js

@@ -1,13 +0,0 @@
-module.exports = {
-  webpack: {
-      configure: (webpackConfig) => {
-          const instanceOfMiniCssExtractPlugin = webpackConfig.plugins.find(
-              (plugin) => plugin.options && plugin.options.ignoreOrder != null,
-          );
-          if(instanceOfMiniCssExtractPlugin)
-              instanceOfMiniCssExtractPlugin.options.ignoreOrder = true;
-
-          return webpackConfig;
-      }
-  }
-}

ui-common/example/package.json

@@ -1,48 +0,0 @@
-{
-  "name": "@up9/mizu-common-example",
-  "homepage": ".",
-  "version": "0.0.0",
-  "private": true,
-  "scripts": {
-    "start": "craco start",
-    "comment-start": "node ../node_modules/react-scripts/bin/react-scripts.js start",
-    "build": "node ../node_modules/react-scripts/bin/react-scripts.js build",
-    "test": "node ../node_modules/react-scripts/bin/react-scripts.js test",
-    "eject": "node ../node_modules/react-scripts/bin/react-scripts.js eject"
-  },
-  "dependencies": {
-    "@testing-library/jest-dom": "file:../node_modules/@testing-library/jest-dom",
-    "@testing-library/react": "file:../node_modules/@testing-library/react",
-    "@testing-library/user-event": "file:../node_modules/@testing-library/user-event",
-    "@types/jest": "file:../node_modules/@types/jest",
-    "@types/node": "file:../node_modules/@types/node",
-    "@types/react": "file:../node_modules/@types/react",
-    "@types/react-dom": "file:../node_modules/@types/react-dom",
-    "@up9/mizu-common": "file:..",
-    "react": "file:../node_modules/react",
-    "react-dom": "file:../node_modules/react-dom"
-  },
-  "devDependencies": {
-    "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
-    "@craco/craco": "^6.4.3",
-    "eslint": "^7.11.0",
-    "node-sass": "^6.0.0",
-    "recoil": "^0.5.2",
-    "react-scripts": "^4.0.3"
-  },
-  "eslintConfig": {
-    "extends": "react-app"
-  },
-  "browserslist": {
-    "production": [
-      ">0.2%",
-      "not dead",
-      "not op_mini all"
-    ],
-    "development": [
-      "last 1 chrome version",
-      "last 1 firefox version",
-      "last 1 safari version"
-    ]
-  }
-}

ui-common/example/public/index.html

@@ -1,48 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico" />
-    <meta
-      name="viewport"
-      content="width=device-width, initial-scale=1, shrink-to-fit=no"
-    />
-    <meta name="theme-color" content="#000000" />
-
-    <!--
-      manifest.json provides metadata used when your web app is added to the
-      homescreen on Android. See https://developers.google.com/web/fundamentals/engage-and-retain/web-app-manifest/
-    -->
-    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
-
-    <!--
-      Notice the use of %PUBLIC_URL% in the tags above.
-      It will be replaced with the URL of the `public` folder during the build.
-      Only files inside the `public` folder can be referenced from the HTML.
-
-      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
-      work correctly both with client-side routing and a non-root public URL.
-      Learn how to configure a non-root public URL by running `npm run build`.
-    -->
-    <title>@up9/mizu-common</title>
-  </head>
-
-  <body>
-    <noscript>
-      You need to enable JavaScript to run this app.
-    </noscript>
-
-    <div id="root"></div>
-
-    <!--
-      This HTML file is a template.
-      If you open it directly in the browser, you will see an empty page.
-
-      You can add webfonts, meta tags, or analytics to this file.
-      The build step will place the bundled scripts into the <body> tag.
-
-      To begin the development, run `npm start` or `yarn start`.
-      To create a production bundle, use `npm run build` or `yarn build`.
-    -->
-  </body>
-</html>

ui-common/example/public/manifest.json

@@ -1,15 +0,0 @@
-{
-  "short_name": "@up9/mizu-common",
-  "name": "@up9/mizu-common",
-  "icons": [
-    {
-      "src": "favicon.ico",
-      "sizes": "64x64 32x32 24x24 16x16",
-      "type": "image/x-icon"
-    }
-  ],
-  "start_url": ".",
-  "display": "standalone",
-  "theme_color": "#000000",
-  "background_color": "#ffffff"
-}

ui-common/example/src/App.test.tsx

@@ -1,9 +0,0 @@
-import React from 'react'
-import ReactDOM from 'react-dom'
-import App from './App'
-
-it('renders without crashing', () => {
-  const div = document.createElement('div')
-  ReactDOM.render(<App />, div)
-  ReactDOM.unmountComponentAtNode(div)
-})

ui-common/example/src/App.tsx

@@ -1,24 +0,0 @@
-import TrafficViewer,{useWS, DEFAULT_QUERY, OasModal} from '@up9/mizu-common';
-import "@up9/mizu-common/dist/index.css"
-import {useEffect} from 'react';
-import Api, {getWebsocketUrl} from "./api";
-
-const api = Api.getInstance()
-
-const App = () => {
-  const {message,error,isOpen, openSocket, closeSocket, sendQueryWhenWsOpen} = useWS(getWebsocketUrl())
-  const trafficViewerApi = {...api, webSocket:{open : openSocket, close: closeSocket, sendQueryWhenWsOpen: sendQueryWhenWsOpen}}
-  sendQueryWhenWsOpen(DEFAULT_QUERY);
-
-  useEffect(() => {
-    return () =>{
-      closeSocket()
-    }
-  },[])
-
-  return <>
-
-  </>
-}
-
-export default App

ui-common/example/src/api.js

@@ -1,119 +0,0 @@
-import * as axios from "axios";
-
-export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL :
-    window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`;
-
-export const FormValidationErrorType = "formError";
-
-const CancelToken = axios.CancelToken;
-
-const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`;
-
-let token = null
-let client = null
-let source = null
-
-export default class Api {
-    static instance;
-
-    static getInstance() {
-        if (!Api.instance) {
-            Api.instance = new Api();
-        }
-        return Api.instance;
-    }
-
-    constructor() {
-        token = localStorage.getItem("token");
-
-        client = this.getAxiosClient();
-        source = null;
-    }
-
-    tapStatus = async () => {
-        const response = await client.get("/status/tap");
-        return response.data;
-    }
-
-
-    analyzeStatus = async () => {
-        const response = await client.get("/status/analyze");
-        return response.data;
-    }
-
-    getEntry = async (id, query) => {
-        const response = await client.get(`/entries/${id}?query=${query}`);
-        return response.data;
-    }
-
-    fetchEntries = async (leftOff, direction, query, limit, timeoutMs) => {
-        const response = await client.get(`/entries/?leftOff=${leftOff}&direction=${direction}&query=${query}&limit=${limit}&timeoutMs=${timeoutMs}`).catch(function (thrown) {
-            console.error(thrown.message);
-            return {};
-        });
-        return response.data;
-    }
-
-    getOasServices = async () => {
-        const response = await client.get("/oas");
-        return response.data;
-    }
-
-    getOasByService = async (selectedService) => {
-        const response = await client.get(`/oas/${selectedService}`);
-        return response.data;
-    }
-
-    validateQuery = async (query) => {
-        if (source) {
-            source.cancel();
-        }
-        source = CancelToken.source();
-
-        const form = new FormData();
-        form.append('query', query)
-        const response = await client.post(`/query/validate`, form, {
-            cancelToken: source.token
-        }).catch(function (thrown) {
-            if (!axios.isCancel(thrown)) {
-                console.error('Validate error', thrown.message);
-            }
-        });
-
-        if (!response) {
-            return null;
-        }
-
-        return response.data;
-    }
-
-    persistToken = (tk) => {
-        token = tk;
-        client = this.getAxiosClient();
-        localStorage.setItem('token', token);
-    }
-
-    getAxiosClient = () => {
-        const headers = {
-            Accept: "application/json"
-        }
-
-        if (token) {
-            headers['x-session-token'] = `${token}`; // we use `x-session-token` instead of `Authorization` because the latter is reserved by kubectl proxy, making mizu view not work
-        }
-        return axios.create({
-            baseURL: apiURL,
-            timeout: 31000,
-            headers
-        });
-    }
-}
-
-export function getWebsocketUrl(){
-    let websocketUrl = MizuWebsocketURL;
-    if (token) {
-      websocketUrl += `/${token}`;
-    }
-
-    return websocketUrl;
-}

ui-common/example/src/index.css

@@ -1,14 +0,0 @@
-body {
-  margin: 0;
-  padding: 0;
-  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
-    'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
-    sans-serif;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-code {
-  font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
-    monospace;
-}

ui-common/example/src/index.tsx

@@ -1,7 +0,0 @@
-import './index.css'
-
-import ReactDOM from 'react-dom'
-import App from './App'
-
-
-ReactDOM.render(<App />, document.getElementById('root'))

ui-common/example/src/react-app-env.d.ts

@@ -1 +0,0 @@
-/// <reference types="react-scripts" />

ui-common/example/src/setupTests.ts

@@ -1,5 +0,0 @@
-// jest-dom adds custom jest matchers for asserting on DOM nodes.
-// allows you to do things like:
-// expect(element).toHaveTextContent(/react/i)
-// learn more: https://github.com/testing-library/jest-dom
-import '@testing-library/jest-dom/extend-expect';

ui-common/example/tsconfig.json

@@ -1,41 +0,0 @@
-{
-  "compilerOptions": {
-    "outDir": "dist",
-    "module": "esnext",
-    "lib": [
-      "dom",
-      "esnext"
-    ],
-    "moduleResolution": "node",
-    "jsx": "react-jsx",
-    "sourceMap": true,
-    "declaration": true,
-    "esModuleInterop": true,
-    "noImplicitReturns": true,
-    "noImplicitThis": true,
-    "noImplicitAny": true,
-    "strictNullChecks": false,
-    "suppressImplicitAnyIndexErrors": true,
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "allowSyntheticDefaultImports": true,
-    "target": "es5",
-    "allowJs": true,
-    "skipLibCheck": true,
-  
-    "strict": true,
-    "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true,
-    "isolatedModules": true,
-    "noEmit": true,
-    "noFallthroughCasesInSwitch": true
-  },
-  "include": [
-    "src"
-  ],
-  "exclude": [
-    "node_modules",
-    "build",
-    "exmaple"
-  ]
-}

ui-common/src/components/OasModal/OasModal.module.sass

@@ -1,6 +1,13 @@
 @import '../../variables.module.scss'
 
-.boxContainer 
+.closeIcon
+    cursor: pointer
+    user-select: none
+    margin-top: -12px
+    margin-right: -12px
+    float: right
+
+.boxContainer
     display: flex
     justifyContent: space-between
     padding: 10px
@@ -17,7 +24,7 @@
     width: 43px
 
 .title
-    color:#494677
+    color: #494677
     font-family: Source Sans Pro,Lucida Grande,Tahoma,sans-serif
     font-size: 28px
     font-weight: 600
@@ -37,4 +44,3 @@
 
 .root
     width: 100%
-

ui-common/src/components/OasModal/OasModal.tsx

@@ -1,5 +1,5 @@
-import { Box, Fade, FormControl, MenuItem, Modal, Backdrop } from "@material-ui/core";
-import { useEffect, useState } from "react";
+import { Box, Fade, FormControl, Modal, Backdrop } from "@material-ui/core";
+import { useCallback, useEffect, useState } from "react";
 import { RedocStandalone } from "redoc";
 import closeIcon from "assets/closeIcon.svg";
 import { toast } from 'react-toastify';
@@ -30,10 +30,10 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
   const [oasServices, setOasServices] = useState([] as string[])
   const [selectedServiceName, setSelectedServiceName] = useState("");
   const [selectedServiceSpec, setSelectedServiceSpec] = useState(null);
-  
-  const classes = {root: style.root}
 
-  const onSelectedOASService = async (selectedService) => {
+  const classes = { root: style.root }
+
+  const onSelectedOASService = useCallback (async (selectedService) => {
     if (oasServices.length === 0) {
       setSelectedServiceSpec(null);
       setSelectedServiceName("");
@@ -49,7 +49,8 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
       toast.error("Error occurred while fetching service OAS spec", { containerId: TOAST_CONTAINER_ID });
       console.error(e);
     }
-  };
+    // eslint-disable-next-line
+  },[oasServices]);
 
   useEffect(() => {
     (async () => {
@@ -60,11 +61,13 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
         console.error(e);
       }
     })();
+    // eslint-disable-next-line
   }, [openModal]);
 
+
   useEffect(() => {
     onSelectedOASService(null);
-  }, [oasServices])
+  }, [oasServices, onSelectedOASService])
 
   return (
     <Modal
@@ -80,18 +83,17 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
     >
       <Fade in={openModal}>
         <Box sx={modalStyle}>
+          <img src={closeIcon} alt="close" onClick={handleCloseModal} className={style.closeIcon} />
           <div className={style.boxContainer}>
             <div className={style.selectHeader}>
               <div><img src={openApiLogo} alt="openAPI" className={style.openApilogo} /></div>
               <div className={style.title}>Service Catalog</div>
             </div>
-            <div style={{ cursor: "pointer" }}>
-              <img src={closeIcon} alt="close" onClick={handleCloseModal} />
-            </div>
           </div>
+
           <div className={style.selectContainer} >
             <FormControl classes={classes}>
-              <SearchableDropdown 
+              <SearchableDropdown
                 options={oasServices}
                 selectedValues={selectedServiceName}
                 onChange={onSelectedOASService}

ui-common/src/components/OasModal/assets/closeIcon.svg

@@ -1,4 +1,4 @@
-<svg width="22" height="22" viewBox="0 0 22 22" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M20.5 11C20.5 16.2467 16.2467 20.5 11 20.5C5.75329 20.5 1.5 16.2467 1.5 11C1.5 5.75329 5.75329 1.5 11 1.5C16.2467 1.5 20.5 5.75329 20.5 11Z" stroke="#2B3560"/>
-<path d="M14.4762 9.05338L13.1448 7.7219L11.1528 9.71382L9.16091 7.7219L7.82943 9.05338L9.82135 11.0453L7.83226 13.0344L9.16374 14.3659L11.1528 12.3768L13.1419 14.3659L14.4734 13.0344L12.4843 11.0453L14.4762 9.05338Z" fill="#627EF7"/>
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M18.591 9.99997C18.591 14.7446 14.7447 18.5909 10.0001 18.5909C5.25546 18.5909 1.40918 14.7446 1.40918 9.99997C1.40918 5.25534 5.25546 1.40906 10.0001 1.40906C14.7447 1.40906 18.591 5.25534 18.591 9.99997Z" fill="#E9EBF8" stroke="#BCCEFD"/>
+<path d="M13.1604 8.23038L11.95 7.01994L10.1392 8.83078L8.32832 7.01994L7.11789 8.23038L8.92872 10.0412L7.12046 11.8495L8.33089 13.0599L10.1392 11.2517L11.9474 13.0599L13.1579 11.8495L11.3496 10.0412L13.1604 8.23038Z" fill="#205CF5"/>
 </svg>

ui-common/src/components/ServiceMapModal/ServiceMapModal.module.sass

@@ -56,3 +56,9 @@
 .separtorLine
   margin-top: 10px
   border: 1px solid #E9EBF8
+
+.closeIcon
+  cursor: pointer
+  user-select: none
+  margin-top: -15px
+  margin-right: 5px

ui-common/src/components/ServiceMapModal/ServiceMapModal.tsx

@@ -203,7 +203,7 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
                                 >
                                     Refresh
                                 </Button>
-                                <img src={closeIcon} alt="close" onClick={() => onClose()} style={{ cursor: "pointer", userSelect: "none" }}></img>
+                                <img src={closeIcon} alt="close" onClick={() => onClose()} className={styles.closeIcon}></img>
                             </div>
                             {isLoading && <div className={spinnerStyle.spinnerContainer}>
                                 <img alt="spinner" src={spinnerImg} style={{ height: 50 }} />

ui-common/src/components/TrafficViewer/EntriesList.tsx

@@ -23,7 +23,7 @@ interface EntriesListProps {
   setIsSnappedToBottom: any;
   noMoreDataTop: boolean;
   setNoMoreDataTop: (flag: boolean) => void;
-  openWebSocket: (query: string, resetEntries: boolean) => void;
+  openWebSocket: (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => void;
   scrollableRef: any;
   ws: any;
 }
@@ -195,11 +195,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({
                 className={`${styles.btnLive} ${isSnappedToBottom && !isWsConnectionClosed ? styles.hideButton : styles.showButton}`}
                 onClick={(_) => {
                   if (isWsConnectionClosed) {
-                    if (query) {
-                      openWebSocket(`(${query}) and leftOff("${leftOffBottom}")`, false);
-                    } else {
-                      openWebSocket(`leftOff("${leftOffBottom}")`, false);
-                    }
+                    openWebSocket(leftOffBottom, query, false, 0, 0);
                   }
                   scrollableRef.current.jumpToBottom();
                   setIsSnappedToBottom(true);

ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.tsx

@@ -274,25 +274,21 @@ export const EntryItem: React.FC<EntryProps> = ({entry, style, headingMode, name
                     >
                         <img
                             src={outgoingIcon}
-                            alt="Ingoing traffic"
-                            title="Ingoing"
+                            alt="Outgoing traffic"
+                            title="Outgoing"
                         />
                     </Queryable>
                     :
                     <Queryable
-                            query={`outgoing == true`}
+                            query={`outgoing == false`}
                             displayIconOnMouseOver={true}
                             flipped={true}
                             iconStyle={{marginTop: "28px"}}
                     >
                         <img
                             src={ingoingIcon}
-                            alt="Outgoing traffic"
-                            title="Outgoing"
-                            onClick={() => {
-                                const query = `outgoing == false`;
-                                setQuery(queryState ? `${queryState} and ${query}` : query);
-                            }}
+                            alt="Ingoing traffic"
+                            title="Ingoing"
                         />
                     </Queryable>
                 }

ui-common/src/components/TrafficViewer/TrafficViewer.tsx

@@ -20,7 +20,7 @@ import {StatusBar} from "../UI/StatusBar";
 import tappingStatusAtom from "../../recoil/tappingStatus/atom";
 import {TOAST_CONTAINER_ID} from "../../configs/Consts";
 import leftOffTopAtom from "../../recoil/leftOffTop";
-import { DEFAULT_QUERY } from '../../hooks/useWS';
+import { DEFAULT_LEFTOFF, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS } from '../../hooks/useWS';
 
 const useLayoutStyles = makeStyles(() => ({
   details: {
@@ -114,11 +114,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
   const ws = useRef(null);
 
   const openEmptyWebSocket = () => {
-    if (query) {
-      openWebSocket(`(${query}) and ${DEFAULT_QUERY}`, true);
-    } else {
-      openWebSocket(DEFAULT_QUERY, true);
-    }
+    openWebSocket(DEFAULT_LEFTOFF, query, true, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS);
   }
 
   const closeWebSocket = () => {
@@ -129,7 +125,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
   }
 
   const listEntry = useRef(null);
-  const openWebSocket = (query: string, resetEntries: boolean) => {
+  const openWebSocket = (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => {
     if (resetEntries) {
       setFocusedEntryId(null);
       setEntries([]);
@@ -138,7 +134,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
     }
     try {
       ws.current = new WebSocket(webSocketUrl);
-      sendQueryWhenWsOpen(query);
+      sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs);
 
       ws.current.onopen = () => {
         setWsReadyState(ws?.current?.readyState);
@@ -157,12 +153,18 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
     }
   }
 
-  const sendQueryWhenWsOpen = (query) => {
+  const sendQueryWhenWsOpen = (leftOff: string, query: string, fetch: number, fetchTimeoutMs: number) => {
     setTimeout(() => {
       if (ws?.current?.readyState === WebSocket.OPEN) {
-        ws.current.send(JSON.stringify({"query": query, "enableFullEntries": false}));
+        ws.current.send(JSON.stringify({
+          "leftOff": leftOff,
+          "query": query,
+          "enableFullEntries": false,
+          "fetch": fetch,
+          "timeoutMs": fetchTimeoutMs
+        }));
       } else {
-        sendQueryWhenWsOpen(query);
+        sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs);
       }
     }, 500)
   }
@@ -240,7 +242,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
 
   return (
     <div className={TrafficViewerStyles.TrafficPage}>
-      {tappingStatus && isShowStatusBar && <StatusBar isDemoBannerView={isDemoBannerView}/>}
+      {tappingStatus && isShowStatusBar && <StatusBar disabled={ws?.current?.readyState !== WebSocket.OPEN} isDemoBannerView={isDemoBannerView}/>}
       <div className={TrafficViewerStyles.TrafficPageHeader}>
         <div className={TrafficViewerStyles.TrafficPageStreamStatus}>
           <img className={TrafficViewerStyles.playPauseIcon}

ui-common/src/components/UI/StatusBar.tsx

@@ -5,6 +5,7 @@ import failIcon from 'assets/failed.svg';
 import successIcon from 'assets/success.svg';
 import {useRecoilValue} from "recoil";
 import tappingStatusAtom, {tappingStatusDetails} from "../../recoil/tappingStatus";
+import Tooltip from "./Tooltip";
 
 const pluralize = (noun: string, amount: number) => {
     return `${noun}${amount !== 1 ? 's' : ''}`
@@ -12,20 +13,22 @@ const pluralize = (noun: string, amount: number) => {
 
 interface StatusBarProps {
    isDemoBannerView: boolean;
+   disabled?: boolean;
   }
 
-export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView}) => {
+export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView, disabled}) => {
     const tappingStatus = useRecoilValue(tappingStatusAtom);
     const [expandedBar, setExpandedBar] = useState(false);
     const {uniqueNamespaces, amountOfPods, amountOfTappedPods, amountOfUntappedPods} = useRecoilValue(tappingStatusDetails);
-    return <div className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
+    return <div style={{opacity: disabled ? 0.4 : 1}} className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar && !disabled ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
         <div className={style.podsCount}>
         {tappingStatus.some(pod => !pod.isTapped) && <img src={warningIcon} alt="warning"/>}
+          {disabled &&  <Tooltip title={"Tapping status is not updated when streaming is paused"} isSimple><img src={warningIcon} alt="warning"/></Tooltip>}
             <span className={style.podsCountText} data-cy="podsCountText">
                 {`Tapping ${amountOfUntappedPods > 0 ? amountOfTappedPods + " / " + amountOfPods : amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}
             </span>
         </div>
-        {expandedBar && <div style={{marginTop: 20}}>
+        {expandedBar && !disabled && <div style={{marginTop: 20}}>
             <table>
                 <thead>
                     <tr>

ui-common/src/hooks/useWS.tsx

@@ -7,7 +7,9 @@ enum WebSocketReadyState {
   CLOSED
 }
 
-export const DEFAULT_QUERY = `leftOff("latest")`;
+export const DEFAULT_LEFTOFF = `latest`;
+export const DEFAULT_FETCH = 50;
+export const DEFAULT_FETCH_TIMEOUT_MS = 3000;
 
 const useWS = (wsUrl: string) => {
   const [message, setMessage] = useState(null);

ui-common/src/index.tsx

@@ -1,11 +1,11 @@
 import TrafficViewer from './components/TrafficViewer/TrafficViewer';
 import * as UI from "./components/UI"
 import { StatusBar } from './components/UI';
-import useWS, { DEFAULT_QUERY } from './hooks/useWS';
+import useWS, { DEFAULT_LEFTOFF } from './hooks/useWS';
 import { AnalyzeButton } from "./components/AnalyzeButton/AnalyzeButton"
 import OasModal from './components/OasModal/OasModal';
 import { ServiceMapModal } from './components/ServiceMapModal/ServiceMapModal';
 
 export { UI, AnalyzeButton, StatusBar, OasModal, ServiceMapModal }
-export { useWS, DEFAULT_QUERY }
+export { useWS, DEFAULT_LEFTOFF }
 export default TrafficViewer;