Refactor entries controller logic (#949)

* wip

* Update entries_controller.go and entries_provider.go

* Update entries_controller.go

* change entries provider into a struct + interface

* Update entries_provider.go

* Update entries_provider.go

.github/workflows/acceptance_tests.yml

@@ -43,7 +43,7 @@ jobs:
         with:
           status: ${{ job.status }}
           notification_title: 'Mizu {workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha} ${{ github.event.head_commit.message }}> ${{ github.event.head_commit.committer.name }} <${{ github.event.head_commit.committer.email }}>'
           footer: 'Linked Repo <{repo_url}|{repo}>'
           notify_when: 'failure'
         env:

.github/workflows/build.yml

@@ -45,7 +45,7 @@ jobs:
       
       - name: Check modified files
         id: modified_files
-        run: devops/check_modified_files.sh agent/ shared/ tap/ ui/ Dockerfile
+        run: devops/check_modified_files.sh agent/ shared/ tap/ ui/ ui-common/ Dockerfile
 
       - name: Set up Docker Buildx
         if: steps.modified_files.outputs.matched == 'true'

Dockerfile

@@ -1,12 +1,23 @@
 ARG BUILDARCH=amd64
 ARG TARGETARCH=amd64
 
+### Front-end common
+FROM node:16 AS front-end-common
+
+WORKDIR /app/ui-build
+COPY ui-common/package.json .
+COPY ui-common/package-lock.json .
+RUN npm i
+COPY ui-common .
+RUN npm pack
+
 ### Front-end
 FROM node:16 AS front-end
 
 WORKDIR /app/ui-build
 
 COPY ui/package.json ui/package-lock.json ./
+COPY --from=front-end-common ["/app/ui-build/up9-mizu-common-0.0.0.tgz", "."]
 RUN npm i
 COPY ui .
 RUN npm run build

agent/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/elastic"
+	"github.com/up9inc/mizu/agent/pkg/entries"
 	"github.com/up9inc/mizu/agent/pkg/middlewares"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/agent/pkg/oas"
@@ -371,4 +372,5 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 func initializeDependencies() {
 	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
 	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance() })
+	dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
 }

agent/pkg/controllers/entries_controller.go

@@ -1,25 +1,20 @@
 package controllers
 
 import (
-	"encoding/json"
 	"net/http"
 	"strconv"
-	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/app"
-	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/entries"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/agent/pkg/validation"
 
 	"github.com/gin-gonic/gin"
 
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
-func Error(c *gin.Context, err error) bool {
+func HandleEntriesError(c *gin.Context, err error) bool {
 	if err != nil {
 		logger.Log.Errorf("Error getting entry: %v", err)
 		_ = c.Error(err)
@@ -49,45 +44,18 @@ func GetEntries(c *gin.Context) {
 		entriesRequest.TimeoutMs = 3000
 	}
 
-	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
-		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
-		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, validationError)
-	}
-
-	response := &models.EntriesResponse{}
-	var dataSlice []interface{}
-
-	for _, row := range data {
-		var entry *tapApi.Entry
-		err = json.Unmarshal(row, &entry)
-		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{
-				"error":     true,
-				"type":      "error",
-				"autoClose": "5000",
-				"msg":       string(row),
-			})
-			return // exit
+	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
+	entries, metadata, err := entriesProvider.GetEntries(entriesRequest)
+	if !HandleEntriesError(c, err) {
+		baseEntries := make([]interface{}, 0)
+		for _, entry := range entries {
+			baseEntries = append(baseEntries, entry.Base)
 		}
-
-		extension := app.ExtensionsMap[entry.Protocol.Name]
-		base := extension.Dissector.Summarize(entry)
-
-		dataSlice = append(dataSlice, base)
+		c.JSON(http.StatusOK, models.EntriesResponse{
+			Data: baseEntries,
+			Meta: metadata,
+		})
 	}
-
-	var metadata *basenine.Metadata
-	err = json.Unmarshal(meta, &metadata)
-	if err != nil {
-		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
-	}
-
-	response.Data = dataSlice
-	response.Meta = metadata
-
-	c.JSON(http.StatusOK, response)
 }
 
 func GetEntry(c *gin.Context) {
@@ -102,54 +70,11 @@ func GetEntry(c *gin.Context) {
 	}
 
 	id, _ := strconv.Atoi(c.Param("id"))
-	var entry *tapApi.Entry
-	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
-	if Error(c, err) {
-		return // exit
-	}
-	err = json.Unmarshal(bytes, &entry)
-	if err != nil {
-		c.JSON(http.StatusNotFound, gin.H{
-			"error":     true,
-			"type":      "error",
-			"autoClose": "5000",
-			"msg":       string(bytes),
-		})
-		return // exit
-	}
 
-	extension := app.ExtensionsMap[entry.Protocol.Name]
-	base := extension.Dissector.Summarize(entry)
-	var representation []byte
-	representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
-	if err != nil {
-		c.JSON(http.StatusNotFound, gin.H{
-			"error":     true,
-			"type":      "error",
-			"autoClose": "5000",
-			"msg":       err.Error(),
-		})
-		return // exit
-	}
+	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
+	entry, err := entriesProvider.GetEntry(singleEntryRequest, id)
 
-	var rules []map[string]interface{}
-	var isRulesEnabled bool
-	if entry.Protocol.Name == "http" {
-		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
-		isRulesEnabled = _isRulesEnabled
-		inrec, _ := json.Marshal(rulesMatched)
-		if err := json.Unmarshal(inrec, &rules); err != nil {
-			logger.Log.Error(err)
-		}
+	if !HandleEntriesError(c, err) {
+		c.JSON(http.StatusOK, entry)
 	}
-
-	c.JSON(http.StatusOK, tapApi.EntryWrapper{
-		Protocol:       entry.Protocol,
-		Representation: string(representation),
-		Data:           entry,
-		Base:           base,
-		Rules:          rules,
-		IsRulesEnabled: isRulesEnabled,
-	})
 }

agent/pkg/dependency/type_names.go

@@ -5,4 +5,5 @@ type DependencyContainerType string
 const (
 	ServiceMapGeneratorDependency = "ServiceMapGeneratorDependency"
 	OasGeneratorDependency        = "OasGeneratorDependency"
+	EntriesProvider               = "EntriesProvider"
 )

agent/pkg/entries/entries_provider.go

@@ -0,0 +1,98 @@
+package entries
+
+import (
+	"encoding/json"
+	"time"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/app"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type EntriesProvider interface {
+	GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
+	GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId int) (*tapApi.EntryWrapper, error)
+}
+
+type BasenineEntriesProvider struct{}
+
+func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
+	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
+		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var dataSlice []*tapApi.EntryWrapper
+
+	for _, row := range data {
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		extension := app.ExtensionsMap[entry.Protocol.Name]
+		base := extension.Dissector.Summarize(entry)
+
+		dataSlice = append(dataSlice, &tapApi.EntryWrapper{
+			Protocol: entry.Protocol,
+			Data:     entry,
+			Base:     base,
+		})
+	}
+
+	var metadata *basenine.Metadata
+	err = json.Unmarshal(meta, &metadata)
+	if err != nil {
+		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+	}
+
+	return dataSlice, metadata, nil
+}
+
+func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId int) (*tapApi.EntryWrapper, error) {
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(bytes, &entry)
+	if err != nil {
+		return nil, err
+	}
+
+	extension := app.ExtensionsMap[entry.Protocol.Name]
+	base := extension.Dissector.Summarize(entry)
+	var representation []byte
+	representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
+	if err != nil {
+		return nil, err
+	}
+
+	var rules []map[string]interface{}
+	var isRulesEnabled bool
+	if entry.Protocol.Name == "http" {
+		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
+		isRulesEnabled = _isRulesEnabled
+		inrec, _ := json.Marshal(rulesMatched)
+		if err := json.Unmarshal(inrec, &rules); err != nil {
+			logger.Log.Error(err)
+		}
+	}
+
+	return &tapApi.EntryWrapper{
+		Protocol:       entry.Protocol,
+		Representation: string(representation),
+		Data:           entry,
+		Base:           base,
+		Rules:          rules,
+		IsRulesEnabled: isRulesEnabled,
+	}, nil
+}

ui-common/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@up9/mizu-common",
-  "version": "1.0.135",
+  "version": "0.0.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@up9/mizu-common",
-      "version": "1.0.135",
+      "version": "0.0.0",
       "license": "MIT",
       "dependencies": {
         "@craco/craco": "^6.4.3",

ui-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@up9/mizu-common",
-  "version": "1.0.144",
+  "version": "0.0.0",
   "description": "Made with create-react-library",
   "author": "",
   "license": "MIT",
@@ -90,4 +90,4 @@
   "files": [
     "dist"
   ]
-}
+}

ui-common/src/components/TrafficViewer/TrafficViewer.tsx

@@ -110,8 +110,27 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({ setAnalyzeStatus,
     isCloseWebSocket && closeWebSocket()
   }, [isCloseWebSocket])
 
+  useEffect(() => {
+    reopenConnection()
+  }, [webSocketUrl])
+
   const ws = useRef(null);
 
+  const openEmptyWebSocket = () => {
+    if (query) {
+      openWebSocket(`(${query}) and leftOff(-1)`, true);
+    } else {
+      openWebSocket(`leftOff(-1)`, true);
+    }
+  }
+
+  const closeWebSocket = () => {
+    if(ws?.current?.readyState === WebSocket.OPEN) {
+      ws.current.close();
+      return true;
+    }
+  }
+
   const listEntry = useRef(null);
   const openWebSocket = (query: string, resetEntries: boolean) => {
     if (resetEntries) {
@@ -153,12 +172,6 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({ setAnalyzeStatus,
     }, 500)
   }
 
-  const closeWebSocket = () => {
-    if (ws?.current?.readyState === WebSocket.OPEN) {
-      ws.current.close();
-    }
-  }
-
   if (ws.current) {
     ws.current.onmessage = (e) => {
       if (!e?.data) return;
@@ -216,8 +229,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({ setAnalyzeStatus,
   useEffect(() => {
     setTrafficViewerApiState({ ...trafficViewerApiProp, webSocket: { close: closeWebSocket } });
     (async () => {
-      openWebSocket("leftOff(-1)", true);
-      try {
+      try{
         const tapStatusResponse = await trafficViewerApiProp.tapStatus();
         setTappingStatus(tapStatusResponse);
         if (setAnalyzeStatus) {
@@ -232,19 +244,18 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({ setAnalyzeStatus,
   }, []);
 
   const toggleConnection = () => {
-    if (ws?.current?.readyState === WebSocket.OPEN) {
-      ws?.current?.close();
-    } else {
-      if (query) {
-        openWebSocket(`(${query}) and leftOff(-1)`, true);
-      } else {
-        openWebSocket(`leftOff(-1)`, true);
-      }
+    if(!closeWebSocket()) {
+      openEmptyWebSocket();
       scrollableRef.current.jumpToBottom();
       setIsSnappedToBottom(true);
     }
   }
 
+  const reopenConnection = async () => {
+    closeWebSocket()
+    openEmptyWebSocket();
+  }
+
   useEffect(() => {
     return () => {
       ws.current.close();

ui/package.json

@@ -13,7 +13,7 @@
     "@types/jest": "^26.0.22",
     "@types/node": "^12.20.10",
     "@uiw/react-textarea-code-editor": "^1.4.12",
-    "@up9/mizu-common": "1.0.144",
+    "@up9/mizu-common": "file:up9-mizu-common-0.0.0.tgz",
     "axios": "^0.25.0",
     "core-js": "^3.20.2",
     "craco-babel-loader": "^1.0.3",
@@ -75,4 +75,4 @@
       "last 1 safari version"
     ]
   }
-}
+}

ui/src/components/Pages/TrafficPage/TrafficPage.tsx

@@ -1,6 +1,6 @@
 import React, {useEffect, useState} from "react";
 import { Button } from "@material-ui/core";
-import Api,{getWebsocketUrl} from "../../../helpers/api";
+import Api, { MizuWebsocketURL } from "../../../helpers/api";
 import debounce from 'lodash/debounce';
 import {useSetRecoilState, useRecoilState} from "recoil";
 import {useCommonStyles} from "../../../helpers/commonStyle"
@@ -65,7 +65,7 @@ const trafficViewerApi = {...api}
 
   return (
   <>
-      <TrafficViewer setAnalyzeStatus={setAnalyzeStatus} webSocketUrl={getWebsocketUrl()} isCloseWebSocket={!openWebSocket}
+      <TrafficViewer setAnalyzeStatus={setAnalyzeStatus} webSocketUrl={MizuWebsocketURL} isCloseWebSocket={!openWebSocket}
                      trafficViewerApiProp={trafficViewerApi} actionButtons={actionButtons} isShowStatusBar={!openOasModal} isDemoBannerView={false}/>
   </>
   );

ui/src/helpers/api.js

@@ -3,13 +3,10 @@ import * as axios from "axios";
 export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL :
     window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`;
 
-export const FormValidationErrorType = "formError";
-
 const CancelToken = axios.CancelToken;
 
 const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`;
 
-let token = ""
 let client = null
 let source = null
 
@@ -24,8 +21,6 @@ export default class Api {
     }
 
     constructor() {
-        token = localStorage.getItem("token");
-
         client = this.getAxiosClient();
         source = null;
     }
@@ -125,20 +120,10 @@ export default class Api {
         return response.data;
     }
 
-    persistToken = (tk) => {
-        token = tk;
-        client = this.getAxiosClient();
-        localStorage.setItem('token', token);
-    }
-
     getAxiosClient = () => {
         const headers = {
             Accept: "application/json"
         }
-
-        if (token) {
-            headers['x-session-token'] = `${token}`; // we use `x-session-token` instead of `Authorization` because the latter is reserved by kubectl proxy, making mizu view not work
-        }
         return axios.create({
             baseURL: apiURL,
             timeout: 31000,
@@ -146,12 +131,3 @@ export default class Api {
         });
     }
 }
-
-export function getWebsocketUrl() {
-    let websocketUrl = MizuWebsocketURL;
-    if (token) {
-        websocketUrl += `/${token}`;
-    }
-
-    return websocketUrl;
-}