Compare commits

...

4 Commits

Author SHA1 Message Date
M. Mert Yıldıran
eef0ee8023 Fetch N number of records in M milliseconds timeout before streaming the records (#1056)
* Fetch N number of records in M milliseconds timeout before streaming the records

* Implement the functionality inside socket data streamer

* Reverse the `fetchData` slice

* #run_acceptance_tests

* Trying to fix the tests.
#run_acceptance_tests

* javascript compilation error.

* #run_acceptance_tests

* Name the method better

* Upgrade Basenine version to `v0.8.0`

* Fix some issues related to `Fetch`

* Upgrade the Basenine version in `Dockerfile` as well

* Remove underscore from the parameter name

* Parameterize fetch timeout ms

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
Co-authored-by: Roee Gadot <roee.gadot@up9.com>
2022-05-09 19:11:27 +03:00
RoyUP9
4c0aeb8146 Fixed default mime type to empty string instead of text/html (#1071) 2022-05-08 16:52:50 +03:00
lirazyehezkel
0dc0459dff Disable tapping status when streaming is paused (#1070) 2022-05-08 15:45:33 +03:00
lirazyehezkel
81f06003d5 Outgoing filter remove redundant code (#1069) 2022-05-08 14:58:13 +03:00
20 changed files with 121 additions and 72 deletions

View File

@@ -94,8 +94,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
-X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
# Download Basenine executable, verify the sha1sum
ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
chmod +x ./basenine_linux_"${GOARCH}" && \

View File

@@ -57,13 +57,6 @@ export function rightOnHoverCheck(path, expectedText) {
cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
}
export function checkThatAllEntriesShown() {
cy.get('#entries-length').then(number => {
if (number.text() === '1')
cy.get('[title="Fetch old records"]').click();
});
}
export function checkFilterByMethod(funcDict) {
const {protocol, method, methodQuery, summary, summaryQuery} = funcDict;
const summaryDict = getSummaryDict(summary, summaryQuery);
@@ -76,12 +69,7 @@ export function checkFilterByMethod(funcDict) {
cy.get('[type="submit"]').click();
cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
cy.get('#entries-length').then(number => {
// if the entries list isn't expanded it expands here
if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
cy.get('[title="Fetch old records"]').click();
cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
cy.get('#entries-length').should('not.have.text', '0').then(() => {
cy.get(`#list [id]`).then(elements => {
const listElmWithIdAttr = Object.values(elements);
let doneCheckOnFirst = false;
@@ -108,7 +96,6 @@ export function checkFilterByMethod(funcDict) {
});
});
});
});
}
export function getEntryId(id) {

View File

@@ -1,5 +1,4 @@
import {
checkThatAllEntriesShown,
isValueExistsInElement,
resizeToHugeMizu,
} from "../testHelpers/TrafficHelper";
@@ -12,13 +11,14 @@ checkEntries();
function checkEntries() {
it('checking all entries', function () {
checkThatAllEntriesShown();
resizeToHugeMizu();
cy.get('#entries-length').should('not.have.text', '0').then(() => {
resizeToHugeMizu();
cy.get('#list [id^=entry]').each(entryElement => {
entryElement.click();
cy.get('#tbody-Headers').should('be.visible');
isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
cy.get('#list [id^=entry]').each(entryElement => {
entryElement.click();
cy.get('#tbody-Headers').should('be.visible');
isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
});
});
});
}

View File

@@ -218,12 +218,8 @@ function checkFilter(filterDetails) {
checkRightSideResponseBody();
});
cy.get('[title="Fetch old records"]').click();
resizeToHugeMizu();
// waiting for the entries number to load
cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
// checking only 'leftTextCheck' on all entries because the rest of the checks require more time
cy.get(`#list [id^=entry]`).each(elem => {
const element = elem[0];

View File

@@ -20,7 +20,7 @@ require (
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/orcaman/concurrent-map v1.0.0
github.com/stretchr/testify v1.7.0
github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337
github.com/up9inc/mizu/logger v0.0.0
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0

View File

@@ -683,8 +683,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk=
github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337 h1:eRXRZnojrZyhbiSuGHl0EPvFtWvx1ZMrsY/bSoBzYNE=
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=

View File

@@ -3,6 +3,7 @@ package api
import (
"context"
"encoding/json"
"time"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/dependency"
@@ -38,6 +39,11 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
entryStreamerSocketConnector.SendToastError(socketId, err)
}
leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
if err != nil {
logger.Log.Errorf("Fetch error: %v", err.Error())
}
handleDataChannel := func(c *basenine.Connection, data chan []byte) {
for {
bytes := <-data
@@ -79,7 +85,7 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
go handleDataChannel(connection, data)
go handleMetaChannel(connection, meta)
if err = connection.Query(query, data, meta); err != nil {
if err = connection.Query(leftOff, query, data, meta); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
@@ -94,3 +100,61 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
return nil
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
if params.Fetch <= 0 {
leftOff = params.LeftOff
return
}
var data [][]byte
var firstMeta []byte
var lastMeta []byte
data, firstMeta, lastMeta, err = basenine.Fetch(
shared.BasenineHost,
shared.BaseninePort,
params.LeftOff,
-1,
params.Query,
params.Fetch,
time.Duration(params.TimeoutMs)*time.Millisecond,
)
if err != nil {
return
}
var firstMetadata *basenine.Metadata
err = json.Unmarshal(firstMeta, &firstMetadata)
if err != nil {
return
}
leftOff = firstMetadata.LeftOff
var lastMetadata *basenine.Metadata
err = json.Unmarshal(lastMeta, &lastMetadata)
if err != nil {
return
}
connector.SendMetadata(socketId, lastMetadata)
data = e.reverseBytesSlice(data)
for _, row := range data {
var entry *tapApi.Entry
err = json.Unmarshal(row, &entry)
if err != nil {
break
}
connector.SendEntry(socketId, entry, params)
}
return
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
for i := len(arr) - 1; i >= 0; i-- {
newArr = append(newArr, arr[i])
}
return newArr
}

View File

@@ -34,8 +34,11 @@ type SocketConnection struct {
}
type WebSocketParams struct {
LeftOff string `json:"leftOff"`
Query string `json:"query"`
EnableFullEntries bool `json:"enableFullEntries"`
Fetch int `json:"fetch"`
TimeoutMs int `json:"timeoutMs"`
}
var (

View File

@@ -22,7 +22,7 @@ type EntriesProvider interface {
type BasenineEntriesProvider struct{}
func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
if err != nil {
@@ -49,7 +49,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
}
var metadata *basenine.Metadata
err = json.Unmarshal(meta, &metadata)
err = json.Unmarshal(lastMeta, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}

View File

@@ -124,8 +124,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
postData, _ := request["postData"].(map[string]interface{})
mimeType := postData["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
text := postData["text"]
postDataText := ""
@@ -177,8 +177,8 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
content, _ := response["content"].(map[string]interface{})
mimeType := content["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
encoding := content["encoding"]
text := content["text"]

View File

@@ -104,7 +104,7 @@ func (g *defaultOasGenerator) runGenerator() {
g.dbMutex.Lock()
defer g.dbMutex.Unlock()
logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil {
if err := g.dbConn.Query("", g.entriesQuery, dataChan, metaChan); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
}

View File

@@ -327,7 +327,7 @@ BasenineReconnect:
go handleMetaChannel(&wg, connection, meta)
wg.Add(2)
if err = connection.Query(query, data, meta); err != nil {
if err = connection.Query("", query, data, meta); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)

View File

@@ -13,4 +13,4 @@ test-pull-bin:
test-pull-expect:
@mkdir -p expect
@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/http/\* expect
@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect9/http/\* expect

View File

@@ -401,8 +401,8 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
postData, _ := request["postData"].(map[string]interface{})
mimeType := postData["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
text := postData["text"]
if text != nil {
@@ -483,8 +483,8 @@ func representResponse(response map[string]interface{}) (repResponse []interface
content, _ := response["content"].(map[string]interface{})
mimeType := content["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
encoding := content["encoding"]
text := content["text"]

View File

@@ -23,7 +23,7 @@ interface EntriesListProps {
setIsSnappedToBottom: any;
noMoreDataTop: boolean;
setNoMoreDataTop: (flag: boolean) => void;
openWebSocket: (query: string, resetEntries: boolean) => void;
openWebSocket: (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => void;
scrollableRef: any;
ws: any;
}
@@ -195,11 +195,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({
className={`${styles.btnLive} ${isSnappedToBottom && !isWsConnectionClosed ? styles.hideButton : styles.showButton}`}
onClick={(_) => {
if (isWsConnectionClosed) {
if (query) {
openWebSocket(`(${query}) and leftOff("${leftOffBottom}")`, false);
} else {
openWebSocket(`leftOff("${leftOffBottom}")`, false);
}
openWebSocket(leftOffBottom, query, false, 0, 0);
}
scrollableRef.current.jumpToBottom();
setIsSnappedToBottom(true);

View File

@@ -289,10 +289,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, style, headingMode, name
src={ingoingIcon}
alt="Ingoing traffic"
title="Ingoing"
onClick={() => {
const query = `outgoing == false`;
setQuery(queryState ? `${queryState} and ${query}` : query);
}}
/>
</Queryable>
}

View File

@@ -20,7 +20,7 @@ import {StatusBar} from "../UI/StatusBar";
import tappingStatusAtom from "../../recoil/tappingStatus/atom";
import {TOAST_CONTAINER_ID} from "../../configs/Consts";
import leftOffTopAtom from "../../recoil/leftOffTop";
import { DEFAULT_QUERY } from '../../hooks/useWS';
import { DEFAULT_LEFTOFF, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS } from '../../hooks/useWS';
const useLayoutStyles = makeStyles(() => ({
details: {
@@ -114,11 +114,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
const ws = useRef(null);
const openEmptyWebSocket = () => {
if (query) {
openWebSocket(`(${query}) and ${DEFAULT_QUERY}`, true);
} else {
openWebSocket(DEFAULT_QUERY, true);
}
openWebSocket(DEFAULT_LEFTOFF, query, true, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS);
}
const closeWebSocket = () => {
@@ -129,7 +125,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
}
const listEntry = useRef(null);
const openWebSocket = (query: string, resetEntries: boolean) => {
const openWebSocket = (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => {
if (resetEntries) {
setFocusedEntryId(null);
setEntries([]);
@@ -138,7 +134,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
}
try {
ws.current = new WebSocket(webSocketUrl);
sendQueryWhenWsOpen(query);
sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs);
ws.current.onopen = () => {
setWsReadyState(ws?.current?.readyState);
@@ -157,12 +153,18 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
}
}
const sendQueryWhenWsOpen = (query) => {
const sendQueryWhenWsOpen = (leftOff: string, query: string, fetch: number, fetchTimeoutMs: number) => {
setTimeout(() => {
if (ws?.current?.readyState === WebSocket.OPEN) {
ws.current.send(JSON.stringify({"query": query, "enableFullEntries": false}));
ws.current.send(JSON.stringify({
"leftOff": leftOff,
"query": query,
"enableFullEntries": false,
"fetch": fetch,
"timeoutMs": fetchTimeoutMs
}));
} else {
sendQueryWhenWsOpen(query);
sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs);
}
}, 500)
}
@@ -240,7 +242,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
return (
<div className={TrafficViewerStyles.TrafficPage}>
{tappingStatus && isShowStatusBar && <StatusBar isDemoBannerView={isDemoBannerView}/>}
{tappingStatus && isShowStatusBar && <StatusBar disabled={ws?.current?.readyState !== WebSocket.OPEN} isDemoBannerView={isDemoBannerView}/>}
<div className={TrafficViewerStyles.TrafficPageHeader}>
<div className={TrafficViewerStyles.TrafficPageStreamStatus}>
<img className={TrafficViewerStyles.playPauseIcon}

View File

@@ -5,6 +5,7 @@ import failIcon from 'assets/failed.svg';
import successIcon from 'assets/success.svg';
import {useRecoilValue} from "recoil";
import tappingStatusAtom, {tappingStatusDetails} from "../../recoil/tappingStatus";
import Tooltip from "./Tooltip";
const pluralize = (noun: string, amount: number) => {
return `${noun}${amount !== 1 ? 's' : ''}`
@@ -12,20 +13,22 @@ const pluralize = (noun: string, amount: number) => {
interface StatusBarProps {
isDemoBannerView: boolean;
disabled?: boolean;
}
export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView}) => {
export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView, disabled}) => {
const tappingStatus = useRecoilValue(tappingStatusAtom);
const [expandedBar, setExpandedBar] = useState(false);
const {uniqueNamespaces, amountOfPods, amountOfTappedPods, amountOfUntappedPods} = useRecoilValue(tappingStatusDetails);
return <div className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
return <div style={{opacity: disabled ? 0.4 : 1}} className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar && !disabled ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
<div className={style.podsCount}>
{tappingStatus.some(pod => !pod.isTapped) && <img src={warningIcon} alt="warning"/>}
{disabled && <Tooltip title={"Tapping status is not updated when streaming is paused"} isSimple><img src={warningIcon} alt="warning"/></Tooltip>}
<span className={style.podsCountText} data-cy="podsCountText">
{`Tapping ${amountOfUntappedPods > 0 ? amountOfTappedPods + " / " + amountOfPods : amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}
</span>
</div>
{expandedBar && <div style={{marginTop: 20}}>
{expandedBar && !disabled && <div style={{marginTop: 20}}>
<table>
<thead>
<tr>

View File

@@ -7,7 +7,9 @@ enum WebSocketReadyState {
CLOSED
}
export const DEFAULT_QUERY = `leftOff("latest")`;
export const DEFAULT_LEFTOFF = `latest`;
export const DEFAULT_FETCH = 50;
export const DEFAULT_FETCH_TIMEOUT_MS = 3000;
const useWS = (wsUrl: string) => {
const [message, setMessage] = useState(null);

View File

@@ -1,11 +1,11 @@
import TrafficViewer from './components/TrafficViewer/TrafficViewer';
import * as UI from "./components/UI"
import { StatusBar } from './components/UI';
import useWS, { DEFAULT_QUERY } from './hooks/useWS';
import useWS, { DEFAULT_LEFTOFF } from './hooks/useWS';
import { AnalyzeButton } from "./components/AnalyzeButton/AnalyzeButton"
import OasModal from './components/OasModal/OasModal';
import { ServiceMapModal } from './components/ServiceMapModal/ServiceMapModal';
export { UI, AnalyzeButton, StatusBar, OasModal, ServiceMapModal }
export { useWS, DEFAULT_QUERY }
export { useWS, DEFAULT_LEFTOFF }
export default TrafficViewer;