Compare commits

..

11 Commits

Author SHA1 Message Date
David Levanon
5c012641a5 Feature/limit fd to address map (#1078)
* limit fd to address map

* limit fd to address map

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2022-05-16 12:32:26 +03:00
Igor Gov
74bd4b180f Removing elastic dump experimental feature (#1086)
* Removing elastic dump experimental feature
2022-05-15 15:23:03 +03:00
RoyUP9
8ea2dabb34 Added send to socket error validation (#1085) 2022-05-15 14:44:48 +03:00
leon-up9
366d34b8d0 Ui/fix/selectList-sticky-header-fix (#1084)
* selectList sticky header

* selectList changed & servicemap adapted

* ignore eslint

Co-authored-by: Leon <>
Co-authored-by: AmitUp9 <96980485+AmitUp9@users.noreply.github.com>
2022-05-15 14:39:09 +03:00
M. Mert Yıldıran
5fc3e38c1a Fix the Kafka ApiKey query and add ApiKeyName field (human-readable ApiKey) (#1080)
* Fix the Kafka `ApiKey` query and add `ApiKeyName` field (human-readable `ApiKey`)

* Update the dataset for Kafka unit tests

* #run_acceptance_tests
2022-05-15 09:42:32 +03:00
RoyUP9
09a0fca2c2 Extracted insert to database functionality (#1082) 2022-05-15 09:19:33 +03:00
M. Mert Yıldıran
0437586908 Replace the gRPC reference link with a better one (#1081) 2022-05-14 19:43:43 +03:00
M. Mert Yıldıran
f8181ccb07 Remove ReassemblyStream interface (duplicate of gopacket/reassembly.Stream) (#1079) 2022-05-14 18:37:16 +03:00
leon-up9
414e5cfe5a match selectlist ui (#1077)
* insert filter and header to selectlist

* handle single select

* rename search var

* font size changed
2022-05-11 19:51:12 +03:00
RoyUP9
2fac0009ea Fixed oas query (#1076) 2022-05-11 15:36:46 +03:00
M. Mert Yıldıran
36d59ede07 Fix acceptance tests failures caused by fetch 50 and a race in the tests (#1074)
* #run_acceptance_tests

* Wait 4 seconds after filter submission #run_acceptance_tests

* Revert "Wait 4 seconds after filter submission #run_acceptance_tests"

This reverts commit 3c20ccfcec.

* Upgrade Basenine version to `v0.8.2` #run_acceptance_tests

* #run_acceptance_tests

* Revert "Revert "Wait 4 seconds after filter submission #run_acceptance_tests""

This reverts commit f0c595150d.

* Reduce the delay to 2 seconds #run_acceptance_tests

* Use `cy.wait()` instead of `setTimeout`

* #run_acceptance_tests

* Increase it to 4 seconds #run_acceptance_tests

* #run_acceptance_tests

* Wait for a second and pause the stream to preserve the DOM #run_acceptance_tests

* Wait in a better place #run_acceptance_tests

* Wait `pause-icon` to disappear #run_acceptance_tests

* Wait one more second #run_acceptance_tests

* Fix `setup.sh`

* #run_acceptance_tests

* Fix the place of `npm i` call

* #run_acceptance_tests

* Fix the other workflow as well #run_acceptance_tests

* Fix the workflows once more

* #run_acceptance_tests

* Don't ignore `*/cypress/support` #run_acceptance_tests

* Do `npm i` in `setup.sh` #run_acceptance_tests

* Wait for at least 4 seconds #run_acceptance_tests

* #run_acceptance_tests

* Fix #run_acceptance_tests

* Comment #run_acceptance_tests

* Fix `{alt+enter}` to `{ctrl+enter}` and update the `checkFilter` and `deeperCheck` method signatures #run_acceptance_tests

* #run_acceptance_tests

* #run_acceptance_tests

* Extract it to a function #run_acceptance_tests

* Remove `cypress-wait-until`

* Revert `.gitignore` #run_acceptance_tests
2022-05-11 13:14:01 +03:00
31 changed files with 285 additions and 295 deletions

2
.gitignore vendored
View File

@@ -55,4 +55,4 @@ tap/extensions/*/expect
*.editorconfig
# Ignore *.log files
*.log
*.log

View File

@@ -94,8 +94,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
-X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
# Download Basenine executable, verify the sha1sum
ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
chmod +x ./basenine_linux_"${GOARCH}" && \

View File

@@ -65,70 +65,70 @@ it('right side sanity test', function () {
checkIllegalFilter('invalid filter');
checkFilter({
name: 'http',
filter: 'http',
leftSidePath: '> :nth-child(1) > :nth-child(1)',
leftSideExpectedText: 'HTTP',
rightSidePath: '[title=HTTP]',
rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
applyByEnter: true
applyByCtrlEnter: true
});
checkFilter({
name: 'response.status == 200',
filter: 'response.status == 200',
leftSidePath: '[title="Status Code"]',
leftSideExpectedText: '200',
rightSidePath: '> :nth-child(2) [title="Status Code"]',
rightSideExpectedText: '200',
applyByEnter: false
applyByCtrlEnter: false
});
if (Cypress.env('shouldCheckSrcAndDest')) {
serviceMapCheck();
checkFilter({
name: 'src.name == ""',
filter: 'src.name == ""',
leftSidePath: '[title="Source Name"]',
leftSideExpectedText: '[Unresolved]',
rightSidePath: '> :nth-child(2) [title="Source Name"]',
rightSideExpectedText: '[Unresolved]',
applyByEnter: false
applyByCtrlEnter: false
});
checkFilter({
name: `dst.name == "httpbin.mizu-tests"`,
filter: `dst.name == "httpbin.mizu-tests"`,
leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
leftSideExpectedText: 'httpbin.mizu-tests',
rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
rightSideExpectedText: 'httpbin.mizu-tests',
applyByEnter: false
applyByCtrlEnter: false
});
}
checkFilter({
name: 'request.method == "GET"',
filter: 'request.method == "GET"',
leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
leftSideExpectedText: 'GET',
rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
rightSideExpectedText: 'GET',
applyByEnter: true
applyByCtrlEnter: true
});
checkFilter({
name: 'request.path == "/get"',
filter: 'request.path == "/get"',
leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
leftSideExpectedText: '/get',
rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
rightSideExpectedText: '/get',
applyByEnter: false
applyByCtrlEnter: false
});
checkFilter({
name: 'src.ip == "127.0.0.1"',
filter: 'src.ip == "127.0.0.1"',
leftSidePath: '[title="Source IP"]',
leftSideExpectedText: '127.0.0.1',
rightSidePath: '> :nth-child(2) [title="Source IP"]',
rightSideExpectedText: '127.0.0.1',
applyByEnter: false
applyByCtrlEnter: false
});
checkFilterNoResults('request.method == "POST"');
@@ -182,17 +182,19 @@ function checkIllegalFilter(illegalFilterName) {
function checkFilter(filterDetails) {
const {
name,
filter,
leftSidePath,
rightSidePath,
rightSideExpectedText,
leftSideExpectedText,
applyByEnter
applyByCtrlEnter
} = filterDetails;
const entriesForDeeperCheck = 5;
it(`checking the filter: ${name}`, function () {
it(`checking the filter: ${filter}`, function () {
waitForFetch50AndPause();
cy.get('#total-entries').should('not.have.text', '0').then(number => {
const totalEntries = number.text();
@@ -200,21 +202,23 @@ function checkFilter(filterDetails) {
const element = elem[0];
const entryId = getEntryId(element.id);
// checks the hover on the last entry (the only one in DOM at the beginning)
leftOnHoverCheck(entryId, leftSidePath, name);
leftOnHoverCheck(entryId, leftSidePath, filter);
cy.get('.w-tc-editor-text').clear();
// applying the filter with alt+enter or with the button
cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
cy.get('.w-tc-editor-text').type(`${filter}${applyByCtrlEnter ? '{ctrl+enter}' : ''}`);
cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
if (!applyByEnter)
if (!applyByCtrlEnter)
cy.get('[type="submit"]').click();
waitForFetch50AndPause();
// only one entry in DOM after filtering, checking all checks on it
leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
leftOnHoverCheck(entryId, leftSidePath, name);
leftOnHoverCheck(entryId, leftSidePath, filter);
rightTextCheck(rightSidePath, rightSideExpectedText);
rightOnHoverCheck(rightSidePath, name);
rightOnHoverCheck(rightSidePath, filter);
checkRightSideResponseBody();
});
@@ -228,7 +232,7 @@ function checkFilter(filterDetails) {
});
// making the other 3 checks on the first X entries (longer time for each check)
deeperCheck(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
deeperCheck(leftSidePath, rightSidePath, filter, rightSideExpectedText, entriesForDeeperCheck);
// reloading then waiting for the entries number to load
resizeToNormalMizu();
@@ -238,7 +242,14 @@ function checkFilter(filterDetails) {
});
}
function deeperCheck(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
function waitForFetch50AndPause() {
// wait half a second and pause the stream to preserve the DOM
cy.wait(500);
cy.get('#pause-icon').click();
cy.get('#pause-icon').should('not.be.visible');
}
function deeperCheck(leftSidePath, rightSidePath, filterName, rightSideExpectedText, entriesNumToCheck) {
cy.get(`#list [id^=entry]`).each((element, index) => {
if (index < entriesNumToCheck) {
const entryId = getEntryId(element[0].id);

View File

@@ -6,7 +6,6 @@ require (
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
github.com/chanced/openapi v0.0.8
github.com/djherbis/atime v1.1.0
github.com/elastic/go-elasticsearch/v7 v7.17.0
github.com/getkin/kin-openapi v0.89.0
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.7.7
@@ -20,7 +19,7 @@ require (
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/orcaman/concurrent-map v1.0.0
github.com/stretchr/testify v1.7.0
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337
github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4
github.com/up9inc/mizu/logger v0.0.0
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0

View File

@@ -166,8 +166,6 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/elastic/go-elasticsearch/v7 v7.17.0 h1:0fcSh4qeC/i1+7QU1KXpmq2iUAdMk4l0/vmbtW1+KJM=
github.com/elastic/go-elasticsearch/v7 v7.17.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -683,8 +681,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337 h1:eRXRZnojrZyhbiSuGHl0EPvFtWvx1ZMrsY/bSoBzYNE=
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4 h1:nNOrU1HVH0fnaG7GNhxCc8kNPVL035Iix7ihUF6lZT8=
github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=

View File

@@ -17,7 +17,6 @@ import (
"github.com/gin-contrib/static"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/elastic"
"github.com/up9inc/mizu/agent/pkg/entries"
"github.com/up9inc/mizu/agent/pkg/middlewares"
"github.com/up9inc/mizu/agent/pkg/models"
@@ -205,7 +204,6 @@ func enableExpFeatureIfNeeded() {
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
serviceMapGenerator.Enable()
}
elastic.GetInstance().Configure(config.Config.Elastic)
}
func getSyncEntriesConfig() *shared.SyncEntriesConfig {
@@ -373,6 +371,7 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
func initializeDependencies() {
dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance() })
dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })

View File

@@ -5,20 +5,19 @@ import (
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/logger"
tapApi "github.com/up9inc/mizu/tap/api"
)
type EntryStreamerSocketConnector interface {
SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams)
SendMetadata(socketId int, metadata *basenine.Metadata)
SendToastError(socketId int, err error)
SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
SendMetadata(socketId int, metadata *basenine.Metadata) error
SendToastError(socketId int, err error) error
CleanupSocket(socketId int)
}
type DefaultEntryStreamerSocketConnector struct{}
func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) {
func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
var message []byte
if params.EnableFullEntries {
message, _ = models.CreateFullEntryWebSocketMessage(entry)
@@ -29,26 +28,32 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
}
if err := SendToSocket(socketId, message); err != nil {
logger.Log.Error(err)
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) {
func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
if err := SendToSocket(socketId, metadataBytes); err != nil {
logger.Log.Error(err)
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) {
func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
Type: "error",
AutoClose: 5000,
Text: fmt.Sprintf("Syntax error: %s", err.Error()),
})
if err := SendToSocket(socketId, toastBytes); err != nil {
logger.Log.Error(err)
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {

View File

@@ -14,7 +14,6 @@ import (
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/elastic"
"github.com/up9inc/mizu/agent/pkg/har"
"github.com/up9inc/mizu/agent/pkg/holder"
"github.com/up9inc/mizu/agent/pkg/providers"
@@ -25,10 +24,7 @@ import (
"github.com/up9inc/mizu/agent/pkg/utils"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
tapApi "github.com/up9inc/mizu/tap/api"
basenine "github.com/up9inc/basenine/client/go"
)
var k8sResolver *resolver.Resolver
@@ -103,20 +99,6 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
panic("Channel of captured messages is nil")
}
BasenineReconnect:
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
time.Sleep(shared.BasenineReconnectInterval * time.Second)
goto BasenineReconnect
}
if err = connection.InsertMode(); err != nil {
logger.Log.Errorf("Insert mode call failed: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)
goto BasenineReconnect
}
disableOASValidation := false
ctx := context.Background()
doc, contractContent, router, err := loadOAS(ctx)
@@ -163,17 +145,13 @@ BasenineReconnect:
providers.EntryAdded(len(data))
if err = connection.SendText(string(data)); err != nil {
logger.Log.Errorf("An error occured while inserting a new record to database: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)
goto BasenineReconnect
entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
if err := entryInserter.Insert(mizuEntry); err != nil {
logger.Log.Errorf("Error inserting entry, err: %v", err)
}
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
elastic.GetInstance().PushEntry(mizuEntry)
}
}

View File

@@ -0,0 +1,71 @@
package api
import (
"encoding/json"
"fmt"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
"sync"
"time"
)
type EntryInserter interface {
Insert(entry *api.Entry) error
}
type BasenineEntryInserter struct {
connection *basenine.Connection
}
var instance *BasenineEntryInserter
var once sync.Once
func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
once.Do(func() {
instance = &BasenineEntryInserter{}
})
return instance
}
func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
if e.connection == nil {
e.connection = initializeConnection()
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("error marshling entry, err: %v", err)
}
if err := e.connection.SendText(string(data)); err != nil {
e.connection.Close()
e.connection = nil
return fmt.Errorf("error sending text to database, err: %v", err)
}
return nil
}
func initializeConnection() *basenine.Connection{
for {
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
time.Sleep(shared.BasenineReconnectInterval * time.Second)
continue
}
if err = connection.InsertMode(); err != nil {
logger.Log.Errorf("Insert mode call failed: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)
continue
}
return connection
}
}

View File

@@ -34,14 +34,18 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
meta := make(chan []byte)
query := params.Query
err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
if err != nil {
entryStreamerSocketConnector.SendToastError(socketId, err)
if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
return err
}
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
}
leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
if err != nil {
logger.Log.Errorf("Fetch error: %v", err.Error())
logger.Log.Errorf("Fetch error: %v", err)
}
handleDataChannel := func(c *basenine.Connection, data chan []byte) {
@@ -53,13 +57,15 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
}
var entry *tapApi.Entry
err = json.Unmarshal(bytes, &entry)
if err != nil {
logger.Log.Debugf("Error unmarshalling entry: %v", err.Error())
if err = json.Unmarshal(bytes, &entry); err != nil {
logger.Log.Debugf("Error unmarshalling entry: %v", err)
continue
}
entryStreamerSocketConnector.SendEntry(socketId, entry, params)
if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
logger.Log.Errorf("Error sending entry to socket, err: %v", err)
return
}
}
}
@@ -72,13 +78,15 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
}
var metadata *basenine.Metadata
err = json.Unmarshal(bytes, &metadata)
if err != nil {
logger.Log.Debugf("Error unmarshalling metadata: %v", err.Error())
if err = json.Unmarshal(bytes, &metadata); err != nil {
logger.Log.Debugf("Error unmarshalling metadata: %v", err)
continue
}
entryStreamerSocketConnector.SendMetadata(socketId, metadata)
if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
return
}
}
}
@@ -125,28 +133,31 @@ func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, con
}
var firstMetadata *basenine.Metadata
err = json.Unmarshal(firstMeta, &firstMetadata)
if err != nil {
if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
return
}
leftOff = firstMetadata.LeftOff
var lastMetadata *basenine.Metadata
err = json.Unmarshal(lastMeta, &lastMetadata)
if err != nil {
if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
return
}
if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
return
}
connector.SendMetadata(socketId, lastMetadata)
data = e.reverseBytesSlice(data)
for _, row := range data {
var entry *tapApi.Entry
err = json.Unmarshal(row, &entry)
if err != nil {
if err = json.Unmarshal(row, &entry); err != nil {
break
}
connector.SendEntry(socketId, entry, params)
if err = connector.SendEntry(socketId, entry, params); err != nil {
return
}
}
return
}

View File

@@ -5,6 +5,7 @@ type DependencyContainerType string
const (
ServiceMapGeneratorDependency = "ServiceMapGeneratorDependency"
OasGeneratorDependency = "OasGeneratorDependency"
EntriesInserter = "EntriesInserter"
EntriesProvider = "EntriesProvider"
EntriesSocketStreamer = "EntriesSocketStreamer"
EntryStreamerSocketConnector = "EntryStreamerSocketConnector"

View File

@@ -1,116 +0,0 @@
package elastic
import (
"bytes"
"crypto/tls"
"encoding/json"
"net/http"
"sync"
"time"
"github.com/elastic/go-elasticsearch/v7"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
type client struct {
es *elasticsearch.Client
index string
insertedCount int
}
var instance *client
var once sync.Once
func GetInstance() *client {
once.Do(func() {
instance = newClient()
})
return instance
}
func (client *client) Configure(config shared.ElasticConfig) {
if config.Url == "" || config.User == "" || config.Password == "" {
if client.es != nil {
client.es = nil
}
logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
return
}
transport := http.DefaultTransport
tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
transport.(*http.Transport).TLSClientConfig = tlsClientConfig
cfg := elasticsearch.Config{
Addresses: []string{config.Url},
Username: config.User,
Password: config.Password,
Transport: transport,
}
es, err := elasticsearch.NewClient(cfg)
if err != nil {
logger.Log.Errorf("Failed to initialize elastic client %v", err)
}
// Have the client instance return a response
res, err := es.Info()
if err != nil {
logger.Log.Errorf("Elastic client.Info() ERROR: %v", err)
} else {
client.es = es
client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
client.insertedCount = 0
logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
}
defer res.Body.Close()
}
func newClient() *client {
return &client{
es: nil,
index: "",
}
}
type httpEntry struct {
Source *api.TCP `json:"src"`
Destination *api.TCP `json:"dst"`
Outgoing bool `json:"outgoing"`
CreatedAt time.Time `json:"createdAt"`
Request map[string]interface{} `json:"request"`
Response map[string]interface{} `json:"response"`
ElapsedTime int64 `json:"elapsedTime"`
}
func (client *client) PushEntry(entry *api.Entry) {
if client.es == nil {
return
}
if entry.Protocol.Name != "http" {
return
}
entryToPush := httpEntry{
Source: entry.Source,
Destination: entry.Destination,
Outgoing: entry.Outgoing,
CreatedAt: entry.StartTime,
Request: entry.Request,
Response: entry.Response,
ElapsedTime: entry.ElapsedTime,
}
entryJson, err := json.Marshal(entryToPush)
if err != nil {
logger.Log.Errorf("json.Marshal ERROR: %v", err)
return
}
var buffer bytes.Buffer
buffer.WriteString(string(entryJson))
res, _ := client.es.Index(client.index, &buffer)
if res.StatusCode == 201 {
client.insertedCount += 1
}
}

View File

@@ -104,7 +104,7 @@ func (g *defaultOasGenerator) runGenerator() {
g.dbMutex.Lock()
defer g.dbMutex.Unlock()
logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
if err := g.dbConn.Query("", g.entriesQuery, dataChan, metaChan); err != nil {
if err := g.dbConn.Query("latest", g.entriesQuery, dataChan, metaChan); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
}

View File

@@ -327,7 +327,7 @@ BasenineReconnect:
go handleMetaChannel(&wg, connection, meta)
wg.Add(2)
if err = connection.Query("", query, data, meta); err != nil {
if err = connection.Query("latest", query, data, meta); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)

View File

@@ -164,7 +164,6 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
ServiceMap: config.Config.ServiceMap,
OAS: config.Config.OAS,
Telemetry: config.Config.Telemetry,
Elastic: config.Config.Elastic,
}
return &mizuAgentConfig

View File

@@ -41,7 +41,6 @@ type ConfigStruct struct {
LogLevelStr string `yaml:"log-level,omitempty" default:"INFO" readonly:""`
ServiceMap bool `yaml:"service-map" default:"true"`
OAS bool `yaml:"oas" default:"true"`
Elastic shared.ElasticConfig `yaml:"elastic"`
}
func (config *ConfigStruct) validate() error {

View File

@@ -45,13 +45,6 @@ type MizuAgentConfig struct {
ServiceMap bool `json:"serviceMap"`
OAS bool `json:"oas"`
Telemetry bool `json:"telemetry"`
Elastic ElasticConfig `json:"elastic"`
}
type ElasticConfig struct {
User string `yaml:"user,omitempty" default:"" readonly:""`
Password string `yaml:"password,omitempty" default:"" readonly:""`
Url string `yaml:"url,omitempty" default:"" readonly:""`
}
type WebSocketMessageMetadata struct {

View File

@@ -66,7 +66,7 @@ var grpcProtocol api.Protocol = api.Protocol{
BackgroundColor: "#244c5a",
ForegroundColor: "#ffffff",
FontSize: 11,
ReferenceLink: "https://grpc.github.io/grpc/core/md_doc_statuscodes.html",
ReferenceLink: "https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html",
Ports: []string{"80", "443", "8080", "50051"},
Priority: 0,
}

View File

@@ -13,4 +13,4 @@ test-pull-bin:
test-pull-expect:
@mkdir -p expect
@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/kafka/\* expect
@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect9/kafka/\* expect

View File

@@ -3,13 +3,14 @@ package kafka
import (
"encoding/json"
"fmt"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"reflect"
"sort"
"strconv"
"strings"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"github.com/fatih/camelcase"
"github.com/ohler55/ojg/jp"
"github.com/ohler55/ojg/oj"
@@ -36,9 +37,14 @@ type KafkaWrapper struct {
func representRequestHeader(data map[string]interface{}, rep []interface{}) []interface{} {
requestHeader, _ := json.Marshal([]api.TableData{
{
Name: "ApiKeyName",
Value: data["apiKeyName"].(string),
Selector: `request.apiKeyName`,
},
{
Name: "ApiKey",
Value: apiNames[int(data["apiKey"].(float64))],
Value: int(data["apiKey"].(float64)),
Selector: `request.apiKey`,
},
{

View File

@@ -96,8 +96,8 @@ func (d dissecting) Summarize(entry *api.Entry) *api.BaseEntry {
statusQuery := ""
apiKey := ApiKey(entry.Request["apiKey"].(float64))
method := apiNames[apiKey]
methodQuery := fmt.Sprintf("request.apiKey == %d", int(entry.Request["apiKey"].(float64)))
method := entry.Request["apiKeyName"].(string)
methodQuery := fmt.Sprintf(`request.apiKeyName == "%s"`, method)
summary := ""
summaryQuery := ""

View File

@@ -11,6 +11,7 @@ import (
type Request struct {
Size int32 `json:"size"`
ApiKeyName string `json:"apiKeyName"`
ApiKey ApiKey `json:"apiKey"`
ApiVersion int16 `json:"apiVersion"`
CorrelationID int32 `json:"correlationID"`
@@ -202,6 +203,7 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, ca
request := &Request{
Size: size,
ApiKeyName: apiNames[apiKey],
ApiKey: apiKey,
ApiVersion: apiVersion,
CorrelationID: correlationID,

View File

@@ -10,12 +10,6 @@ import (
"github.com/up9inc/mizu/tap/diagnose"
)
type ReassemblyStream interface {
Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool
ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext)
ReassemblyComplete(ac reassembly.AssemblerContext) bool
}
type tcpReassemblyStream struct {
ident string
tcpState *reassembly.TCPSimpleFSM
@@ -25,7 +19,7 @@ type tcpReassemblyStream struct {
tcpStream api.TcpStream
}
func NewTcpReassemblyStream(ident string, tcp *layers.TCP, fsmOptions reassembly.TCPSimpleFSMOptions, stream api.TcpStream) ReassemblyStream {
func NewTcpReassemblyStream(ident string, tcp *layers.TCP, fsmOptions reassembly.TCPSimpleFSMOptions, stream api.TcpStream) reassembly.Stream {
return &tcpReassemblyStream{
ident: ident,
tcpState: reassembly.NewTCPSimpleFSM(fsmOptions),

View File

@@ -68,7 +68,7 @@ struct fd_info {
BPF_HASH(pids_map, __u32, __u32);
BPF_LRU_HASH(ssl_write_context, __u64, struct ssl_info);
BPF_LRU_HASH(ssl_read_context, __u64, struct ssl_info);
BPF_HASH(file_descriptor_to_ipv4, __u64, struct fd_info);
BPF_LRU_HASH(file_descriptor_to_ipv4, __u64, struct fd_info);
BPF_PERF_OUTPUT(chunks_buffer);
BPF_PERF_OUTPUT(log_buffer);

Binary file not shown.

Binary file not shown.

View File

@@ -67,7 +67,6 @@
height: 100%
display: flex
flex-direction: column
margin-right: 10px
width: 100%
border-radius: 4px
@@ -82,22 +81,18 @@
margin-top: 10px
margin-right: 10px
.protocolsFilterList, .servicesFilter
.card
background: white
padding: 10px
border-radius: 4px
user-select: none
box-shadow: 0px 1px 5px #979797
.servicesFilter
margin-top: 10px
.servicesFilterWrapper
margin-top: 20px
margin-bottom: 3px
height: 100%
overflow: hidden
border-radius: 4px
& .servicesFilterList
height: calc(100% - 30px - 52px)
.totalSelected
font-size: 12px
color: $light-blue-color
font-weight: 700
.servicesFilterList
height: calc(100% - 30px - 52px)

View File

@@ -68,7 +68,6 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
const [checkedProtocols, setCheckedProtocols] = useState([])
const [checkedServices, setCheckedServices] = useState([])
const [serviceMapApiData, setServiceMapApiData] = useState<ServiceMapGraph>({ edges: [], nodes: [] })
const [servicesSearchVal, setServicesSearchVal] = useState("")
const [graphOptions, setGraphOptions] = useState(ServiceMapOptions);
const [isFilterClicked, setIsFilterClicked] = useState(true)
@@ -159,6 +158,7 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
if (checkedProtocols.length === 0) {
setCheckedProtocols(getProtocolsForFilter.map(x => x.key))
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [getProtocolsForFilter])
useEffect(() => {
@@ -218,23 +218,15 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
<div className={styles.filterSection + ` ${isFilterClicked ? styles.show : ""}`}>
<Resizeable minWidth={170} maxWidth={320}>
<div className={styles.filterWrapper}>
<div className={styles.protocolsFilterList}>
<h3 className={styles.subSectionHeader} style={{ marginLeft: "10px" }}>
PROTOCOLS
<span className={styles.totalSelected}>&nbsp;({checkedProtocols.length})</span>
</h3>
<SelectList items={getProtocolsForFilter} checkBoxWidth="5%" tableName={"All"} multiSelect={true}
checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters} />
<div className={styles.card}>
<SelectList items={getProtocolsForFilter} checkBoxWidth="5%" tableName={"PROTOCOLS"} multiSelect={true}
checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters}
inputSearchClass={styles.servicesFilterSearch} isFilterable={false}/>
</div>
<div className={styles.servicesFilter}>
<h3 className={styles.subSectionHeader} style={{ marginLeft: "10px" }}>
SERVICES
<span className={styles.totalSelected}>&nbsp;({checkedServices.length})</span>
</h3>
<input className={commonClasses.textField + ` ${styles.servicesFilterSearch}`} placeholder="Search" value={servicesSearchVal} onChange={(event) => setServicesSearchVal(event.target.value)} />
<div className={styles.servicesFilterWrapper + ` ${styles.card}`}>
<div className={styles.servicesFilterList}>
<SelectList items={getServicesForFilter} tableName={"All"} tableClassName={styles.filters} multiSelect={true} searchValue={servicesSearchVal}
checkBoxWidth="5%" checkedValues={checkedServices} setCheckedValues={onServiceChanges} />
<SelectList items={getServicesForFilter} tableName={"SERVICES"} tableClassName={styles.filters} multiSelect={true}
checkBoxWidth="5%" checkedValues={checkedServices} setCheckedValues={onServiceChanges} inputSearchClass={styles.servicesFilterSearch}/>
</div>
</div>
</div>

View File

@@ -245,13 +245,18 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
{tappingStatus && isShowStatusBar && <StatusBar disabled={ws?.current?.readyState !== WebSocket.OPEN} isDemoBannerView={isDemoBannerView}/>}
<div className={TrafficViewerStyles.TrafficPageHeader}>
<div className={TrafficViewerStyles.TrafficPageStreamStatus}>
<img className={TrafficViewerStyles.playPauseIcon}
style={{visibility: wsReadyState === WebSocket.OPEN ? "visible" : "hidden"}} alt="pause"
src={pauseIcon} onClick={toggleConnection}/>
<img className={TrafficViewerStyles.playPauseIcon}
<img id="pause-icon"
className={TrafficViewerStyles.playPauseIcon}
style={{visibility: wsReadyState === WebSocket.OPEN ? "visible" : "hidden"}}
alt="pause"
src={pauseIcon}
onClick={toggleConnection}/>
<img id="play-icon"
className={TrafficViewerStyles.playPauseIcon}
style={{position: "absolute", visibility: wsReadyState === WebSocket.OPEN ? "hidden" : "visible"}}
alt="play"
src={playIcon} onClick={toggleConnection}/>
src={playIcon}
onClick={toggleConnection}/>
<div className={TrafficViewerStyles.connectionText}>
{getConnectionTitle()}
{getConnectionIndicator()}

View File

@@ -3,6 +3,7 @@ import Radio from "./Radio";
import styles from './style/SelectList.module.sass'
import NoDataMessage from "./NoDataMessage";
import Checkbox from "./Checkbox";
import { useCommonStyles } from "../../helpers/commonStyle";
export interface Props {
@@ -10,14 +11,17 @@ export interface Props {
tableName: string;
checkedValues?: string[];
multiSelect: boolean;
searchValue?: string;
setCheckedValues: (newValues) => void;
tableClassName?
checkBoxWidth?: string
tableClassName?;
checkBoxWidth?: string;
inputSearchClass? : string
isFilterable? : boolean
}
const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], multiSelect = true, searchValue = "", setCheckedValues, tableClassName,
checkBoxWidth = 50 }) => {
const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], multiSelect = true, setCheckedValues, tableClassName,
checkBoxWidth = 50 ,inputSearchClass,isFilterable = true}) => {
const commonClasses = useCommonStyles();
const [searchValue, setSearchValue] = useState("")
const noItemsMessage = "No items to show";
const [headerChecked, setHeaderChecked] = useState(false)
@@ -73,16 +77,15 @@ const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], mul
<th style={{ width: checkBoxWidth }}><Checkbox data-cy="checkbox-all" checked={headerChecked}
onToggle={(isChecked) => toggleAll(isChecked)} /></th>
<th>
{tableName}
All
</th>
</tr> :
<tr style={{ borderBottomWidth: "2px" }}>
<th>{tableName}</th>
<tr>
</tr>
const tableBody = filteredValues.length === 0 ?
<tr>
<td colSpan={2}>
<td colSpan={2} className={styles.displayBlock}>
<NoDataMessage messageText={noItemsMessage} />
</td>
</tr>
@@ -100,7 +103,14 @@ const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], mul
}
)
return <div className={tableClassName ? tableClassName + ` ${styles.selectListTable}` : ` ${styles.selectListTable}`}>
return <React.Fragment>
<h3 className={styles.subSectionHeader}>
{tableName}
<span className={styles.totalSelected}>&nbsp;({checkedValues.length})</span>
</h3>
{isFilterable && <input className={commonClasses.textField + ` ${inputSearchClass}`} placeholder="Search" value={searchValue}
onChange={(event) => setSearchValue(event.target.value)} data-cy="searchInput" />}
<div className={tableClassName ? tableClassName + ` ${styles.selectListTable}` : ` ${styles.selectListTable}`} style={{marginTop: !multiSelect ? "20px": ""}}>
<table cellPadding={5} style={{ borderCollapse: "collapse" }}>
<thead>
{tableHead}
@@ -110,6 +120,7 @@ const SelectList: React.FC<Props> = ({ items, tableName, checkedValues = [], mul
</tbody>
</table>
</div>
</React.Fragment>
}
export default SelectList;
export default SelectList;

View File

@@ -1,33 +1,70 @@
@import '../../../variables.module'
@import '../../../components'
.selectListTable
overflow: auto
height: 100%
user-select: none // when resizble moved we get unwanted beheviour
height: 100%
table
width: 100%
margin-top: 20px
border-collapse: collapse
table-layout: fixed
height: 100%
display: flex
flex-flow: column
height: 100%
th
color: $blue-gray
text-align: left
padding: 10px
position: sticky
top: 0
background: $main-background-color
thead
display: table
table-layout: fixed
flex: 0 0 auto
width: calc(100% - 0.9em)
tr
border-bottom-width: 1px
border-bottom-color: $data-background-color
border-bottom-style: solid
width: 100%
tbody
display: block
overflow: auto
width: 100%
height: 100%
flex: 1 1 auto
td
color: $light-gray
padding: 10px
font-size: 16px
tbody tr:hover
background: $header-background-color
th
color: $blue-gray
text-align: left
padding: 10px
background: $main-background-color
font-size: 12px
tr
border-bottom-width: 1px
border-bottom-color: $data-background-color
border-bottom-style: solid
width: 100%
display: block
position: relative
display: table
table-layout: fixed
td
color: $light-gray
padding: 10px
font-size: 11px
font-weight: 600
padding-top: 5px
padding-bottom: 5px
.nowrap
white-space: nowrap
.totalSelected
font-size: 12px
color: $light-blue-color
font-weight: 700
.displayBlock
display: block
.filterInput
margin-bottom: 20px