Compare commits

...
15 Commits
Author SHA1 Message Date
RoyUP9andGitHub 30a85a4b92 Added tap pre check (#848) 2022-02-24 11:48:42 +02:00
Adam KolandGitHub cdbacff996 Cypress: Service Map test (#847) 2022-02-23 17:36:31 +02:00
RamiBermandGitHub cf127c781c Dont mount config map on tappers (#849) 2022-02-23 14:40:47 +02:00
M. Mert YıldıranandGitHub 852a5ff045 Bring back the query argument of getEntry call which is accidentally removed with #631 (#842) 2022-02-23 13:29:47 +03:00
Igor GovandGitHub 371e513249 Remove config dependency from basenine init (#846)
* Remove config dependency from basenine init
2022-02-23 10:06:19 +02:00
Igor GovandGitHub 97cce32e3f Fix: service map component aware of agent config (#845)
* Fix: Service map component aware of mizu config
2022-02-23 09:35:05 +02:00
Igor GovandGitHub d2e91b4ffa Fix: tapper tries to load agent config map (#844) 2022-02-23 09:20:19 +02:00
d5a42a66de apply install typo fix to develop (already in master) (#840)
Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2022-02-22 16:02:33 +02:00
RoyUP9andGitHub f01962085a Return amqp tests (#841) 2022-02-22 15:35:20 +02:00
M. Mert YıldıranandGitHub 73f3e448cf Fix the issue in the WebSocket that's introduced by #819 (#839) 2022-02-22 15:27:50 +03:00
Igor GovandGitHub 58a17897cf Agent logs: output to stderr errors and above (#838)
* Agent logs: output to stderr errors and above
2022-02-22 14:02:51 +02:00
M. Mert YıldıranandGitHub cf3106f636 Silence the logging that comes from Basenine and protocol dissectors (#835) 2022-02-21 18:08:16 +03:00
Andrey PokhilkoandGitHub a553a1b683 OAS: use resolved service names (#827)
* OAS service names to be resolved

* fix test
2022-02-21 17:20:59 +03:00
Igor GovandGitHub 2a6bbd66e6 Update relase template for Mac M1 executable (#836)
* Update release template for Mac M1
2022-02-21 13:32:48 +02:00
Adam KolandGitHub 5a4baa05ca rabbit test is skipped temporarily (#833) 2022-02-20 18:10:00 +02:00
31 changed files with 297 additions and 134 deletions
+2 -2
View File
@@ -78,8 +78,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
-X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent . -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
# Download Basenine executable, verify the sha1sum # Download Basenine executable, verify the sha1sum
ADD https://github.com/up9inc/basenine/releases/download/v0.4.16/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH} ADD https://github.com/up9inc/basenine/releases/download/v0.4.17/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.4.16/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256 ADD https://github.com/up9inc/basenine/releases/download/v0.4.17/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256 RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256
RUN chmod +x ./basenine_linux_${GOARCH} RUN chmod +x ./basenine_linux_${GOARCH}
RUN mv ./basenine_linux_${GOARCH} ./basenine RUN mv ./basenine_linux_${GOARCH} ./basenine
@@ -64,6 +64,8 @@ it('right side sanity test', function () {
}); });
}); });
serviceMapCheck();
checkIllegalFilter('invalid filter'); checkIllegalFilter('invalid filter');
checkFilter({ checkFilter({
@@ -188,7 +190,7 @@ function checkFilter(filterDetails){
const entriesForDeeperCheck = 5; const entriesForDeeperCheck = 5;
it(`checking the filter: ${name}`, function () { it(`checking the filter: ${name}`, function () {
cy.get('#total-entries').then(number => { cy.get('#total-entries').should('not.have.text', '0').then(number => {
const totalEntries = number.text(); const totalEntries = number.text();
// checks the hover on the last entry (the only one in DOM at the beginning) // checks the hover on the last entry (the only one in DOM at the beginning)
@@ -320,3 +322,42 @@ function checkOnlyLineNumberes(jsonItems, decodedText) {
cy.get(`${Cypress.env('bodyJsonClass')} >`).should('have.length', 1).and('have.text', decodedText); cy.get(`${Cypress.env('bodyJsonClass')} >`).should('have.length', 1).and('have.text', decodedText);
cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems) cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems)
} }
function serviceMapCheck() {
it('service map test', function () {
cy.intercept(`${Cypress.env('testUrl')}/servicemap/get`).as('serviceMapRequest');
cy.get('#total-entries').should('not.have.text', '0').then(() => {
cy.get('#total-entries').invoke('text').then(entriesNum => {
cy.get('[alt="service-map"]').click();
cy.wait('@serviceMapRequest').then(({response}) => {
const body = response.body;
const nodeParams = {
destination: 'httpbin.mizu-tests',
source: '127.0.0.1'
};
serviceMapAPICheck(body, parseInt(entriesNum), nodeParams);
cy.reload();
});
});
});
});
}
function serviceMapAPICheck(body, entriesNum, nodeParams) {
const {nodes, edges} = body;
expect(nodes.length).to.equal(Object.keys(nodeParams).length, `Expected nodes count`);
expect(edges.some(edge => edge.source.name === nodeParams.source)).to.be.true;
expect(edges.some(edge => edge.destination.name === nodeParams.destination)).to.be.true;
let count = 0;
edges.forEach(edge => {
count += edge.count;
if (edge.destination.name === nodeParams.destination) {
expect(edge.source.name).to.equal(nodeParams.source);
}
});
expect(count).to.equal(entriesNum);
}
+1 -1
View File
@@ -155,7 +155,7 @@ func TestAmqp(t *testing.T) {
defer conn.Close() defer conn.Close()
// Temporary fix for missing amqp entries // Temporary fix for missing amqp entries
time.Sleep(5 * time.Second) time.Sleep(10 * time.Second)
for i := 0; i < defaultEntriesCount/5; i++ { for i := 0; i < defaultEntriesCount/5; i++ {
ch, err := conn.Channel() ch, err := conn.Channel()
+2 -1
View File
@@ -150,9 +150,10 @@ func runCypressTests(t *testing.T, cypressRunCmd string) {
t.Logf("running command: %v", cypressCmd.String()) t.Logf("running command: %v", cypressCmd.String())
out, err := cypressCmd.Output() out, err := cypressCmd.Output()
if err != nil { if err != nil {
t.Errorf("%s", out) t.Errorf("error running cypress, error: %v", err)
return return
} }
t.Logf("%s", out) t.Logf("%s", out)
} }
+1 -1
View File
@@ -22,7 +22,7 @@ require (
github.com/ory/kratos-client-go v0.8.2-alpha.1 github.com/ory/kratos-client-go v0.8.2-alpha.1
github.com/patrickmn/go-cache v2.1.0+incompatible github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/stretchr/testify v1.7.0 github.com/stretchr/testify v1.7.0
github.com/up9inc/basenine/client/go v0.0.0-20220125035757-926e42208705 github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1
github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0 github.com/up9inc/mizu/tap v0.0.0
github.com/up9inc/mizu/tap/api v0.0.0 github.com/up9inc/mizu/tap/api v0.0.0
+2 -2
View File
@@ -853,8 +853,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ= github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/up9inc/basenine/client/go v0.0.0-20220125035757-926e42208705 h1:5LLhzv0cjb/F+dU0z3j8teVGjQInMYAocTyAZohKUwY= github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1 h1:0XN8s3HtwUBr9hbWRAFulFMsu1f2cabfJbwpz/sOoLA=
github.com/up9inc/basenine/client/go v0.0.0-20220125035757-926e42208705/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI= github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+7 -6
View File
@@ -56,11 +56,9 @@ const (
func main() { func main() {
logLevel := determineLogLevel() logLevel := determineLogLevel()
logger.InitLoggerStderrOnly(logLevel) logger.InitLoggerStd(logLevel)
flag.Parse() flag.Parse()
if err := config.LoadConfig(); err != nil {
logger.Log.Fatalf("Error loading config file %v", err)
}
app.LoadExtensions() app.LoadExtensions()
if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode { if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
@@ -139,7 +137,10 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engin
} }
func runInApiServerMode(namespace string) *gin.Engine { func runInApiServerMode(namespace string) *gin.Engine {
app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort) if err := config.LoadConfig(); err != nil {
logger.Log.Fatalf("Error loading config file %v", err)
}
app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel)
startTime = time.Now().UnixNano() / int64(time.Millisecond) startTime = time.Now().UnixNano() / int64(time.Millisecond)
api.StartResolving(namespace) api.StartResolving(namespace)
@@ -215,7 +216,7 @@ func enableExpFeatureIfNeeded() {
oas.GetOasGeneratorInstance().Start() oas.GetOasGeneratorInstance().Start()
} }
if config.Config.ServiceMap { if config.Config.ServiceMap {
servicemap.GetInstance().SetConfig(config.Config) servicemap.GetInstance().Enable()
} }
elastic.GetInstance().Configure(config.Config.Elastic) elastic.GetInstance().Configure(config.Config.Elastic)
} }
+11 -1
View File
@@ -140,7 +140,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
mizuEntry.Rules = rules mizuEntry.Rules = rules
} }
entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name, Id: mizuEntry.Id} entryWSource := oas.EntryWithSource{
Entry: *harEntry,
Source: mizuEntry.Source.Name,
Destination: mizuEntry.Destination.Name,
Id: mizuEntry.Id,
}
if entryWSource.Destination == "" {
entryWSource.Destination = mizuEntry.Destination.IP + ":" + mizuEntry.Destination.Port
}
oas.GetOasGeneratorInstance().PushEntry(&entryWSource) oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
} }
+21 -21
View File
@@ -30,6 +30,11 @@ type SocketConnection struct {
isTapper bool isTapper bool
} }
type WebSocketParams struct {
Query string `json:"query"`
EnableFullEntries bool `json:"enableFullEntries"`
}
var ( var (
websocketUpgrader = websocket.Upgrader{ websocketUpgrader = websocket.Upgrader{
ReadBufferSize: 1024, ReadBufferSize: 1024,
@@ -110,31 +115,26 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
logger.Log.Error(err) logger.Log.Error(err)
} }
out: var params WebSocketParams
for { for {
// params[0]: query _, msg, err := ws.ReadMessage()
// params[1]: enableFullEntries (empty: disable, non-empty: enable) if err != nil {
params := make([][]byte, 2) if _, ok := err.(*websocket.CloseError); ok {
for i := range params { logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
_, params[i], err = ws.ReadMessage() } else {
if err != nil { logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
if _, ok := err.(*websocket.CloseError); ok {
logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
} else {
logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
}
break out
} }
}
enableFullEntries := false break
if len(params[1]) > 0 {
enableFullEntries = true
} }
if !isTapper && !isQuerySet { if !isTapper && !isQuerySet {
query := string(params[0]) if err := json.Unmarshal(msg, &params); err != nil {
logger.Log.Errorf("Error: %v", socketId, err)
}
query := params.Query
err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query) err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
if err != nil { if err != nil {
toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{ toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
@@ -162,7 +162,7 @@ out:
err = json.Unmarshal(bytes, &entry) err = json.Unmarshal(bytes, &entry)
var message []byte var message []byte
if enableFullEntries { if params.EnableFullEntries {
message, _ = models.CreateFullEntryWebSocketMessage(entry) message, _ = models.CreateFullEntryWebSocketMessage(entry)
} else { } else {
base := tapApi.Summarize(entry) base := tapApi.Summarize(entry)
@@ -201,7 +201,7 @@ out:
connection.Query(query, data, meta) connection.Query(query, data, meta)
} else { } else {
eventHandlers.WebSocketMessage(socketId, params[0]) eventHandlers.WebSocketMessage(socketId, msg)
} }
} }
} }
+4 -12
View File
@@ -9,7 +9,6 @@ import (
"github.com/op/go-logging" "github.com/op/go-logging"
basenine "github.com/up9inc/basenine/client/go" basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/api" "github.com/up9inc/mizu/agent/pkg/api"
"github.com/up9inc/mizu/agent/pkg/config"
"github.com/up9inc/mizu/agent/pkg/controllers" "github.com/up9inc/mizu/agent/pkg/controllers"
"github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api" tapApi "github.com/up9inc/mizu/tap/api"
@@ -60,27 +59,21 @@ func LoadExtensions() {
return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
}) })
for _, extension := range Extensions {
logger.Log.Infof("Extension Properties: %+v", extension)
}
controllers.InitExtensionsMap(ExtensionsMap) controllers.InitExtensionsMap(ExtensionsMap)
} }
func ConfigureBasenineServer(host string, port string) { func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level) {
if !wait.New( if !wait.New(
wait.WithProto("tcp"), wait.WithProto("tcp"),
wait.WithWait(200*time.Millisecond), wait.WithWait(200*time.Millisecond),
wait.WithBreak(50*time.Millisecond), wait.WithBreak(50*time.Millisecond),
wait.WithDeadline(5*time.Second), wait.WithDeadline(5*time.Second),
wait.WithDebug(config.Config.LogLevel == logging.DEBUG), wait.WithDebug(logLevel == logging.DEBUG),
).Do([]string{fmt.Sprintf("%s:%s", host, port)}) { ).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
logger.Log.Panicf("Basenine is not available!") logger.Log.Panicf("Basenine is not available!")
} }
// Limit the database size to default 200MB if err := basenine.Limit(host, port, dbSize); err != nil {
err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
if err != nil {
logger.Log.Panicf("Error while limiting database size: %v", err) logger.Log.Panicf("Error while limiting database size: %v", err)
} }
@@ -88,8 +81,7 @@ func ConfigureBasenineServer(host string, port string) {
for _, extension := range Extensions { for _, extension := range Extensions {
macros := extension.Dissector.Macros() macros := extension.Dissector.Macros()
for macro, expanded := range macros { for macro, expanded := range macros {
err = basenine.Macro(host, port, macro, expanded) if err := basenine.Macro(host, port, macro, expanded); err != nil {
if err != nil {
logger.Log.Panicf("Error while adding a macro: %v", err) logger.Log.Panicf("Error while adding a macro: %v", err)
} }
} }
@@ -11,7 +11,6 @@ import (
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/up9inc/mizu/shared"
tapApi "github.com/up9inc/mizu/tap/api" tapApi "github.com/up9inc/mizu/tap/api"
) )
@@ -59,9 +58,7 @@ type ServiceMapControllerSuite struct {
func (s *ServiceMapControllerSuite) SetupTest() { func (s *ServiceMapControllerSuite) SetupTest() {
s.c = NewServiceMapController() s.c = NewServiceMapController()
s.c.service.SetConfig(&shared.MizuAgentConfig{ s.c.service.Enable()
ServiceMap: true,
})
s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp) s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
s.w = httptest.NewRecorder() s.w = httptest.NewRecorder()
+7 -1
View File
@@ -6,6 +6,7 @@ import (
"errors" "errors"
"io" "io"
"io/ioutil" "io/ioutil"
"net/url"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
@@ -139,7 +140,12 @@ func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
logger.Log.Debugf("Interesting: %s", entry.Request.URL) logger.Log.Debugf("Interesting: %s", entry.Request.URL)
} }
ews := EntryWithSource{Entry: *entry, Source: source, Id: uint(0)} u, err := url.Parse(entry.Request.URL)
if err != nil {
logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
}
ews := EntryWithSource{Entry: *entry, Source: source, Destination: u.Host, Id: uint(0)}
if isSync { if isSync {
GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right? GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
} else { } else {
+7 -6
View File
@@ -54,11 +54,11 @@ func (g *oasGenerator) runGeneretor() {
logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err) logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
} }
val, found := g.ServiceSpecs.Load(u.Host) val, found := g.ServiceSpecs.Load(entryWithSource.Destination)
var gen *SpecGen var gen *SpecGen
if !found { if !found {
gen = NewGen(u.Scheme + "://" + u.Host) gen = NewGen(u.Scheme + "://" + entryWithSource.Destination)
g.ServiceSpecs.Store(u.Host, gen) g.ServiceSpecs.Store(entryWithSource.Destination, gen)
} else { } else {
gen = val.(*SpecGen) gen = val.(*SpecGen)
} }
@@ -105,9 +105,10 @@ func newOasGenerator() *oasGenerator {
} }
type EntryWithSource struct { type EntryWithSource struct {
Source string Source string
Entry har.Entry Destination string
Id uint Entry har.Entry
Id uint
} }
type oasGenerator struct { type oasGenerator struct {
+1 -1
View File
@@ -41,7 +41,7 @@ func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
} }
func TestEntries(t *testing.T) { func TestEntries(t *testing.T) {
logger.InitLoggerStderrOnly(logging.INFO) logger.InitLoggerStd(logging.INFO)
files, err := getFiles("./test_artifacts/") files, err := getFiles("./test_artifacts/")
if err != nil { if err != nil {
t.Log(err) t.Log(err)
+6 -10
View File
@@ -3,7 +3,6 @@ package servicemap
import ( import (
"sync" "sync"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api" tapApi "github.com/up9inc/mizu/tap/api"
) )
@@ -26,13 +25,13 @@ func GetInstance() ServiceMap {
} }
type serviceMap struct { type serviceMap struct {
config *shared.MizuAgentConfig enabled bool
graph *graph graph *graph
entriesProcessed int entriesProcessed int
} }
type ServiceMap interface { type ServiceMap interface {
SetConfig(config *shared.MizuAgentConfig) Enable()
IsEnabled() bool IsEnabled() bool
NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol) NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol)
GetStatus() ServiceMapStatus GetStatus() ServiceMapStatus
@@ -46,7 +45,7 @@ type ServiceMap interface {
func newServiceMap() *serviceMap { func newServiceMap() *serviceMap {
return &serviceMap{ return &serviceMap{
config: nil, enabled: false,
entriesProcessed: 0, entriesProcessed: 0,
graph: newDirectedGraph(), graph: newDirectedGraph(),
} }
@@ -156,15 +155,12 @@ func (s *serviceMap) addEdge(u, v *entryData, p *tapApi.Protocol) {
s.entriesProcessed++ s.entriesProcessed++
} }
func (s *serviceMap) SetConfig(config *shared.MizuAgentConfig) { func (s *serviceMap) Enable() {
s.config = config s.enabled = true
} }
func (s *serviceMap) IsEnabled() bool { func (s *serviceMap) IsEnabled() bool {
if s.config != nil && s.config.ServiceMap { return s.enabled
return true
}
return false
} }
func (s *serviceMap) NewTCPEntry(src *tapApi.TCP, dst *tapApi.TCP, p *tapApi.Protocol) { func (s *serviceMap) NewTCPEntry(src *tapApi.TCP, dst *tapApi.TCP, p *tapApi.Protocol) {
+1 -4
View File
@@ -6,7 +6,6 @@ import (
"testing" "testing"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/up9inc/mizu/shared"
tapApi "github.com/up9inc/mizu/tap/api" tapApi "github.com/up9inc/mizu/tap/api"
) )
@@ -96,9 +95,7 @@ func (s *ServiceMapDisabledSuite) SetupTest() {
func (s *ServiceMapEnabledSuite) SetupTest() { func (s *ServiceMapEnabledSuite) SetupTest() {
s.instance = GetInstance() s.instance = GetInstance()
s.instance.SetConfig(&shared.MizuAgentConfig{ s.instance.Enable()
ServiceMap: true,
})
} }
func (s *ServiceMapDisabledSuite) TestServiceMapInstance() { func (s *ServiceMapDisabledSuite) TestServiceMapInstance() {
+1 -1
View File
@@ -10,7 +10,7 @@ curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin
**Mac** (AArch64/Apple M1 silicon) **Mac** (AArch64/Apple M1 silicon)
``` ```
curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_arm64 && chmod 755 mizu rm -f mizu && curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_arm64 && chmod 755 mizu
``` ```
**Linux** (x86-64) **Linux** (x86-64)
+10
View File
@@ -1,8 +1,11 @@
package cmd package cmd
import ( import (
"github.com/creasty/defaults"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/config/configStructs"
"github.com/up9inc/mizu/cli/telemetry" "github.com/up9inc/mizu/cli/telemetry"
"github.com/up9inc/mizu/shared/logger"
) )
var checkCmd = &cobra.Command{ var checkCmd = &cobra.Command{
@@ -17,4 +20,11 @@ var checkCmd = &cobra.Command{
func init() { func init() {
rootCmd.AddCommand(checkCmd) rootCmd.AddCommand(checkCmd)
defaultCheckConfig := configStructs.CheckConfig{}
if err := defaults.Set(&defaultCheckConfig); err != nil {
logger.Log.Debug(err)
}
checkCmd.Flags().Bool(configStructs.PreTapCheckName, defaultCheckConfig.PreTap, "Check pre-tap Mizu installation for potential problems")
} }
+94 -12
View File
@@ -3,6 +3,10 @@ package cmd
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/up9inc/mizu/shared"
rbac "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"regexp" "regexp"
"github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/apiserver"
@@ -14,7 +18,7 @@ import (
) )
func runMizuCheck() { func runMizuCheck() {
logger.Log.Infof("Mizu install checks\n===================") logger.Log.Infof("Mizu checks\n===================")
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() // cancel will be called when this function exits defer cancel() // cancel will be called when this function exits
@@ -25,17 +29,23 @@ func runMizuCheck() {
checkPassed = checkKubernetesVersion(kubernetesVersion) checkPassed = checkKubernetesVersion(kubernetesVersion)
} }
var isInstallCommand bool if config.Config.Check.PreTap {
if checkPassed { if checkPassed {
checkPassed, isInstallCommand = checkMizuMode(ctx, kubernetesProvider) checkPassed = checkK8sTapPermissions(ctx, kubernetesProvider)
} }
} else {
var isInstallCommand bool
if checkPassed {
checkPassed, isInstallCommand = checkMizuMode(ctx, kubernetesProvider)
}
if checkPassed { if checkPassed {
checkPassed = checkK8sResources(ctx, kubernetesProvider, isInstallCommand) checkPassed = checkK8sResources(ctx, kubernetesProvider, isInstallCommand)
} }
if checkPassed { if checkPassed {
checkPassed = checkServerConnection(kubernetesProvider) checkPassed = checkServerConnection(kubernetesProvider)
}
} }
if checkPassed { if checkPassed {
@@ -273,9 +283,81 @@ func checkResourceExist(resourceName string, resourceType string, exist bool, er
} else if !exist { } else if !exist {
logger.Log.Errorf("%v '%v' %v doesn't exist", fmt.Sprintf(uiUtils.Red, "✗"), resourceName, resourceType) logger.Log.Errorf("%v '%v' %v doesn't exist", fmt.Sprintf(uiUtils.Red, "✗"), resourceName, resourceType)
return false return false
} else {
logger.Log.Infof("%v '%v' %v exists", fmt.Sprintf(uiUtils.Green, "√"), resourceName, resourceType)
} }
logger.Log.Infof("%v '%v' %v exists", fmt.Sprintf(uiUtils.Green, "√"), resourceName, resourceType)
return true
}
func checkK8sTapPermissions(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
logger.Log.Infof("\nkubernetes-permissions\n--------------------")
var filePath string
if config.Config.IsNsRestrictedMode() {
filePath = "./examples/roles/permissions-ns-tap.yaml"
} else {
filePath = "./examples/roles/permissions-all-namespaces-tap.yaml"
}
data, err := shared.ReadFromFile(filePath)
if err != nil {
logger.Log.Errorf("%v error while checking kubernetes permissions, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
return false
}
obj, err := getDecodedObject(data)
if err != nil {
logger.Log.Errorf("%v error while checking kubernetes permissions, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
return false
}
var rules []rbac.PolicyRule
if config.Config.IsNsRestrictedMode() {
rules = obj.(*rbac.Role).Rules
} else {
rules = obj.(*rbac.ClusterRole).Rules
}
return checkPermissions(ctx, kubernetesProvider, rules)
}
func getDecodedObject(data []byte) (runtime.Object, error) {
decode := scheme.Codecs.UniversalDeserializer().Decode
obj, _, err := decode(data, nil, nil)
if err != nil {
return nil, err
}
return obj, nil
}
func checkPermissions(ctx context.Context, kubernetesProvider *kubernetes.Provider, rules []rbac.PolicyRule) bool {
permissionsExist := true
for _, rule := range rules {
for _, group := range rule.APIGroups {
for _, resource := range rule.Resources {
for _, verb := range rule.Verbs {
exist, err := kubernetesProvider.CanI(ctx, config.Config.MizuResourcesNamespace, resource, verb, group)
permissionsExist = checkPermissionExist(group, resource, verb, exist, err) && permissionsExist
}
}
}
}
return permissionsExist
}
func checkPermissionExist(group string, resource string, verb string, exist bool, err error) bool {
if err != nil {
logger.Log.Errorf("%v error checking permission for %v %v in group '%v', err: %v", fmt.Sprintf(uiUtils.Red, "✗"), verb, resource, group, err)
return false
} else if !exist {
logger.Log.Errorf("%v can't %v %v in group '%v'", fmt.Sprintf(uiUtils.Red, "✗"), verb, resource, group)
return false
}
logger.Log.Infof("%v can %v %v in group '%v'", fmt.Sprintf(uiUtils.Green, "√"), verb, resource, group)
return true return true
} }
+2 -2
View File
@@ -14,10 +14,10 @@ var installCmd = &cobra.Command{
logger.Log.Infof("This command has been deprecated, please use helm as described below.\n\n") logger.Log.Infof("This command has been deprecated, please use helm as described below.\n\n")
logger.Log.Infof("To install stable build of Mizu on your cluster using helm, run the following command:") logger.Log.Infof("To install stable build of Mizu on your cluster using helm, run the following command:")
logger.Log.Infof(" helm install mizu https://static.up9.com/mizu/helm --namespace=mizu --create-namespace\n\n") logger.Log.Infof(" helm install mizu mizu --repo https://static.up9.com/mizu/helm --namespace=mizu --create-namespace\n\n")
logger.Log.Infof("To install development build of Mizu on your cluster using helm, run the following command:") logger.Log.Infof("To install development build of Mizu on your cluster using helm, run the following command:")
logger.Log.Infof(" helm install mizu https://static.up9.com/mizu/helm-develop --namespace=mizu --create-namespace") logger.Log.Infof(" helm install mizu mizu --repo https://static.up9.com/mizu/helm-develop --namespace=mizu --create-namespace\n")
return nil return nil
}, },
+1
View File
@@ -22,6 +22,7 @@ const (
type ConfigStruct struct { type ConfigStruct struct {
Tap configStructs.TapConfig `yaml:"tap"` Tap configStructs.TapConfig `yaml:"tap"`
Check configStructs.CheckConfig `yaml:"check"`
Version configStructs.VersionConfig `yaml:"version"` Version configStructs.VersionConfig `yaml:"version"`
View configStructs.ViewConfig `yaml:"view"` View configStructs.ViewConfig `yaml:"view"`
Logs configStructs.LogsConfig `yaml:"logs"` Logs configStructs.LogsConfig `yaml:"logs"`
+9
View File
@@ -0,0 +1,9 @@
package configStructs
const (
PreTapCheckName = "pre-tap"
)
type CheckConfig struct {
PreTap bool `yaml:"pre-tap"`
}
+20
View File
@@ -0,0 +1,20 @@
package shared
import (
"io/ioutil"
"os"
)
func ReadFromFile(path string) ([]byte, error) {
reader, err := os.Open(path)
if err != nil {
return nil, err
}
data, err := ioutil.ReadAll(reader)
if err != nil {
return nil, err
}
return data, nil
}
+24 -21
View File
@@ -17,6 +17,7 @@ import (
"github.com/up9inc/mizu/shared/semver" "github.com/up9inc/mizu/shared/semver"
"github.com/up9inc/mizu/tap/api" "github.com/up9inc/mizu/tap/api"
v1 "k8s.io/api/apps/v1" v1 "k8s.io/api/apps/v1"
auth "k8s.io/api/authorization/v1"
core "k8s.io/api/core/v1" core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1" rbac "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -443,6 +444,26 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{}) return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
} }
func (provider *Provider) CanI(ctx context.Context, namespace string, resource string, verb string, group string) (bool, error) {
selfSubjectAccessReview := &auth.SelfSubjectAccessReview{
Spec: auth.SelfSubjectAccessReviewSpec{
ResourceAttributes: &auth.ResourceAttributes{
Namespace: namespace,
Resource: resource,
Verb: verb,
Group: group,
},
},
}
response, err := provider.clientSet.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, selfSubjectAccessReview, metav1.CreateOptions{})
if err != nil {
return false, err
}
return response.Status.Allowed, nil
}
func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) { func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) {
namespaceResource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{}) namespaceResource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(namespaceResource, err) return provider.doesResourceExist(namespaceResource, err)
@@ -829,7 +850,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
if tls { if tls {
mizuCmd = append(mizuCmd, "--tls") mizuCmd = append(mizuCmd, "--tls")
} }
if serviceMesh || tls { if serviceMesh || tls {
mizuCmd = append(mizuCmd, "--procfs", procfsMountPath) mizuCmd = append(mizuCmd, "--procfs", procfsMountPath)
} }
@@ -939,24 +960,6 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
sysfsVolumeMount := applyconfcore.VolumeMount().WithName(sysfsVolumeName).WithMountPath(sysfsMountPath).WithReadOnly(true) sysfsVolumeMount := applyconfcore.VolumeMount().WithName(sysfsVolumeName).WithMountPath(sysfsMountPath).WithReadOnly(true)
agentContainer.WithVolumeMounts(sysfsVolumeMount) agentContainer.WithVolumeMounts(sysfsVolumeMount)
volumeName := ConfigMapName
configMapVolume := applyconfcore.VolumeApplyConfiguration{
Name: &volumeName,
VolumeSourceApplyConfiguration: applyconfcore.VolumeSourceApplyConfiguration{
ConfigMap: &applyconfcore.ConfigMapVolumeSourceApplyConfiguration{
LocalObjectReferenceApplyConfiguration: applyconfcore.LocalObjectReferenceApplyConfiguration{
Name: &volumeName,
},
},
},
}
mountPath := shared.ConfigDirPath
configMapVolumeMount := applyconfcore.VolumeMountApplyConfiguration{
Name: &volumeName,
MountPath: &mountPath,
}
agentContainer.WithVolumeMounts(&configMapVolumeMount)
podSpec := applyconfcore.PodSpec() podSpec := applyconfcore.PodSpec()
podSpec.WithHostNetwork(true) podSpec.WithHostNetwork(true)
podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet) podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet)
@@ -967,7 +970,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
podSpec.WithContainers(agentContainer) podSpec.WithContainers(agentContainer)
podSpec.WithAffinity(affinity) podSpec.WithAffinity(affinity)
podSpec.WithTolerations(noExecuteToleration, noScheduleToleration) podSpec.WithTolerations(noExecuteToleration, noScheduleToleration)
podSpec.WithVolumes(&configMapVolume, procfsVolume, sysfsVolume) podSpec.WithVolumes(procfsVolume, sysfsVolume)
podTemplate := applyconfcore.PodTemplateSpec() podTemplate := applyconfcore.PodTemplateSpec()
podTemplate.WithLabels(map[string]string{ podTemplate.WithLabels(map[string]string{
@@ -981,7 +984,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName}) labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName})
applyOptions := metav1.ApplyOptions{ applyOptions := metav1.ApplyOptions{
Force: true, Force: true,
FieldManager: fieldManagerName, FieldManager: fieldManagerName,
} }
+16 -5
View File
@@ -29,10 +29,21 @@ func InitLogger(logPath string) {
logging.SetBackend(backend1Leveled, backend2Formatter) logging.SetBackend(backend1Leveled, backend2Formatter)
} }
func InitLoggerStderrOnly(level logging.Level) { func InitLoggerStd(level logging.Level) {
backend := logging.NewLogBackend(os.Stderr, "", 0) var backends []logging.Backend
backendFormatter := logging.NewBackendFormatter(backend, format)
logging.SetBackend(backendFormatter) stderrBackend := logging.NewLogBackend(os.Stderr, "", 0)
logging.SetLevel(level, "") stderrFormater := logging.NewBackendFormatter(stderrBackend, format)
stderrLeveled := logging.AddModuleLevel(stderrFormater)
stderrLeveled.SetLevel(logging.ERROR, "")
backends = append(backends, stderrLeveled)
if level >= logging.WARNING {
stdoutBackend := logging.NewLogBackend(os.Stdout, "", 0)
stdoutFormater := logging.NewBackendFormatter(stdoutBackend, format)
stdoutLeveled := logging.AddModuleLevel(stdoutFormater)
stdoutLeveled.SetLevel(level, "")
backends = append(backends, stdoutLeveled)
}
logging.SetBackend(backends...)
} }
-4
View File
@@ -27,10 +27,6 @@ var protocol api.Protocol = api.Protocol{
Priority: 1, Priority: 1,
} }
func init() {
log.Println("Initializing AMQP extension...")
}
type dissecting string type dissecting string
func (d dissecting) Register(extension *api.Extension) { func (d dissecting) Register(extension *api.Extension) {
-4
View File
@@ -76,10 +76,6 @@ const (
TypeHttpResponse TypeHttpResponse
) )
func init() {
log.Println("Initializing HTTP extension...")
}
type dissecting string type dissecting string
func (d dissecting) Register(extension *api.Extension) { func (d dissecting) Register(extension *api.Extension) {
-4
View File
@@ -25,10 +25,6 @@ var _protocol api.Protocol = api.Protocol{
Priority: 2, Priority: 2,
} }
func init() {
log.Println("Initializing Kafka extension...")
}
type dissecting string type dissecting string
func (d dissecting) Register(extension *api.Extension) { func (d dissecting) Register(extension *api.Extension) {
-4
View File
@@ -24,10 +24,6 @@ var protocol api.Protocol = api.Protocol{
Priority: 3, Priority: 3,
} }
func init() {
log.Println("Initializing Redis extension...")
}
type dissecting string type dissecting string
func (d dissecting) Register(extension *api.Extension) { func (d dissecting) Register(extension *api.Extension) {
+3 -1
View File
@@ -8,6 +8,7 @@ import {toast} from "react-toastify";
import {useRecoilValue} from "recoil"; import {useRecoilValue} from "recoil";
import focusedEntryIdAtom from "../recoil/focusedEntryId"; import focusedEntryIdAtom from "../recoil/focusedEntryId";
import Api from "../helpers/api"; import Api from "../helpers/api";
import queryAtom from "../recoil/query";
const useStyles = makeStyles(() => ({ const useStyles = makeStyles(() => ({
entryTitle: { entryTitle: {
@@ -82,6 +83,7 @@ const api = Api.getInstance();
export const EntryDetailed = () => { export const EntryDetailed = () => {
const focusedEntryId = useRecoilValue(focusedEntryIdAtom); const focusedEntryId = useRecoilValue(focusedEntryIdAtom);
const query = useRecoilValue(queryAtom);
const [entryData, setEntryData] = useState(null); const [entryData, setEntryData] = useState(null);
useEffect(() => { useEffect(() => {
@@ -89,7 +91,7 @@ export const EntryDetailed = () => {
setEntryData(null); setEntryData(null);
(async () => { (async () => {
try { try {
const entryData = await api.getEntry(focusedEntryId); const entryData = await api.getEntry(focusedEntryId, query);
setEntryData(entryData); setEntryData(entryData);
} catch (error) { } catch (error) {
if (error.response?.data?.type) { if (error.response?.data?.type) {
@@ -121,8 +121,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus}) => {
ws.current = new WebSocket(MizuWebsocketURL); ws.current = new WebSocket(MizuWebsocketURL);
ws.current.onopen = () => { ws.current.onopen = () => {
setWsConnection(WsConnectionStatus.Connected); setWsConnection(WsConnectionStatus.Connected);
ws.current.send(query); ws.current.send(JSON.stringify({"query": query, "enableFullEntries": false}));
ws.current.send("");
} }
ws.current.onclose = () => { ws.current.onclose = () => {
setWsConnection(WsConnectionStatus.Closed); setWsConnection(WsConnectionStatus.Closed);