Compare commits

..

10 Commits

Author SHA1 Message Date
Selton Fiuza
dea223bfe1 Feature/tra 3349 validation rules merged with develop (#148)
* Implemented validation rules, based on: https://up9.atlassian.net/browse/TRA-3349

* Color on Entry based on rules

* Background red/green based on rules

* Change flag --validation-rules to --test-rules

* rules tab UI updated

* rules tab font and background-color is changed for objects

* Merged with develop

* Fixed compilation issues.

* Renamed fullEntry -> harEntry where appropriate.

* Change green/red logic

* Update models.go

* Fix latency bug and alignment

* Merge Conflicts fix

* Working after merge

* Working on Nimrod comments

* Resolving conflicts

* Resolving conflicts

* Resolving conflicts

* Nimrod Comments pt.3

* Log Error on configmap creation if the user doesn't have permission.

* Checking configmap permission to ignore --test-rules

* Revert time for mizu to get ready

* Nimrod comments pt 4 && merge develop pt3

* Nimrod comments pt 4 && merge develop pt3

* Const rulePolicyPath and filename

Co-authored-by: Neim <elezin9@gmail.com>
Co-authored-by: nimrod-up9 <nimrod@up9.com>
2021-08-04 09:21:36 -03:00
gadotroee
06c8056443 Tapper stats in stats tracker (#166) 2021-08-04 12:51:51 +03:00
Igor Gov
d18f1f8316 Tapped pods report via endpoint instead of web socket (#164) 2021-08-04 10:41:33 +03:00
Igor Gov
f9202900ee No warning when mizu rbac exists (#163) 2021-08-04 08:41:00 +03:00
Igor Gov
9e34662511 Adding logs and fixing several issues (#162)
* Config grooming and several general fixes
2021-08-04 08:18:07 +03:00
gadotroee
1e726e381b Add option to memory profiling and limit page cache (#158) 2021-08-03 17:30:13 +03:00
RoyUP9
69a9deab4b Feature/tra 3474 config refactor (#155) 2021-08-03 17:23:52 +03:00
RamiBerm
f9396e01ca TRA-3415 ignore unready pods (#160)
* Update tapRunner.go and provider.go

* Update tapRunner.go

* Update tapRunner.go

* Update tapRunner.go

* Update tapRunner.go and provider.go

Co-authored-by: RamiBerm <rami.berman@up9.com>
2021-08-03 15:02:31 +03:00
Nimrod Gilboa Markevich
2d5b170406 Added a default SEM_VER (#159)
* Set default SEM_VER to 0.0.0.

* Build docker with both latest and semver tags.

* Use docker tag instead of building twice.

* Build and tag in one command.
2021-08-03 14:35:02 +03:00
Nimrod Gilboa Markevich
dc59fb6931 Added/edited description to Makefile commands. (#157)
* Added missing description to Makefile commands.

* Uppercase.
2021-08-03 11:41:02 +03:00
54 changed files with 1501 additions and 593 deletions

View File

@@ -19,15 +19,16 @@ help: ## This help.
TS_SUFFIX="$(shell date '+%s')"
GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
export SEM_VER?=0.0.0
ui: ## build UI
ui: ## Build UI.
@(cd ui; npm i ; npm run build; )
@ls -l ui/build
cli: # build CLI
cli: ## Build CLI.
@echo "building cli"; cd cli && $(MAKE) build
agent: ## build mizuagent server
agent: ## Build agent.
@(echo "building mizu agent .." )
@(cd agent; go build -o build/mizuagent main.go)
@ls -l agent/build
@@ -36,17 +37,17 @@ agent: ## build mizuagent server
# @(cd tap; go build -o build/tap ./src)
# @ls -l tap/build
docker: ## build Docker image
docker: ## Build Docker image.
@(echo "building docker image" )
./build-push-featurebranch.sh
push: push-docker push-cli ## build and publish Mizu docker image & CLI
push: push-docker push-cli ## Build and publish agent docker image & CLI.
push-docker:
push-docker: ## Build and publish agent docker image.
@echo "publishing Docker image .. "
./build-push-featurebranch.sh
push-cli:
push-cli: ## Build and publish CLI.
@echo "publishing CLI .. "
@cd cli; $(MAKE) build-all
@echo "publishing file ${OUTPUT_FILE} .."
@@ -55,17 +56,17 @@ push-cli:
gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts
clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
clean-ui:
clean-ui: ## Clean UI.
@(rm -rf ui/build ; echo "UI cleanup done" )
clean-agent:
clean-agent: ## Clean agent.
@(rm -rf agent/build ; echo "agent cleanup done" )
clean-cli:
clean-cli: ## Clean CLI.
@(cd cli; make clean ; echo "CLI cleanup done" )
clean-docker:
clean-docker:
@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )

View File

@@ -13,16 +13,17 @@ require (
github.com/go-playground/validator/v10 v10.5.0
github.com/google/martian v2.1.0+incompatible
github.com/gorilla/websocket v1.4.2
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
go.mongodb.org/mongo-driver v1.5.1
gorm.io/driver/sqlite v1.1.4
gorm.io/gorm v1.21.8
k8s.io/api v0.21.0
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
github.com/patrickmn/go-cache v2.1.0+incompatible
)
replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

View File

@@ -287,6 +287,8 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

View File

@@ -85,7 +85,7 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
app := gin.Default()
app.GET("/echo", func(c *gin.Context) {
c.String(http.StatusOK, "Hello, World 👋!")
c.String(http.StatusOK, "Here is Mizu agent")
})
eventHandlers := api.RoutesEventHandlers{
@@ -95,9 +95,10 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
app.Use(static.ServeRoot("/", "./site"))
app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
routes.WebSocketRoutes(app, &eventHandlers)
api.WebSocketRoutes(app, &eventHandlers)
routes.EntriesRoutes(app)
routes.MetadataRoutes(app)
routes.StatusRoutes(app)
routes.NotFoundRoute(app)
utils.StartServer(app)

View File

@@ -5,10 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/google/martian/har"
"github.com/romana/rlog"
"github.com/up9inc/mizu/tap"
"go.mongodb.org/mongo-driver/bson/primitive"
"mizuserver/pkg/holder"
"net/url"
"os"
@@ -17,6 +13,11 @@ import (
"strings"
"time"
"github.com/google/martian/har"
"github.com/romana/rlog"
"github.com/up9inc/mizu/tap"
"go.mongodb.org/mongo-driver/bson/primitive"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/resolver"
@@ -88,9 +89,9 @@ func startReadingFiles(workingDir string) {
for _, entry := range inputHar.Log.Entries {
time.Sleep(time.Millisecond * 250)
connectionInfo := &tap.ConnectionInfo{
ClientIP: fileInfo.Name(),
ClientIP: fileInfo.Name(),
ClientPort: "",
ServerIP: "",
ServerIP: "",
ServerPort: "",
IsOutgoing: false,
}
@@ -118,7 +119,6 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
}
}
func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
entryBytes, _ := json.Marshal(entry)
serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
@@ -167,8 +167,10 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
return
}
baseEntry.Rules = models.RunValidationRulesState(*entry, serviceName)
baseEntry.Latency = entry.Timings.Receive
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
broadcastToBrowserClients(baseEntryBytes)
BroadcastToBrowserClients(baseEntryBytes)
}
func getServiceNameFromUrl(inputUrl string) (string, string) {
@@ -196,6 +198,5 @@ func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int {
sizeBytes += 8 // SizeBytes bytes
sizeBytes += 1 // IsOutgoing bytes
return sizeBytes
}

View File

@@ -1,4 +1,4 @@
package routes
package api
import (
"errors"
@@ -18,10 +18,10 @@ type EventHandlers interface {
}
type SocketConnection struct {
connection *websocket.Conn
lock *sync.Mutex
connection *websocket.Conn
lock *sync.Mutex
eventHandlers EventHandlers
isTapper bool
isTapper bool
}
var websocketUpgrader = websocket.Upgrader{
@@ -91,7 +91,7 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
}
var db = debounce.NewDebouncer(time.Second * 5, func() {
var db = debounce.NewDebouncer(time.Second*5, func() {
fmt.Println("Successfully sent to socket")
})
@@ -102,7 +102,7 @@ func SendToSocket(socketId int, message []byte) error {
}
var sent = false
time.AfterFunc(time.Second * 5, func() {
time.AfterFunc(time.Second*5, func() {
if !sent {
fmt.Println("Socket timed out")
socketCleanup(socketId, socketObj)

View File

@@ -3,26 +3,26 @@ package api
import (
"encoding/json"
"fmt"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/up9"
"sync"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/routes"
"mizuserver/pkg/up9"
"sync"
)
var browserClientSocketUUIDs = make([]int, 0)
var socketListLock = sync.Mutex{}
type RoutesEventHandlers struct {
routes.EventHandlers
EventHandlers
SocketHarOutChannel chan<- *tap.OutputChannelItem
}
func init() {
go up9.UpdateAnalyzeStatus(broadcastToBrowserClients)
go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
}
func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
@@ -47,15 +47,14 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
}
}
func broadcastToBrowserClients(message []byte) {
func BroadcastToBrowserClients(message []byte) {
for _, socketId := range browserClientSocketUUIDs {
go func(socketId int) {
err := routes.SendToSocket(socketId, message)
err := SendToSocket(socketId, message)
if err != nil {
fmt.Printf("error sending message to socket ID %d: %v", socketId, err)
}
}(socketId)
}
}
@@ -81,7 +80,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
broadcastToBrowserClients(message)
BroadcastToBrowserClients(message)
}
case shared.WebsocketMessageTypeOutboundLink:
var outboundLinkMessage models.WebsocketOutboundLinkMessage
@@ -116,7 +115,7 @@ func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
rlog.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
} else {
fmt.Printf("Broadcasting outboundlink message %s\n", string(marshaledMessage))
broadcastToBrowserClients(marshaledMessage)
BroadcastToBrowserClients(marshaledMessage)
}
}

View File

@@ -3,17 +3,19 @@ package controllers
import (
"encoding/json"
"fmt"
"github.com/gin-gonic/gin"
"github.com/google/martian/har"
"github.com/romana/rlog"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/up9"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"net/http"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/google/martian/har"
"github.com/romana/rlog"
)
func GetEntries(c *gin.Context) {
@@ -217,7 +219,14 @@ func GetEntry(c *gin.Context) {
"msg": "Can't get entry details",
})
}
c.JSON(http.StatusOK, fullEntry)
fullEntryWithPolicy := models.FullEntryWithPolicy{}
if err := models.GetEntry(&entryData, &fullEntryWithPolicy); err != nil {
c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": true,
"msg": "Can't get entry details",
})
}
c.JSON(http.StatusOK, fullEntryWithPolicy)
}
func DeleteAllEntries(c *gin.Context) {
@@ -241,3 +250,15 @@ func GetGeneralStats(c *gin.Context) {
database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
c.JSON(http.StatusOK, result)
}
func GetTappingStatus(c *gin.Context) {
c.JSON(http.StatusOK, providers.TapStatus)
}
func AnalyzeInformation(c *gin.Context) {
c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
}
func GetRecentTLSLinks(c *gin.Context) {
c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
}

View File

@@ -1,20 +1,32 @@
package controllers
import (
"encoding/json"
"github.com/gin-gonic/gin"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/api"
"mizuserver/pkg/providers"
"mizuserver/pkg/up9"
"mizuserver/pkg/validation"
"net/http"
)
func GetTappingStatus(c *gin.Context) {
c.JSON(http.StatusOK, providers.TapStatus)
}
func AnalyzeInformation(c *gin.Context) {
c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
}
func GetRecentTLSLinks(c *gin.Context) {
c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
func PostTappedPods(c *gin.Context) {
tapStatus := &shared.TapStatus{}
if err := c.Bind(tapStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if err := validation.Validate(tapStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
rlog.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
providers.TapStatus.Pods = tapStatus.Pods
message := shared.CreateWebSocketStatusMessage(*tapStatus)
if jsonBytes, err := json.Marshal(message); err != nil {
rlog.Errorf("Could not Marshal message %v\n", err)
} else {
api.BroadcastToBrowserClients(jsonBytes)
}
}

View File

@@ -2,11 +2,14 @@ package models
import (
"encoding/json"
"mizuserver/pkg/rules"
"mizuserver/pkg/utils"
"time"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/utils"
"time"
)
type DataUnmarshaler interface {
@@ -33,19 +36,33 @@ type MizuEntry struct {
ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"`
EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"`
}
type BaseEntryDetails struct {
Id string `json:"id,omitempty"`
Url string `json:"url,omitempty"`
RequestSenderIp string `json:"requestSenderIp,omitempty"`
Service string `json:"service,omitempty"`
Path string `json:"path,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Method string `json:"method,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
Id string `json:"id,omitempty"`
Url string `json:"url,omitempty"`
RequestSenderIp string `json:"requestSenderIp,omitempty"`
Service string `json:"service,omitempty"`
Path string `json:"path,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Method string `json:"method,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
Latency int64 `json:"latency,omitempty"`
Rules ApplicableRules `json:"rules,omitempty"`
}
type ApplicableRules struct {
Latency int64 `json:"latency,omitempty"`
Status bool `json:"status,omitempty"`
}
func NewApplicableRules(status bool, latency int64) ApplicableRules {
ar := ApplicableRules{}
ar.Status = status
ar.Latency = latency
return ar
}
type FullEntryDetails struct {
@@ -101,11 +118,6 @@ func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
return nil
}
type EntryData struct {
Entry string `json:"entry,omitempty"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
}
type EntriesFilter struct {
Limit int `query:"limit" validate:"required,min=1,max=200"`
Operator string `query:"operator" validate:"required,oneof='lt' 'gt'"`
@@ -186,3 +198,27 @@ type ExtendedCreator struct {
*har.Creator
Source *string `json:"_source"`
}
type FullEntryWithPolicy struct {
RulesMatched []rules.RulesMatched `json:"rulesMatched,omitempty"`
Entry har.Entry `json:"entry"`
Service string `json:"service"`
}
func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error {
if err := json.Unmarshal([]byte(entry.Entry), &fewp.Entry); err != nil {
return err
}
_, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service)
fewp.RulesMatched = resultPolicyToSend
fewp.Service = entry.Service
return nil
}
func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules {
numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
statusPolicyToSend, latency := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
ar := NewApplicableRules(statusPolicyToSend, latency)
return ar
}

View File

@@ -0,0 +1,12 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
)
func StatusRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/status")
routeGroup.POST("/tappedPods", controllers.PostTappedPods)
}

110
agent/pkg/rules/models.go Normal file
View File

@@ -0,0 +1,110 @@
package rules
import (
"encoding/json"
"fmt"
"reflect"
"regexp"
"strings"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
jsonpath "github.com/yalp/jsonpath"
)
type RulesMatched struct {
Matched bool `json:"matched"`
Rule shared.RulePolicy `json:"rule"`
}
func appendRulesMatched(rulesMatched []RulesMatched, matched bool, rule shared.RulePolicy) []RulesMatched {
return append(rulesMatched, RulesMatched{Matched: matched, Rule: rule})
}
func ValidatePath(URLFromRule string, URL string) bool {
if URLFromRule != "" {
matchPath, err := regexp.MatchString(URLFromRule, URL)
if err != nil || !matchPath {
return false
}
}
return true
}
func ValidateService(serviceFromRule string, service string) bool {
if serviceFromRule != "" {
matchService, err := regexp.MatchString(serviceFromRule, service)
if err != nil || !matchService {
return false
}
}
return true
}
func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched) {
enforcePolicy, _ := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
var resultPolicyToSend []RulesMatched
for _, rule := range enforcePolicy.Rules {
if !ValidatePath(rule.Path, harEntry.Request.URL) || !ValidateService(rule.Service, service) {
continue
}
if rule.Type == "json" {
var bodyJsonMap interface{}
if err := json.Unmarshal(harEntry.Response.Content.Text, &bodyJsonMap); err != nil {
continue
}
out, err := jsonpath.Read(bodyJsonMap, rule.Key)
if err != nil || out == nil {
continue
}
var matchValue bool
if reflect.TypeOf(out).Kind() == reflect.String {
matchValue, err = regexp.MatchString(rule.Value, out.(string))
if err != nil {
continue
}
} else {
val := fmt.Sprint(out)
matchValue, err = regexp.MatchString(rule.Value, val)
if err != nil {
continue
}
}
resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
} else if rule.Type == "header" {
for j := range harEntry.Response.Headers {
matchKey, err := regexp.MatchString(rule.Key, harEntry.Response.Headers[j].Name)
if err != nil {
continue
}
if matchKey {
matchValue, err := regexp.MatchString(rule.Value, harEntry.Response.Headers[j].Value)
if err != nil {
continue
}
resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
}
}
} else {
resultPolicyToSend = appendRulesMatched(resultPolicyToSend, true, rule)
}
}
return len(enforcePolicy.Rules), resultPolicyToSend
}
func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64) {
if len(rulesMatched) == 0 {
return false, 0
}
for _, rule := range rulesMatched {
if rule.Matched == false {
return false, -1
}
}
for _, rule := range rulesMatched {
if strings.ToLower(rule.Rule.Type) == "latency" {
return true, rule.Rule.Latency
}
}
return true, -1
}

View File

@@ -5,7 +5,9 @@ SERVER_NAME=mizu
GCP_PROJECT=up9-docker-hub
REPOSITORY=gcr.io/$GCP_PROJECT
GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
DOCKER_TAGGED_BUILD=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH:latest
SEM_VER=${SEM_VER=0.0.0}
DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
then
@@ -13,8 +15,12 @@ then
exit 1
fi
echo "building $DOCKER_TAGGED_BUILD"
docker build -t "$DOCKER_TAGGED_BUILD" --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
echo "building ${DOCKER_TAGGED_BUILDS[@]}"
DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
echo pushing to "$REPOSITORY"
docker push "$DOCKER_TAGGED_BUILD"
for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
do
echo pushing "$DOCKER_TAG"
docker push "$DOCKER_TAG"
done

View File

@@ -3,6 +3,7 @@ COMMIT_HASH=$(shell git rev-parse HEAD)
GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
BUILD_TIMESTAMP=$(shell date +%s)
export SEM_VER?=0.0.0
.PHONY: help
.DEFAULT_GOAL := help
@@ -13,7 +14,7 @@ help: ## This help.
install:
go install mizu.go
build: ## build mizu CLI binary (select platform via GOOS / GOARCH env variables)
build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
-X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
-X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
@@ -21,7 +22,7 @@ build: ## build mizu CLI binary (select platform via GOOS / GOARCH env variables
-o bin/mizu_$(SUFFIX) mizu.go
(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
build-all: ## build for all supported platforms
build-all: ## Build for all supported platforms.
@echo "Compiling for every OS and Platform"
@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" > bin/README.md
@$(MAKE) build GOOS=darwin GOARCH=amd64
@@ -35,6 +36,6 @@ build-all: ## build for all supported platforms
@echo "---------"
@find ./bin -ls
clean: ## clean all build artifacts
clean: ## Clean all build artifacts.
go clean
rm -rf ./bin/*

View File

@@ -8,17 +8,24 @@ import (
"io/ioutil"
)
var outputFileName string
var regenerateFile bool
var configCmd = &cobra.Command{
Use: "config",
Short: "Generate example config file to stdout",
Short: "Generate config with default values",
RunE: func(cmd *cobra.Command, args []string) error {
template := mizu.GetTemplateConfig()
if outputFileName != "" {
template, err := mizu.GetConfigWithDefaults()
if err != nil {
mizu.Log.Errorf("Failed generating config with defaults %v", err)
return nil
}
if regenerateFile {
data := []byte(template)
_ = ioutil.WriteFile(outputFileName, data, 0644)
mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, outputFileName)))
if err := ioutil.WriteFile(mizu.GetConfigFilePath(), data, 0644); err != nil {
mizu.Log.Errorf("Failed writing config %v", err)
return nil
}
mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, mizu.GetConfigFilePath())))
} else {
mizu.Log.Debugf("Writing template config.\n%v", template)
fmt.Printf("%v", template)
@@ -29,6 +36,5 @@ var configCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(configCmd)
configCmd.Flags().StringVarP(&outputFileName, "file", "f", "", "Save content to local file")
configCmd.Flags().BoolVarP(&regenerateFile, "regenerate", "r", false, fmt.Sprintf("Regenerate the config file with default values %s", mizu.GetConfigFilePath()))
}

View File

@@ -1,30 +1,23 @@
package cmd
import (
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
type MizuFetchOptions struct {
FromTimestamp int64
ToTimestamp int64
Directory string
MizuPort uint16
}
var mizuFetchOptions = MizuFetchOptions{}
var fetchCmd = &cobra.Command{
Use: "fetch",
Short: "Download recorded traffic to files",
RunE: func(cmd *cobra.Command, args []string) error {
go mizu.ReportRun("fetch", mizuTapOptions)
if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
go mizu.ReportRun("fetch", mizu.Config.Fetch)
if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.MizuPort); err != nil {
return err
} else if !isCompatible {
return nil
}
RunMizuFetch(&mizuFetchOptions)
RunMizuFetch()
return nil
},
}
@@ -32,8 +25,11 @@ var fetchCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(fetchCmd)
fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries")
fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries")
fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu")
defaultFetchConfig := configStructs.FetchConfig{}
defaults.Set(&defaultFetchConfig)
fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
fetchCmd.Flags().Uint16P(configStructs.MizuPortFetchName, "p", defaultFetchConfig.MizuPort, "Custom port for mizu")
}

View File

@@ -15,9 +15,9 @@ import (
"strings"
)
func RunMizuFetch(fetch *MizuFetchOptions) {
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(fetch.MizuPort)
resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, fetch.FromTimestamp, fetch.ToTimestamp))
func RunMizuFetch() {
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.MizuPort)
resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp))
if err != nil {
log.Fatal(err)
}
@@ -33,8 +33,8 @@ func RunMizuFetch(fetch *MizuFetchOptions) {
if err != nil {
log.Fatal(err)
}
_ = Unzip(zipReader, fetch.Directory)
_ = Unzip(zipReader, mizu.Config.Fetch.Directory)
}
func Unzip(reader *zip.Reader, dest string) error {

View File

@@ -7,26 +7,23 @@ import (
"github.com/up9inc/mizu/cli/mizu"
)
var commandLineFlags []string
var rootCmd = &cobra.Command{
Use: "mizu",
Short: "A web traffic viewer for kubernetes",
Long: `A web traffic viewer for kubernetes
Further info is available at https://github.com/up9inc/mizu`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := mizu.InitConfig(commandLineFlags); err != nil {
if err := mizu.InitConfig(cmd); err != nil {
mizu.Log.Errorf("Invalid config, Exit %s", err)
return errors.New(fmt.Sprintf("%v", err))
}
prettifiedConfig := mizu.GetConfigStr()
mizu.Log.Debugf("Final Config: %s", prettifiedConfig)
return nil
},
}
func init() {
rootCmd.PersistentFlags().StringSliceVar(&commandLineFlags, "set", []string{}, "Override values using --set")
rootCmd.PersistentFlags().StringSlice(mizu.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", mizu.SetCommandName))
}
// Execute adds all child commands to the root command and sets flags appropriately.

View File

@@ -2,39 +2,15 @@ package cmd
import (
"errors"
"fmt"
"os"
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared/units"
"os"
"regexp"
"strings"
)
type MizuTapOptions struct {
GuiPort uint16
Namespace string
AllNamespaces bool
Analysis bool
AnalysisDestination string
KubeConfigPath string
MizuImage string
PlainTextFilterRegexes []string
TapOutgoing bool
HideHealthChecks bool
MaxEntriesDBSizeBytes int64
SleepIntervalSec uint16
DisableRedaction bool
}
var mizuTapOptions = &MizuTapOptions{}
var direction string
var humanMaxEntriesDBSize string
var regex *regexp.Regexp
const maxEntriesDBSizeFlagName = "max-entries-db-size"
const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
var tapCmd = &cobra.Command{
@@ -43,53 +19,31 @@ var tapCmd = &cobra.Command{
Long: `Record the ingoing traffic of a kubernetes pod.
Supported protocols are HTTP and gRPC.`,
RunE: func(cmd *cobra.Command, args []string) error {
go mizu.ReportRun("tap", mizuTapOptions)
RunMizuTap(regex, mizuTapOptions)
go mizu.ReportRun("tap", mizu.Config.Tap)
RunMizuTap()
return nil
},
PreRunE: func(cmd *cobra.Command, args []string) error {
mizu.Log.Debugf("Getting params")
mizuTapOptions.AnalysisDestination = mizu.GetString(mizu.ConfigurationKeyAnalyzingDestination)
mizuTapOptions.SleepIntervalSec = uint16(mizu.GetInt(mizu.ConfigurationKeyUploadInterval))
mizuTapOptions.MizuImage = mizu.GetString(mizu.ConfigurationKeyMizuImage)
mizu.Log.Debugf(uiUtils.PrettyJson(mizuTapOptions))
if len(args) == 0 {
return errors.New("POD REGEX argument is required")
if len(args) == 1 {
mizu.Config.Tap.PodRegexStr = args[0]
} else if len(args) > 1 {
return errors.New("unexpected number of arguments")
}
var compileErr error
regex, compileErr = regexp.Compile(args[0])
if compileErr != nil {
return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], compileErr))
if err := mizu.Config.Tap.Validate(); err != nil {
return err
}
var parseHumanDataSizeErr error
mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
}
mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", mizu.Config.Tap.HumanMaxEntriesDBSize)
directionLowerCase := strings.ToLower(direction)
if directionLowerCase == "any" {
mizuTapOptions.TapOutgoing = true
} else if directionLowerCase == "in" {
mizuTapOptions.TapOutgoing = false
} else {
return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", direction))
}
if mizuTapOptions.Analysis {
if mizu.Config.Tap.Analysis {
mizu.Log.Infof(analysisMessageToConfirm)
if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
mizu.Log.Infof("You can always run mizu without analysis, aborting")
os.Exit(0)
}
}
return nil
},
}
@@ -97,14 +51,19 @@ Supported protocols are HTTP and gRPC.`,
func init() {
rootCmd.AddCommand(tapCmd)
tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")
tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb")
tapCmd.Flags().BoolVar(&mizuTapOptions.DisableRedaction, "no-redact", false, "Disables redaction of potentially sensitive request/response headers and body values")
defaultTapConfig := configStructs.TapConfig{}
defaults.Set(&defaultTapConfig)
tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringP(configStructs.NamespaceTapName, "n", defaultTapConfig.Namespace, "Namespace selector")
tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
tapCmd.Flags().StringP(configStructs.KubeConfigPathTapName, "k", defaultTapConfig.KubeConfigPath, "Path to kube-config file")
tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "hides requests with kube-probe or prometheus user-agent headers")
tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "override the default max entries db size of 200mb")
tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
tapCmd.Flags().String(configStructs.EnforcePolicyFile, "", "Yaml file with policy rules")
}

View File

@@ -1,17 +1,10 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/clientcmd"
"log"
"net/http"
"net/url"
"os"
@@ -19,6 +12,17 @@ import (
"regexp"
"syscall"
"time"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
yaml "gopkg.in/yaml.v3"
core "k8s.io/api/core/v1"
errors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/clientcmd"
)
var mizuServiceAccountExists bool
@@ -31,13 +35,21 @@ const (
var currentlyTappedPods []core.Pod
func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
mizuApiFilteringOptions, err := getMizuApiFilteringOptions(tappingOptions)
func RunMizuTap() {
mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
if err != nil {
return
}
var mizuValidationRules string
if mizu.Config.Tap.EnforcePolicyFile != "" {
mizuValidationRules, err = readValidationRules(mizu.Config.Tap.EnforcePolicyFile)
if err != nil {
mizu.Log.Infof("error: %v", err)
return
}
}
kubernetesProvider, err := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.Tap.KubeConfigPath)
if err != nil {
if clientcmd.IsEmptyConfig(err) {
mizu.Log.Infof(uiUtils.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
@@ -53,22 +65,21 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel() // cancel will be called when this function exits
targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil {
mizu.Log.Infof("Error listing pods: %v", err)
return
} else {
currentlyTappedPods = matchingPods
}
targetNamespace := getNamespace(kubernetesProvider)
var namespacesStr string
if targetNamespace != mizu.K8sAllNamespaces {
namespacesStr = fmt.Sprintf("namespace \"%s\"", targetNamespace)
} else {
namespacesStr = "all namespaces"
}
mizu.CheckNewerVersion()
mizu.Log.Infof("Tapping pods in %s", namespacesStr)
if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace); err != nil {
mizu.Log.Infof("Error listing pods: %v", err)
return
}
if len(currentlyTappedPods) == 0 {
var suggestionStr string
if targetNamespace != mizu.K8sAllNamespaces {
@@ -77,38 +88,60 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
mizu.Log.Infof("Did not find any pods matching the regex argument%s", suggestionStr)
}
if mizu.Config.Tap.DryRun {
return
}
nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
if err != nil {
return
}
if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions, mizuApiFilteringOptions); err != nil {
if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
return
}
urlReadyChan := make(chan string)
mizu.CheckNewerVersion()
go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions, urlReadyChan) // TODO convert this to job for built in pod ttl or have the running app handle this
go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions, urlReadyChan)
go syncApiStatus(ctx, cancel, tappingOptions)
go createProxyToApiServerPod(ctx, kubernetesProvider, cancel)
go watchPodsForTapping(ctx, kubernetesProvider, cancel)
//block until exit signal or error
waitForFinish(ctx, cancel)
}
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
func readValidationRules(file string) (string, error) {
rules, err := shared.DecodeEnforcePolicy(file)
if err != nil {
return "", err
}
newContent, _ := yaml.Marshal(&rules)
return string(newContent), nil
}
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error {
if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
return err
}
if err := createMizuApiServer(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
return err
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
return err
}
if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
return err
}
return nil
}
func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error {
err := kubernetesProvider.ApplyConfigMap(ctx, mizu.ResourcesNamespace, mizu.ConfigMapName, data)
if err != nil {
fmt.Printf("Error creating mizu configmap: %v\n", err)
}
return nil
}
@@ -116,12 +149,13 @@ func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Pro
_, err := kubernetesProvider.CreateNamespace(ctx, mizu.ResourcesNamespace)
if err != nil {
mizu.Log.Infof("Error creating Namespace %s: %v", mizu.ResourcesNamespace, err)
return err
}
return err
mizu.Log.Debugf("Successfully creating Namespace %s", mizu.ResourcesNamespace)
return nil
}
func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
var err error
mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
@@ -131,27 +165,30 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
} else {
serviceAccountName = ""
}
_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, tappingOptions.MizuImage, serviceAccountName, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.Config.MizuImage, serviceAccountName, mizuApiFilteringOptions, mizu.Config.Tap.MaxEntriesDBSizeBytes())
if err != nil {
mizu.Log.Infof("Error creating mizu %s pod: %v", mizu.ApiServerPodName, err)
return err
}
mizu.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
if err != nil {
mizu.Log.Infof("Error creating mizu %s service: %v", mizu.ApiServerPodName, err)
return err
}
mizu.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
return nil
}
func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) {
func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
var compiledRegexSlice []*shared.SerializableRegexp
if tappingOptions.PlainTextFilterRegexes != nil && len(tappingOptions.PlainTextFilterRegexes) > 0 {
if mizu.Config.Tap.PlainTextFilterRegexes != nil && len(mizu.Config.Tap.PlainTextFilterRegexes) > 0 {
compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
for _, regexStr := range mizu.Config.Tap.PlainTextFilterRegexes {
compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
if err != nil {
mizu.Log.Infof("Regex %s is invalid: %v", regexStr, err)
@@ -161,10 +198,10 @@ func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.Traffic
}
}
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks, DisableRedaction: tappingOptions.DisableRedaction}, nil
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil
}
func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
if len(nodeToTappedPodIPMap) > 0 {
var serviceAccountName string
if mizuServiceAccountExists {
@@ -177,19 +214,20 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
ctx,
mizu.ResourcesNamespace,
mizu.TapperDaemonSetName,
tappingOptions.MizuImage,
mizu.Config.MizuImage,
mizu.TapperPodName,
fmt.Sprintf("%s.%s.svc.cluster.local", apiServerService.Name, apiServerService.Namespace),
nodeToTappedPodIPMap,
serviceAccountName,
tappingOptions.TapOutgoing,
mizu.Config.Tap.TapOutgoing(),
); err != nil {
mizu.Log.Infof("Error creating mizu tapper daemonset: %v", err)
return err
}
mizu.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
} else {
if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
mizu.Log.Infof("Error deleting mizu tapper daemonset: %v", err)
mizu.Log.Errorf("Error deleting mizu tapper daemonset: %v", err)
return err
}
}
@@ -232,53 +270,76 @@ func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
}
}
func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions, urlReadyChan chan string) {
targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
func reportTappedPods() {
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.MizuPort)
tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl)
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), podRegex)
podInfos := make([]shared.PodInfo, 0)
for _, pod := range currentlyTappedPods {
podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
}
tapStatus := shared.TapStatus{Pods: podInfos}
if jsonValue, err := json.Marshal(tapStatus); err != nil {
mizu.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err)
} else {
if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err)
} else if response.StatusCode != 200 {
mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode)
} else {
mizu.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
}
}
}
func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
targetNamespace := getNamespace(kubernetesProvider)
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), mizu.Config.Tap.PodRegex())
restartTappers := func() {
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil {
mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err)
err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace)
if err != nil {
mizu.Log.Errorf("Error getting pods by regex: %s (%v,%+v)", err, err, err)
cancel()
} else {
currentlyTappedPods = matchingPods
}
if !changeFound {
mizu.Log.Debugf("Nothing changed update tappers not needed")
return
}
reportTappedPods()
nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
if err != nil {
mizu.Log.Infof("Error building node to ips map: %s (%v,%+v)", err, err, err)
mizu.Log.Errorf("Error building node to ips map: %s (%v,%+v)", err, err, err)
cancel()
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
mizu.Log.Infof("Error updating daemonset: %s (%v,%+v)", err, err, err)
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
mizu.Log.Errorf("Error updating daemonset: %s (%v,%+v)", err, err, err)
cancel()
}
}
restartTappersDebouncer := debounce.NewDebouncer(updateTappersDelay, restartTappers)
timer := time.AfterFunc(time.Second*10, func() {
mizu.Log.Debugf("Waiting for URL...")
mizu.Log.Infof("Mizu is available at http://%s", <-urlReadyChan)
})
for {
select {
case newTarget := <-added:
mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", newTarget.Name))
timer.Reset(time.Second * 2)
case removedTarget := <-removed:
mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedTarget.Name))
timer.Reset(time.Second * 2)
case pod := <-added:
mizu.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case modifiedTarget := <-modified:
case pod := <-removed:
mizu.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case pod := <-modified:
mizu.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
// Act only if the modified pod has already obtained an IP address.
// After filtering for IPs, on a normal pod restart this includes the following events:
// - Pod deletion
// - Pod reaches start state
// - Pod reaches ready state
// Ready/unready transitions might also trigger this event.
if modifiedTarget.Status.PodIP != "" {
if pod.Status.PodIP != "" {
restartTappersDebouncer.SetOn()
}
@@ -292,7 +353,53 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
}
}
func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions, urlReadyChan chan string) {
func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespace string) (error, bool) {
changeFound := false
if matchingPods, err := kubernetesProvider.GetAllRunningPodsMatchingRegex(ctx, mizu.Config.Tap.PodRegex(), targetNamespace); err != nil {
mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err)
return err, false
} else {
addedPods, removedPods := getPodArrayDiff(currentlyTappedPods, matchingPods)
for _, addedPod := range addedPods {
changeFound = true
mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
}
for _, removedPod := range removedPods {
changeFound = true
mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
}
currentlyTappedPods = matchingPods
}
return nil, changeFound
}
func getPodArrayDiff(oldPods []core.Pod, newPods []core.Pod) (added []core.Pod, removed []core.Pod) {
added = getMissingPods(newPods, oldPods)
removed = getMissingPods(oldPods, newPods)
return added, removed
}
//returns pods present in pods1 array and missing in pods2 array
func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
missingPods := make([]core.Pod, 0)
for _, pod1 := range pods1 {
var found = false
for _, pod2 := range pods2 {
if pod1.UID == pod2.UID {
found = true
break
}
}
if !found {
missingPods = append(missingPods, pod1)
}
}
return missingPods
}
func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
isPodReady := false
@@ -301,54 +408,58 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
select {
case <-ctx.Done():
return
case <-added:
mizu.Log.Debugf("Got agent pod added event")
continue
case <-removed:
mizu.Log.Infof("%s removed", mizu.ApiServerPodName)
cancel()
return
case modifiedPod := <-modified:
if modifiedPod.Status.Phase == "Running" && !isPodReady {
mizu.Log.Debugf("Got agent pod modified event, status phase: %v", modifiedPod.Status.Phase)
if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
isPodReady = true
go func() {
err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
if err != nil {
mizu.Log.Infof("Error occurred while running k8s proxy %v", err)
mizu.Log.Errorf("Error occurred while running k8s proxy %v", err)
cancel()
}
}()
mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort))
time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
requestForAnalysis()
reportTappedPods()
}
urlReadyChan <- kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort)
time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
requestForAnalysis(tappingOptions)
case <-timeAfter:
if !isPodReady {
mizu.Log.Errorf("error: %s pod was not ready in time", mizu.ApiServerPodName)
mizu.Log.Errorf("Error: %s pod was not ready in time", mizu.ApiServerPodName)
cancel()
}
case <-errorChan:
mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.ResourcesNamespace)
cancel()
}
}
}
func requestForAnalysis(tappingOptions *MizuTapOptions) {
if !tappingOptions.Analysis {
func requestForAnalysis() {
if !mizu.Config.Tap.Analysis {
return
}
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort)
urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalysisDestination), tappingOptions.SleepIntervalSec)
u, err := url.ParseRequestURI(urlPath)
if err != nil {
log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err))
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)
urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(mizu.Config.Tap.AnalysisDestination), mizu.Config.Tap.SleepIntervalSec)
u, parseErr := url.ParseRequestURI(urlPath)
if parseErr != nil {
mizu.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
}
mizu.Log.Debugf("Sending get request to %v", u.String())
if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
mizu.Log.Infof("error sending upload entries req, status code: %v, err: %v", response.StatusCode, err)
if response, requestErr := http.Get(u.String()); requestErr != nil {
mizu.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
} else if response.StatusCode != 200 {
mizu.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
} else {
mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
}
@@ -362,7 +473,7 @@ func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.P
}
if !mizuRBACExists {
err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
if err != nil {
if err != nil && !errors.IsAlreadyExists(err) {
mizu.Log.Infof("warning: could not create mizu rbac resources %v", err)
return false
}
@@ -396,33 +507,11 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
}
}
func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort))
controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
if err != nil {
mizu.Log.Infof("error establishing control socket connection %s", err)
cancel()
}
for {
select {
case <-ctx.Done():
return
default:
err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
if err != nil {
mizu.Log.Debugf("error Sending message via control socket %v, error: %s", controlSocketStr, err)
}
time.Sleep(10 * time.Second)
}
}
}
func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string {
if tappingOptions.AllNamespaces {
func getNamespace(kubernetesProvider *kubernetes.Provider) string {
if mizu.Config.Tap.AllNamespaces {
return mizu.K8sAllNamespaces
} else if len(tappingOptions.Namespace) > 0 {
return tappingOptions.Namespace
} else if len(mizu.Config.Tap.Namespace) > 0 {
return mizu.Config.Tap.Namespace
} else {
return kubernetesProvider.CurrentNamespace()
}

View File

@@ -1,24 +1,21 @@
package cmd
import (
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"strconv"
"time"
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
type MizuVersionOptions struct {
DebugInfo bool
}
var mizuVersionOptions = &MizuVersionOptions{}
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version info",
RunE: func(cmd *cobra.Command, args []string) error {
go mizu.ReportRun("version", mizuVersionOptions)
if mizuVersionOptions.DebugInfo {
go mizu.ReportRun("version", mizu.Config.Version)
if mizu.Config.Version.DebugInfo {
timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
@@ -33,6 +30,9 @@ var versionCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(versionCmd)
versionCmd.Flags().BoolVarP(&mizuVersionOptions.DebugInfo, "debug", "d", false, "Provide all information about version")
defaultVersionConfig := configStructs.VersionConfig{}
defaults.Set(&defaultVersionConfig)
versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
}

View File

@@ -1,28 +1,23 @@
package cmd
import (
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
type MizuViewOptions struct {
GuiPort uint16
KubeConfigPath string
}
var mizuViewOptions = &MizuViewOptions{}
var viewCmd = &cobra.Command{
Use: "view",
Short: "Open GUI in browser",
RunE: func(cmd *cobra.Command, args []string) error {
go mizu.ReportRun("view", mizuViewOptions)
if isCompatible, err := mizu.CheckVersionCompatibility(mizuViewOptions.GuiPort); err != nil {
go mizu.ReportRun("view", mizu.Config.View)
if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.View.GuiPort); err != nil {
return err
} else if !isCompatible {
return nil
}
runMizuView(mizuViewOptions)
runMizuView()
return nil
},
}
@@ -30,6 +25,9 @@ var viewCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(viewCmd)
viewCmd.Flags().Uint16VarP(&mizuViewOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
viewCmd.Flags().StringVarP(&mizuViewOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
defaultViewConfig := configStructs.ViewConfig{}
defaults.Set(&defaultViewConfig)
viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
viewCmd.Flags().StringP(configStructs.KubeConfigPathViewName, "k", defaultViewConfig.KubeConfigPath, "Path to kube-config file")
}

View File

@@ -3,15 +3,16 @@ package cmd
import (
"context"
"fmt"
"net/http"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"k8s.io/client-go/tools/clientcmd"
"net/http"
)
func runMizuView(mizuViewOptions *MizuViewOptions) {
kubernetesProvider, err := kubernetes.NewProvider(mizuViewOptions.KubeConfigPath)
func runMizuView() {
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
if err != nil {
if clientcmd.IsEmptyConfig(err) {
mizu.Log.Infof("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'")
@@ -35,16 +36,16 @@ func runMizuView(mizuViewOptions *MizuViewOptions) {
return
}
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort)
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort)
_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
if err == nil {
mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizuViewOptions.GuiPort)
mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizu.Config.View.GuiPort)
return
}
mizu.Log.Infof("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort))
err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort))
err = kubernetes.StartProxy(kubernetesProvider, mizu.Config.View.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
if err != nil {
mizu.Log.Infof("Error occured while running k8s proxy %v", err)
}

View File

@@ -3,10 +3,12 @@ module github.com/up9inc/mizu/cli
go 1.16
require (
github.com/creasty/defaults v1.5.1
github.com/google/go-github/v37 v37.0.0
github.com/gorilla/websocket v1.4.2
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
github.com/up9inc/mizu/shared v0.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/api v0.21.2

View File

@@ -82,6 +82,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM=
github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -215,6 +217,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -408,6 +411,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

View File

@@ -6,15 +6,16 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/homedir"
"os"
"path/filepath"
"regexp"
"strconv"
"github.com/up9inc/mizu/cli/mizu"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/homedir"
"github.com/up9inc/mizu/shared"
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
@@ -130,6 +131,10 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace
if err != nil {
return nil, err
}
configMapVolumeName := &core.ConfigMapVolumeSource{}
configMapVolumeName.Name = mizu.ConfigMapName
configMapOptional := true
configMapVolumeName.Optional = &configMapOptional
cpuLimit, err := resource.ParseQuantity("750m")
if err != nil {
@@ -160,7 +165,13 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace
Name: podName,
Image: podImage,
ImagePullPolicy: core.PullAlways,
Command: []string{"./mizuagent", "--api-server"},
VolumeMounts: []core.VolumeMount{
{
Name: mizu.ConfigMapName,
MountPath: shared.RulePolicyPath,
},
},
Command: []string{"./mizuagent", "--api-server"},
Env: []core.EnvVar{
{
Name: shared.HostModeEnvVar,
@@ -187,6 +198,14 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace
},
},
},
Volumes: []core.Volume{
{
Name: mizu.ConfigMapName,
VolumeSource: core.VolumeSource{
ConfigMap: configMapVolumeName,
},
},
},
DNSPolicy: core.DNSClusterFirstWithHostNet,
TerminationGracePeriodSeconds: new(int64),
},
@@ -370,6 +389,15 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string,
return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error {
if isFound, err := provider.CheckConfigMapExists(ctx, namespace, configMapName); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{})
}
func (provider *Provider) CheckNamespaceExists(ctx context.Context, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
@@ -472,7 +500,53 @@ func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace st
return false, nil
}
func (provider *Provider) CheckConfigMapExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().ConfigMaps(namespace).List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) ApplyConfigMap(ctx context.Context, namespace string, configMapName string, data string) error {
if data == "" {
return nil
}
configMapData := make(map[string]string, 0)
configMapData[shared.RulePolicyFileName] = data
configMap := &core.ConfigMap{
TypeMeta: metav1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: configMapName,
Namespace: namespace,
},
Data: configMapData,
}
_, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{})
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
return fmt.Errorf("User not authorized to create configmap, --test-rules will be ignored")
}
}
return err
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
mizu.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
if len(nodeToTappedPodIPMap) == 0 {
return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
}
@@ -493,12 +567,11 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
mizuCmd = append(mizuCmd, "--anydirection")
}
privileged := true
agentContainer := applyconfcore.Container()
agentContainer.WithName(tapperPodName)
agentContainer.WithImage(podImage)
agentContainer.WithImagePullPolicy(core.PullAlways)
agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged))
agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
agentContainer.WithCommand(mizuCmd...)
agentContainer.WithEnv(
applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
@@ -587,14 +660,14 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
return err
}
func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
func (provider *Provider) GetAllRunningPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
}
matchingPods := make([]core.Pod, 0)
for _, pod := range pods.Items {
if regex.MatchString(pod.Name) {
if regex.MatchString(pod.Name) && isPodRunning(&pod) {
matchingPods = append(matchingPods, pod)
}
}
@@ -635,3 +708,7 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
},
)
}
func isPodRunning(pod *core.Pod) bool {
return pod.Status.Phase == core.PodRunning
}

View File

@@ -3,219 +3,198 @@ package mizu
import (
"errors"
"fmt"
"github.com/up9inc/mizu/cli/uiUtils"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
"path"
"reflect"
"strconv"
"strings"
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/up9inc/mizu/cli/uiUtils"
"gopkg.in/yaml.v3"
)
const separator = "="
var configObj = map[string]interface{}{}
type CommandLineFlag struct {
CommandLineName string
YamlHierarchyName string
DefaultValue interface{}
}
const (
ConfigurationKeyAnalyzingDestination = "tap.dest"
ConfigurationKeyUploadInterval = "tap.uploadInterval"
ConfigurationKeyMizuImage = "mizuImage"
ConfigurationKeyTelemetry = "telemetry"
Separator = "="
SetCommandName = "set"
)
var allowedSetFlags = []CommandLineFlag{
{
CommandLineName: "dest",
YamlHierarchyName: ConfigurationKeyAnalyzingDestination,
DefaultValue: "up9.app",
// TODO: maybe add short description that we can show
},
{
CommandLineName: "uploadInterval",
YamlHierarchyName: ConfigurationKeyUploadInterval,
DefaultValue: 10,
},
{
CommandLineName: "mizuImage",
YamlHierarchyName: ConfigurationKeyMizuImage,
DefaultValue: fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer),
},
{
CommandLineName: "telemetry",
YamlHierarchyName: ConfigurationKeyTelemetry,
DefaultValue: true,
},
}
var Config = ConfigStruct{}
func GetString(key string) string {
return fmt.Sprintf("%v", getValueFromMergedConfig(key))
}
func GetBool(key string) bool {
stringVal := GetString(key)
Log.Debugf("Found string value %v", stringVal)
val, err := strconv.ParseBool(stringVal)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf( "Invalid value %v for key %s, expected bool", stringVal, key))
os.Exit(1)
func InitConfig(cmd *cobra.Command) error {
if err := defaults.Set(&Config); err != nil {
return err
}
return val
}
func GetInt(key string) int {
stringVal := GetString(key)
Log.Debugf("Found string value %v", stringVal)
if err := mergeConfigFile(); err != nil {
Log.Errorf("Could not load config file, error %v", err)
Log.Fatalf("You can regenerate the file using `mizu config -r` or just remove it %v", GetConfigFilePath())
}
val, err := strconv.Atoi(stringVal)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected int", stringVal, key))
os.Exit(1)
}
return val
}
cmd.Flags().Visit(initFlag)
finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
func InitConfig(commandLineValues []string) error {
Log.Debugf("Merging default values")
mergeDefaultValues()
Log.Debugf("Merging config file values")
if err1 := mergeConfigFile(); err1 != nil {
Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid config file\n"))
return err1
}
Log.Debugf("Merging command line values")
if err2 := mergeCommandLineFlags(commandLineValues); err2 != nil {
Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid commanad argument\n"))
return err2
}
finalConfigPrettified, _ := uiUtils.PrettyJson(configObj)
Log.Debugf("Merged all config successfully\n Final config: %v", finalConfigPrettified)
return nil
}
func GetTemplateConfig() string {
templateConfig := map[string]interface{}{}
for _, allowedFlag := range allowedSetFlags {
addToConfigObj(allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue, templateConfig)
func GetConfigWithDefaults() (string, error) {
defaultConf := ConfigStruct{}
if err := defaults.Set(&defaultConf); err != nil {
return "", err
}
prettifiedConfig, _ := uiUtils.PrettyYaml(templateConfig)
return prettifiedConfig
return uiUtils.PrettyYaml(defaultConf)
}
func GetConfigStr() string {
val, _ := uiUtils.PrettyYaml(configObj)
return val
}
func getValueFromMergedConfig(key string) interface{} {
if a, ok := configObj[key]; ok {
return a
}
return nil
}
func mergeDefaultValues() {
for _, allowedFlag := range allowedSetFlags {
Log.Debugf("Setting %v to %v", allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue)
configObj[allowedFlag.YamlHierarchyName] = allowedFlag.DefaultValue
}
func GetConfigFilePath() string {
return path.Join(getMizuFolderPath(), "config.yaml")
}
func mergeConfigFile() error {
Log.Debugf("Merging mizu config file values")
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
return nil
}
reader, openErr := os.Open(path.Join(home, ".mizu", "config.yaml"))
reader, openErr := os.Open(GetConfigFilePath())
if openErr != nil {
return nil
}
buf, readErr := ioutil.ReadAll(reader)
if readErr != nil {
return readErr
}
m := make(map[string]interface{})
if err := yaml.Unmarshal(buf, &m); err != nil {
if err := yaml.Unmarshal(buf, &Config); err != nil {
return err
}
for k, v := range m {
addToConfig(k, v)
}
Log.Debugf("Found config file, merged to default options")
return nil
}
func addToConfig(prefix string, value interface{}) {
typ := reflect.TypeOf(value).Kind()
if typ == reflect.Map {
for k1, v1 := range value.(map[string]interface{}) {
addToConfig(fmt.Sprintf("%s.%s", prefix, k1), v1)
}
} else {
validateConfigFileKey(prefix)
configObj[prefix] = value
func initFlag(f *pflag.Flag) {
configElem := reflect.ValueOf(&Config).Elem()
sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
if !isSliceValue {
mergeFlagValue(configElem, f.Name, f.Value.String())
return
}
if f.Name == SetCommandName {
if setError := mergeSetFlag(sliceValue.GetSlice()); setError != nil {
Log.Infof(uiUtils.Red, "Invalid set argument")
}
return
}
mergeFlagValues(configElem, f.Name, sliceValue.GetSlice())
}
func mergeCommandLineFlags(commandLineValues []string) error {
Log.Debugf("Merging Command line flags")
for _, e := range commandLineValues {
if !strings.Contains(e, separator) {
return errors.New(fmt.Sprintf("invalid set argument %s", e))
func mergeSetFlag(setValues []string) error {
configElem := reflect.ValueOf(&Config).Elem()
for _, setValue := range setValues {
if !strings.Contains(setValue, Separator) {
return errors.New(fmt.Sprintf("invalid set argument %s", setValue))
}
split := strings.SplitN(e, separator, 2)
split := strings.SplitN(setValue, Separator, 2)
if len(split) != 2 {
return errors.New(fmt.Sprintf("invalid set argument %s", e))
}
setFlagKey, argumentValue := split[0], split[1]
argumentNameInConfig, err := flagFromAllowed(setFlagKey)
if err != nil {
return err
return errors.New(fmt.Sprintf("invalid set argument %s", setValue))
}
configObj[argumentNameInConfig] = argumentValue
argumentKey, argumentValue := split[0], split[1]
mergeFlagValue(configElem, argumentKey, argumentValue)
}
return nil
}
func flagFromAllowed(setFlagKey string) (string, error) {
for _, allowedFlag := range allowedSetFlags {
if strings.ToLower(allowedFlag.CommandLineName) == strings.ToLower(setFlagKey) {
return allowedFlag.YamlHierarchyName, nil
}
}
return "", errors.New(fmt.Sprintf("invalid set argument %s", setFlagKey))
}
func mergeFlagValue(currentElem reflect.Value, flagKey string, flagValue string) {
for i := 0; i < currentElem.NumField(); i++ {
currentField := currentElem.Type().Field(i)
currentFieldByName := currentElem.FieldByName(currentField.Name)
func validateConfigFileKey(configFileKey string) {
for _, allowedFlag := range allowedSetFlags {
if allowedFlag.YamlHierarchyName == configFileKey {
if currentField.Type.Kind() == reflect.Struct {
mergeFlagValue(currentFieldByName, flagKey, flagValue)
continue
}
if currentField.Tag.Get("yaml") != flagKey {
continue
}
flagValueKind := currentField.Type.Kind()
parsedValue, err := getParsedValue(flagValueKind, flagValue)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected %s", flagValue, flagKey, flagValueKind))
return
}
currentFieldByName.Set(parsedValue)
}
Log.Info(fmt.Sprintf("Unknown argument: %s. Exit", configFileKey))
os.Exit(1)
}
func addToConfigObj(key string, value interface{}, configObj map[string]interface{}) {
typ := reflect.TypeOf(value).Kind()
if typ != reflect.Map {
if strings.Contains(key, ".") {
split := strings.SplitN(key, ".", 2)
firstLevelKey := split[0]
if _, ok := configObj[firstLevelKey]; !ok {
configObj[firstLevelKey] = map[string]interface{}{}
}
addToConfigObj(split[1], value, configObj[firstLevelKey].(map[string]interface{}))
} else {
configObj[key] = value
func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []string) {
for i := 0; i < currentElem.NumField(); i++ {
currentField := currentElem.Type().Field(i)
currentFieldByName := currentElem.FieldByName(currentField.Name)
if currentField.Type.Kind() == reflect.Struct {
mergeFlagValues(currentFieldByName, flagKey, flagValues)
continue
}
if currentField.Tag.Get("yaml") != flagKey {
continue
}
flagValueKind := currentField.Type.Elem().Kind()
parsedValues := reflect.MakeSlice(reflect.SliceOf(currentField.Type.Elem()), 0, 0)
for _, flagValue := range flagValues {
parsedValue, err := getParsedValue(flagValueKind, flagValue)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected %s", flagValue, flagKey, flagValueKind))
return
}
parsedValues = reflect.Append(parsedValues, parsedValue)
}
currentFieldByName.Set(parsedValues)
}
}
func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
switch kind {
case reflect.String:
return reflect.ValueOf(value), nil
case reflect.Bool:
boolArgumentValue, err := strconv.ParseBool(value)
if err != nil {
break
}
return reflect.ValueOf(boolArgumentValue), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
intArgumentValue, err := strconv.ParseInt(value, 10, 64)
if err != nil {
break
}
return reflect.ValueOf(intArgumentValue), nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
if err != nil {
break
}
return reflect.ValueOf(uintArgumentValue), nil
}
return reflect.ValueOf(nil), errors.New("value to parse does not match type")
}

20
cli/mizu/configStruct.go Normal file
View File

@@ -0,0 +1,20 @@
package mizu
import (
"fmt"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
type ConfigStruct struct {
Tap configStructs.TapConfig `yaml:"tap"`
Fetch configStructs.FetchConfig `yaml:"fetch"`
Version configStructs.VersionConfig `yaml:"version"`
View configStructs.ViewConfig `yaml:"view"`
MizuImage string `yaml:"mizu-image"`
Telemetry bool `yaml:"telemetry" default:"true"`
}
func (config *ConfigStruct) SetDefaults() {
config.MizuImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer)
}

View File

@@ -0,0 +1,15 @@
package configStructs
const (
DirectoryFetchName = "directory"
FromTimestampFetchName = "from"
ToTimestampFetchName = "to"
MizuPortFetchName = "port"
)
type FetchConfig struct {
Directory string `yaml:"directory" default:"."`
FromTimestamp int `yaml:"from" default:"0"`
ToTimestamp int `yaml:"to" default:"0"`
MizuPort uint16 `yaml:"port" default:"8899"`
}

View File

@@ -0,0 +1,81 @@
package configStructs
import (
"errors"
"fmt"
"regexp"
"strings"
"github.com/up9inc/mizu/shared/units"
)
const (
GuiPortTapName = "gui-port"
NamespaceTapName = "namespace"
AnalysisTapName = "analysis"
AllNamespacesTapName = "all-namespaces"
KubeConfigPathTapName = "kube-config"
PlainTextFilterRegexesTapName = "regex-masking"
HideHealthChecksTapName = "hide-healthchecks"
DisableRedactionTapName = "no-redact"
HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
DirectionTapName = "direction"
DryRunTapName = "dry-run"
EnforcePolicyFile = "test-rules"
)
type TapConfig struct {
AnalysisDestination string `yaml:"dest" default:"up9.app"`
SleepIntervalSec int `yaml:"upload-interval" default:"10"`
PodRegexStr string `yaml:"regex" default:".*"`
GuiPort uint16 `yaml:"gui-port" default:"8899"`
Namespace string `yaml:"namespace"`
Analysis bool `yaml:"analysis" default:"false"`
AllNamespaces bool `yaml:"all-namespaces" default:"false"`
KubeConfigPath string `yaml:"kube-config"`
PlainTextFilterRegexes []string `yaml:"regex-masking"`
HideHealthChecks bool `yaml:"hide-healthchecks" default:"false"`
DisableRedaction bool `yaml:"no-redact" default:"false"`
HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"`
Direction string `yaml:"direction" default:"in"`
DryRun bool `yaml:"dry-run" default:"false"`
EnforcePolicyFile string `yaml:"test-rules"`
}
func (config *TapConfig) PodRegex() *regexp.Regexp {
podRegex, _ := regexp.Compile(config.PodRegexStr)
return podRegex
}
func (config *TapConfig) TapOutgoing() bool {
directionLowerCase := strings.ToLower(config.Direction)
if directionLowerCase == "any" {
return true
}
return false
}
func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
return maxEntriesDBSizeBytes
}
func (config *TapConfig) Validate() error {
_, compileErr := regexp.Compile(config.PodRegexStr)
if compileErr != nil {
return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
}
_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
}
directionLowerCase := strings.ToLower(config.Direction)
if directionLowerCase != "any" && directionLowerCase != "in" {
return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
}
return nil
}

View File

@@ -0,0 +1,9 @@
package configStructs
const (
DebugInfoVersionName = "debug"
)
type VersionConfig struct {
DebugInfo bool `yaml:"debug" default:"false"`
}

View File

@@ -0,0 +1,11 @@
package configStructs
const (
GuiPortViewName = "gui-port"
KubeConfigPathViewName = "kube-config"
)
type ViewConfig struct {
GuiPort uint16 `yaml:"gui-port" default:"8899"`
KubeConfigPath string `yaml:"kube-config"`
}

View File

@@ -1,5 +1,10 @@
package mizu
import (
"os"
"path"
)
var (
SemVer = "0.0.1"
Branch = "develop"
@@ -17,4 +22,13 @@ const (
ServiceAccountName = "mizu-service-account"
TapperDaemonSetName = "mizu-tapper-daemon-set"
TapperPodName = "mizu-tapper"
ConfigMapName = "mizu-policy"
)
func getMizuFolderPath() string {
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
return ""
}
return path.Join(home, ".mizu")
}

View File

@@ -14,10 +14,9 @@ var format = logging.MustStringFormatter(
)
func InitLogger() {
homeDirPath, _ := os.UserHomeDir()
mizuDirPath := path.Join(homeDirPath, ".mizu")
mizuDirPath := getMizuFolderPath()
if err := os.MkdirAll(mizuDirPath, os.ModePerm); err != nil {
panic(fmt.Sprintf("Failed creating .mizu dir: %v, err %v", mizuDirPath, err))
panic(fmt.Sprintf("Failed creating mizu dir: %v, err %v", mizuDirPath, err))
}
logPath := path.Join(mizuDirPath, "log.log")
f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
@@ -35,5 +34,6 @@ func InitLogger() {
logging.SetBackend(backend1Leveled, backend2Formatter)
Log.Debugf("\n\n\n")
Log.Debugf("Running mizu version %v", SemVer)
}

View File

@@ -10,15 +10,11 @@ import (
const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
func ReportRun(cmd string, args interface{}) {
if !GetBool(ConfigurationKeyTelemetry) {
if !Config.Telemetry {
Log.Debugf("not reporting due to config value")
return
}
if Branch != "main" {
Log.Debugf("reporting only on main branch")
return
}
argsBytes, _ := json.Marshal(args)
argsMap := map[string]string{
"telemetry_type": "execution",
@@ -26,6 +22,7 @@ func ReportRun(cmd string, args interface{}) {
"args": string(argsBytes),
"component": "mizu_cli",
"BuildTimestamp": BuildTimestamp,
"Branch": Branch,
"version": SemVer}
argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))

View File

@@ -4,14 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/google/go-github/v37/github"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/semver"
"io/ioutil"
"net/http"
"net/url"
"time"
"github.com/google/go-github/v37/github"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/semver"
)
func getApiVersion(port uint16) (string, error) {

View File

@@ -6,4 +6,6 @@ const (
NodeNameEnvVar = "NODE_NAME"
TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
MaxEntriesDBSizeBytesEnvVar = "MAX_ENTRIES_DB_BYTES"
RulePolicyPath = "/app/enforce-policy/"
RulePolicyFileName = "enforce-policy.yaml"
)

View File

@@ -3,7 +3,8 @@ module github.com/up9inc/mizu/shared
go 1.16
require (
github.com/google/martian v2.1.0+incompatible // indirect
github.com/gorilla/websocket v1.4.2
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect
github.com/docker/go-units v0.4.0
)

View File

@@ -1,4 +1,8 @@
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=

View File

@@ -1,5 +1,13 @@
package shared
import (
"fmt"
"io/ioutil"
"strings"
yaml "gopkg.in/yaml.v3"
)
type WebSocketMessageType string
const (
@@ -32,7 +40,7 @@ type WebSocketStatusMessage struct {
}
type TapStatus struct {
Pods []PodInfo `json:"pods"`
Pods []PodInfo `json:"pods"`
TLSLinks []TLSLinkInfo `json:"tlsLinks"`
}
@@ -76,3 +84,75 @@ type VersionResponse struct {
SemVer string `json:"semver"`
}
type RulesPolicy struct {
Rules []RulePolicy `yaml:"rules"`
}
type RulePolicy struct {
Type string `yaml:"type"`
Service string `yaml:"service"`
Path string `yaml:"path"`
Method string `yaml:"method"`
Key string `yaml:"key"`
Value string `yaml:"value"`
Latency int64 `yaml:"latency"`
Name string `yaml:"name"`
}
func (r *RulePolicy) validateType() bool {
permitedTypes := []string{"json", "header", "latency"}
_, found := Find(permitedTypes, r.Type)
if !found {
fmt.Printf("\nRule with name %s will be ignored. Err: only json, header and latency types are supported on rule definition.\n", r.Name)
}
if strings.ToLower(r.Type) == "latency" {
if r.Latency == 0 {
fmt.Printf("\nRule with name %s will be ignored. Err: when type=latency, the field Latency should be specified and have a value >= 1\n\n", r.Name)
found = false
}
}
return found
}
func (rules *RulesPolicy) ValidateRulesPolicy() []int {
invalidIndex := make([]int, 0)
for i := range rules.Rules {
validated := rules.Rules[i].validateType()
if !validated {
invalidIndex = append(invalidIndex, i)
}
}
return invalidIndex
}
func (rules *RulesPolicy) RemoveRule(idx int) {
rules.Rules = append(rules.Rules[:idx], rules.Rules[idx+1:]...)
}
func Find(slice []string, val string) (int, bool) {
for i, item := range slice {
if item == val {
return i, true
}
}
return -1, false
}
func DecodeEnforcePolicy(path string) (RulesPolicy, error) {
content, err := ioutil.ReadFile(path)
enforcePolicy := RulesPolicy{}
if err != nil {
return enforcePolicy, err
}
err = yaml.Unmarshal([]byte(content), &enforcePolicy)
if err != nil {
return enforcePolicy, err
}
invalidIndex := enforcePolicy.ValidateRulesPolicy()
if len(invalidIndex) != 0 {
for i := range invalidIndex {
enforcePolicy.RemoveRule(invalidIndex[i])
}
}
return enforcePolicy, nil
}

View File

@@ -1,6 +1,7 @@
package tap
import (
"github.com/romana/rlog"
"sync"
"time"
@@ -20,19 +21,21 @@ type Cleaner struct {
cleanPeriod time.Duration
connectionTimeout time.Duration
stats CleanerStats
statsMutex sync.Mutex
statsMutex sync.Mutex
}
func (cl *Cleaner) clean() {
startCleanTime := time.Now()
cl.assemblerMutex.Lock()
rlog.Debugf("Assembler Stats before cleaning %s", cl.assembler.Dump())
flushed, closed := cl.assembler.FlushCloseOlderThan(startCleanTime.Add(-cl.connectionTimeout))
cl.assemblerMutex.Unlock()
deleted := cl.matcher.deleteOlderThan(startCleanTime.Add(-cl.connectionTimeout))
cl.statsMutex.Lock()
rlog.Debugf("Assembler Stats after cleaning %s", cl.assembler.Dump())
cl.stats.flushed += flushed
cl.stats.closed += closed
cl.stats.deleted += deleted
@@ -55,7 +58,7 @@ func (cl *Cleaner) dumpStats() CleanerStats {
stats := CleanerStats{
flushed: cl.stats.flushed,
closed : cl.stats.closed,
closed: cl.stats.closed,
deleted: cl.stats.deleted,
}

View File

@@ -43,7 +43,7 @@ func openNewHarFile(filename string) *HarFile {
}
type HarFile struct {
file *os.File
file *os.File
entryCount int
}
@@ -105,13 +105,13 @@ func NewEntry(request *http.Request, requestTime time.Time, response *http.Respo
harEntry := har.Entry{
StartedDateTime: time.Now().UTC(),
Time: totalTime,
Request: harRequest,
Response: harResponse,
Cache: &har.Cache{},
Time: totalTime,
Request: harRequest,
Response: harResponse,
Cache: &har.Cache{},
Timings: &har.Timings{
Send: -1,
Wait: -1,
Send: -1,
Wait: -1,
Receive: totalTime,
},
}
@@ -155,14 +155,14 @@ func (f *HarFile) Close() {
}
}
func (f*HarFile) writeHeader() {
func (f *HarFile) writeHeader() {
header := []byte(`{"log": {"version": "1.2", "creator": {"name": "Mizu", "version": "0.0.1"}, "entries": [`)
if _, err := f.file.Write(header); err != nil {
log.Panicf("Failed to write header to output file: %s (%v,%+v)", err, err, err)
}
}
func (f*HarFile) writeTrailer() {
func (f *HarFile) writeTrailer() {
trailer := []byte("]}}")
if _, err := f.file.Write(trailer); err != nil {
log.Panicf("Failed to write trailer to output file: %s (%v,%+v)", err, err, err)
@@ -172,26 +172,27 @@ func (f*HarFile) writeTrailer() {
func NewHarWriter(outputDir string, maxEntries int) *HarWriter {
return &HarWriter{
OutputDirPath: outputDir,
MaxEntries: maxEntries,
PairChan: make(chan *PairChanItem),
OutChan: make(chan *OutputChannelItem, 1000),
currentFile: nil,
done: make(chan bool),
MaxEntries: maxEntries,
PairChan: make(chan *PairChanItem),
OutChan: make(chan *OutputChannelItem, 1000),
currentFile: nil,
done: make(chan bool),
}
}
type OutputChannelItem struct {
HarEntry *har.Entry
ConnectionInfo *ConnectionInfo
HarEntry *har.Entry
ConnectionInfo *ConnectionInfo
ValidationRulesChecker string
}
type HarWriter struct {
OutputDirPath string
MaxEntries int
PairChan chan *PairChanItem
OutChan chan *OutputChannelItem
currentFile *HarFile
done chan bool
MaxEntries int
PairChan chan *PairChanItem
OutChan chan *OutputChannelItem
currentFile *HarFile
done chan bool
}
func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, connectionInfo *ConnectionInfo) {
@@ -240,7 +241,7 @@ func (hw *HarWriter) Start() {
hw.closeFile()
}
hw.done <- true
} ()
}()
}
func (hw *HarWriter) Stop() {

View File

@@ -35,7 +35,8 @@ const AppPortsEnvVar = "APP_PORTS"
const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT"
const maxHTTP2DataLenDefault = 1 * 1024 * 1024 // 1MB
const cleanPeriod = time.Second * 10
var remoteOnlyOutboundPorts = []int { 80, 443 }
var remoteOnlyOutboundPorts = []int{80, 443}
func parseAppPorts(appPortsList string) []int {
ports := make([]int, 0)
@@ -50,15 +51,15 @@ func parseAppPorts(appPortsList string) []int {
return ports
}
var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit")
var maxcount = flag.Int64("c", -1, "Only grab this many packets, then exit")
var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)")
var statsevery = flag.Int("stats", 60, "Output statistics every N seconds")
var lazy = flag.Bool("lazy", false, "If true, do lazy decoding")
var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
var checksum = flag.Bool("checksum", false, "Check TCP checksum") // global
var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)") // global
var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors") // global
var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence") // global
var checksum = flag.Bool("checksum", false, "Check TCP checksum") // global
var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)") // global
var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors") // global
var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence") // global
var verbose = flag.Bool("verbose", false, "Be verbose")
var debug = flag.Bool("debug", false, "Display debug information")
var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
@@ -68,7 +69,7 @@ var nohttp = flag.Bool("nohttp", false, "Disable HTTP parsing")
var output = flag.String("output", "", "Path to create file for HTTP 200 OK responses")
var writeincomplete = flag.Bool("writeincomplete", false, "Write incomplete response")
var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex") // global
var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex") // global
var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex")
// capture
@@ -87,7 +88,7 @@ var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files")
var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files")
var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file")
var reqResMatcher = createResponseRequestMatcher() // global
var reqResMatcher = createResponseRequestMatcher() // global
var statsTracker = StatsTracker{}
// global
@@ -117,8 +118,8 @@ var outputLevel int
var errorsMap map[string]uint
var errorsMapMutex sync.Mutex
var nErrors uint
var ownIps []string // global
var hostMode bool // global
var ownIps []string // global
var hostMode bool // global
/* minOutputLevel: Error will be printed only if outputLevel is above this value
* t: key for errorsMap (counting errors)
@@ -174,6 +175,10 @@ type Context struct {
CaptureInfo gopacket.CaptureInfo
}
func GetStats() AppStats {
return statsTracker.appStats
}
func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
return c.CaptureInfo
}
@@ -196,6 +201,37 @@ func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *Outbo
return nil, outboundLinkWriter.OutChan
}
func startMemoryProfiler() {
dirname := "/app/pprof"
rlog.Info("Profiling is on, results will be written to %s", dirname)
go func() {
if _, err := os.Stat(dirname); os.IsNotExist(err) {
if err := os.Mkdir(dirname, 0777); err != nil {
log.Fatal("could not create directory for profile: ", err)
}
}
for true {
t := time.Now()
filename := fmt.Sprintf("%s/%s__mem.prof", dirname, t.Format("15_04_05"))
rlog.Info("Writing memory profile to %s\n", filename)
f, err := os.Create(filename)
if err != nil {
log.Fatal("could not create memory profile: ", err)
}
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
log.Fatal("could not write memory profile: ", err)
}
_ = f.Close()
time.Sleep(time.Minute)
}
}()
}
func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWriter) {
log.SetFlags(log.LstdFlags | log.LUTC | log.Lshortfile)
@@ -304,19 +340,23 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
source.Lazy = *lazy
source.NoCopy = true
rlog.Info("Starting to read packets")
count := 0
bytes := int64(0)
start := time.Now()
statsTracker.setStartTime(time.Now())
defragger := ip4defrag.NewIPv4Defragmenter()
streamFactory := &tcpStreamFactory{
doHTTP: !*nohttp,
harWriter: harWriter,
doHTTP: !*nohttp,
harWriter: harWriter,
outbountLinkWriter: outboundLinkWriter,
}
streamPool := reassembly.NewStreamPool(streamFactory)
assembler := reassembly.NewAssembler(streamPool)
maxBufferedPagesTotal := GetMaxBufferedPagesPerConnection()
maxBufferedPagesPerConnection := GetMaxBufferedPagesTotal()
rlog.Infof("Assembler options: maxBufferedPagesTotal=%d, maxBufferedPagesPerConnection=%d", maxBufferedPagesTotal, maxBufferedPagesPerConnection)
assembler.AssemblerOptions.MaxBufferedPagesTotal = maxBufferedPagesTotal
assembler.AssemblerOptions.MaxBufferedPagesPerConnection = maxBufferedPagesPerConnection
var assemblerMutex sync.Mutex
signalChan := make(chan os.Signal, 1)
@@ -324,10 +364,10 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
staleConnectionTimeout := time.Second * time.Duration(*staleTimeoutSeconds)
cleaner := Cleaner{
assembler: assembler,
assemblerMutex: &assemblerMutex,
matcher: &reqResMatcher,
cleanPeriod: cleanPeriod,
assembler: assembler,
assemblerMutex: &assemblerMutex,
matcher: &reqResMatcher,
cleanPeriod: cleanPeriod,
connectionTimeout: staleConnectionTimeout,
}
cleaner.start()
@@ -345,9 +385,9 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
errorsSummery := fmt.Sprintf("%v", errorsMap)
errorsMapMutex.Unlock()
log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v) - Errors Summary: %s",
count,
bytes,
time.Since(start),
statsTracker.appStats.TotalPacketsCount,
statsTracker.appStats.TotalProcessedBytes,
time.Since(statsTracker.appStats.StartTime),
nErrors,
errorMapLen,
errorsSummery,
@@ -365,22 +405,26 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
// Since the last print
cleanStats := cleaner.dumpStats()
appStats := statsTracker.dumpStats()
matchedMessages := statsTracker.dumpStats()
log.Printf(
"flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d",
cleanStats.flushed,
cleanStats.closed,
cleanStats.deleted,
appStats.matchedMessages,
matchedMessages,
)
}
}()
if GetMemoryProfilingEnabled() {
startMemoryProfiler()
}
for packet := range source.Packets() {
count++
rlog.Debugf("PACKET #%d", count)
packetsCount := statsTracker.incPacketsCount()
rlog.Debugf("PACKET #%d", packetsCount)
data := packet.Data()
bytes += int64(len(data))
statsTracker.updateProcessedSize(int64(len(data)))
if *hexdumppkt {
rlog.Debugf("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data))
}
@@ -431,12 +475,17 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
assemblerMutex.Unlock()
}
done := *maxcount > 0 && count >= *maxcount
done := *maxcount > 0 && statsTracker.appStats.TotalPacketsCount >= *maxcount
if done {
errorsMapMutex.Lock()
errorMapLen := len(errorsMap)
errorsMapMutex.Unlock()
log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)", count, bytes, time.Since(start), nErrors, errorMapLen)
log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)",
statsTracker.appStats.TotalPacketsCount,
statsTracker.appStats.TotalProcessedBytes,
time.Since(statsTracker.appStats.StartTime),
nErrors,
errorMapLen)
}
select {
case <-signalChan:
@@ -493,4 +542,5 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
for e := range errorsMap {
log.Printf(" %s:\t\t%d", e, errorsMap[e])
}
log.Printf("AppStats: %v", GetStats())
}

View File

@@ -1,5 +1,18 @@
package tap
import (
"os"
"strconv"
)
const (
MemoryProfilingEnabledEnvVarName = "MEMORY_PROFILING_ENABLED"
MaxBufferedPagesTotalEnvVarName = "MAX_BUFFERED_PAGES_TOTAL"
MaxBufferedPagesPerConnectionEnvVarName = "MAX_BUFFERED_PAGES_PER_CONNECTION"
MaxBufferedPagesTotalDefaultValue = 5000
MaxBufferedPagesPerConnectionDefaultValue = 5000
)
type globalSettings struct {
filterPorts []int
filterAuthorities []string
@@ -29,3 +42,23 @@ func GetFilterIPs() []string {
copy(addresses, gSettings.filterAuthorities)
return addresses
}
func GetMaxBufferedPagesTotal() int {
valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesTotalEnvVarName))
if err != nil {
return MaxBufferedPagesTotalDefaultValue
}
return valueFromEnv
}
func GetMaxBufferedPagesPerConnection() int {
valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesPerConnectionEnvVarName))
if err != nil {
return MaxBufferedPagesPerConnectionDefaultValue
}
return valueFromEnv
}
func GetMemoryProfilingEnabled() bool {
return os.Getenv(MemoryProfilingEnabledEnvVarName) == "1"
}

View File

@@ -2,34 +2,54 @@ package tap
import (
"sync"
"time"
)
type AppStats struct {
matchedMessages int
StartTime time.Time `json:"startTime"`
MatchedMessages int `json:"matchedMessages"`
TotalPacketsCount int64 `json:"totalPacketsCount"`
TotalProcessedBytes int64 `json:"totalProcessedBytes"`
TotalMatchedMessages int64 `json:"totalMatchedMessages"`
}
type StatsTracker struct {
stats AppStats
statsMutex sync.Mutex
appStats AppStats
matchedMessagesMutex sync.Mutex
totalPacketsCountMutex sync.Mutex
totalProcessedSizeMutex sync.Mutex
}
func (st *StatsTracker) incMatchedMessages() {
st.statsMutex.Lock()
st.stats.matchedMessages++
st.statsMutex.Unlock()
st.matchedMessagesMutex.Lock()
st.appStats.MatchedMessages++
st.appStats.TotalMatchedMessages++
st.matchedMessagesMutex.Unlock()
}
func (st *StatsTracker) dumpStats() AppStats {
st.statsMutex.Lock()
stats := AppStats{
matchedMessages: st.stats.matchedMessages,
}
st.stats.matchedMessages = 0
st.statsMutex.Unlock()
return stats
func (st *StatsTracker) incPacketsCount() int64 {
st.totalPacketsCountMutex.Lock()
st.appStats.TotalPacketsCount++
currentPacketsCount := st.appStats.TotalPacketsCount
st.totalPacketsCountMutex.Unlock()
return currentPacketsCount
}
func (st *StatsTracker) updateProcessedSize(size int64) {
st.totalProcessedSizeMutex.Lock()
st.appStats.TotalProcessedBytes += size
st.totalProcessedSizeMutex.Unlock()
}
func (st *StatsTracker) setStartTime(startTime time.Time) {
st.appStats.StartTime = startTime
}
func (st *StatsTracker) dumpStats() int {
st.matchedMessagesMutex.Lock()
matchedMessages := st.appStats.MatchedMessages
st.appStats.MatchedMessages = 0
st.matchedMessagesMutex.Unlock()
return matchedMessages
}

85
ui/package-lock.json generated
View File

@@ -32814,6 +32814,23 @@
"universalify": "^2.0.0"
}
},
"jsonpath": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/jsonpath/-/jsonpath-1.1.1.tgz",
"integrity": "sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==",
"requires": {
"esprima": "1.2.2",
"static-eval": "2.0.2",
"underscore": "1.12.1"
},
"dependencies": {
"esprima": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-1.2.2.tgz",
"integrity": "sha1-dqD9Zvz+FU/SkmZ9wmQBl1CxZXs="
}
}
},
"jsprim": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
@@ -37761,6 +37778,69 @@
"resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.2.0.tgz",
"integrity": "sha512-GrdeshiRmS1YLMYgzF16olf2jJ/IzxXY9lhKOskuVziubpTYcYqyOwYeJKzQkwy7uN0fYSsbsC4RQaXf9LCrYA=="
},
"static-eval": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.2.tgz",
"integrity": "sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==",
"requires": {
"escodegen": "^1.8.1"
},
"dependencies": {
"escodegen": {
"version": "1.14.3",
"resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz",
"integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==",
"requires": {
"esprima": "^4.0.1",
"estraverse": "^4.2.0",
"esutils": "^2.0.2",
"optionator": "^0.8.1",
"source-map": "~0.6.1"
}
},
"levn": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
"integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
"requires": {
"prelude-ls": "~1.1.2",
"type-check": "~0.3.2"
}
},
"optionator": {
"version": "0.8.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
"integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==",
"requires": {
"deep-is": "~0.1.3",
"fast-levenshtein": "~2.0.6",
"levn": "~0.3.0",
"prelude-ls": "~1.1.2",
"type-check": "~0.3.2",
"word-wrap": "~1.2.3"
}
},
"prelude-ls": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
"integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
},
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"optional": true
},
"type-check": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
"integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
"requires": {
"prelude-ls": "~1.1.2"
}
}
}
},
"static-extend": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
@@ -38647,6 +38727,11 @@
"which-boxed-primitive": "^1.0.2"
}
},
"underscore": {
"version": "1.12.1",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz",
"integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw=="
},
"unicode-canonical-property-names-ecmascript": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz",

View File

@@ -12,6 +12,7 @@
"@types/node": "^12.20.10",
"@types/react": "^17.0.3",
"@types/react-dom": "^17.0.3",
"jsonpath": "^1.1.1",
"axios": "^0.21.1",
"node-sass": "^5.0.0",
"numeral": "^2.0.6",

View File

@@ -19,6 +19,13 @@ interface HAREntry {
isCurrentRevision?: boolean;
timestamp: Date;
isOutgoing?: boolean;
latency: number;
rules: Rules;
}
interface Rules {
status: boolean;
latency: number
}
interface HAREntryProps {
@@ -48,9 +55,16 @@ export const HarEntry: React.FC<HAREntryProps> = ({entry, setFocusedEntryId, isS
break;
}
}
let backgroundColor = "";
if ('latency' in entry.rules) {
if (entry.rules.latency !== -1) {
backgroundColor = entry.rules.latency >= entry.latency ? styles.ruleSuccessRow : styles.ruleFailureRow
} else {
backgroundColor = entry.rules.status ? styles.ruleSuccessRow : styles.ruleFailureRow
}
}
return <>
<div id={entry.id} className={`${styles.row} ${isSelected ? styles.rowSelected : ''}`} onClick={() => setFocusedEntryId(entry.id)}>
<div id={entry.id} className={`${styles.row} ${isSelected ? styles.rowSelected : backgroundColor}`} onClick={() => setFocusedEntryId(entry.id)}>
{entry.statusCode && <div>
<StatusCode statusCode={entry.statusCode}/>
</div>}

View File

@@ -29,7 +29,7 @@ const HarEntryTitle: React.FC<any> = ({har}) => {
const classes = useStyles();
const {log: {entries}} = har;
const {response, request, timings: {receive}} = entries[0];
const {response, request, timings: {receive}} = entries[0].entry;
const {status, statusText, bodySize} = response;
@@ -40,9 +40,10 @@ const HarEntryTitle: React.FC<any> = ({har}) => {
<div style={{flexGrow: 1, overflow: 'hidden'}}>
<EndpointPath method={request?.method} path={request?.url}/>
</div>
<div style={{margin: "0 24px", opacity: 0.5}}>{formatSize(bodySize)}</div>
<div style={{marginRight: 24, opacity: 0.5}}>{status} {statusText}</div>
<div style={{opacity: 0.5}}>{Math.round(receive)}ms</div>
<div style={{margin: "0 18px", opacity: 0.5}}>{formatSize(bodySize)}</div>
<div style={{marginRight: 18, opacity: 0.5}}>{status} {statusText}</div>
<div style={{marginRight: 18, opacity: 0.5}}>{Math.round(receive)}ms</div>
<div style={{opacity: 0.5}}>{'rulesMatched' in entries[0] ? entries[0].rulesMatched?.length : '0'} Rules Applied</div>
</div>;
};

View File

@@ -40,6 +40,27 @@
width: 1%
max-width: 15rem
.rulesTitleSuccess
color: #0C0B1A
.rulesMatchedSuccess
background: #E8FFF1
padding: 5px
border-radius: 4px
color: #219653
font-style: normal
font-size: 0.7rem
font-weight: 600
.rulesMatchedFailure
background: #FFE9EF
padding: 5px
border-radius: 4px
color: #DB2156
font-style: normal
font-size: 0.7rem
font-weight: 600
.dataValue
color: $blue-gray
margin: 0
@@ -66,7 +87,6 @@
border-top: 1px solid $light-blue-color
padding: 1rem
background: none
table
width: 100%
tr td:first-child

View File

@@ -5,6 +5,7 @@ import CollapsibleContainer from "../CollapsibleContainer";
import FancyTextDisplay from "../FancyTextDisplay";
import Checkbox from "../Checkbox";
import ProtobufDecoder from "protobuf-decoder";
var jp = require('jsonpath');
interface HAREntryViewLineProps {
label: string;
@@ -144,3 +145,122 @@ export const HAREntryTableSection: React.FC<HAREntrySectionProps> = ({title, arr
}
</React.Fragment>
}
interface HAREntryPolicySectionProps {
service: string,
title: string,
response: any,
latency?: number,
arrayToIterate: any[],
}
interface HAREntryPolicySectionCollapsibleTitleProps {
label: string;
matched: string;
isExpanded: boolean;
}
const HAREntryPolicySectionCollapsibleTitle: React.FC<HAREntryPolicySectionCollapsibleTitleProps> = ({label, matched, isExpanded}) => {
return <div className={styles.title}>
<span className={`${styles.button} ${isExpanded ? styles.expanded : ''}`}>
{isExpanded ? '-' : '+'}
</span>
<span>
<tr className={styles.dataLine}>
<td className={`${styles.dataKey} ${styles.rulesTitleSuccess}`}>{label}</td>
<td className={`${styles.dataKey} ${matched === 'Success' ? styles.rulesMatchedSuccess : styles.rulesMatchedFailure}`}>{matched}</td>
</tr>
</span>
</div>
}
interface HAREntryPolicySectionContainerProps {
label: string;
matched: string;
children?: any;
}
export const HAREntryPolicySectionContainer: React.FC<HAREntryPolicySectionContainerProps> = ({label, matched, children}) => {
const [expanded, setExpanded] = useState(false);
return <CollapsibleContainer
className={styles.collapsibleContainer}
isExpanded={expanded}
onClick={() => setExpanded(!expanded)}
title={<HAREntryPolicySectionCollapsibleTitle label={label} matched={matched} isExpanded={expanded}/>}
>
{children}
</CollapsibleContainer>
}
export const HAREntryTablePolicySection: React.FC<HAREntryPolicySectionProps> = ({service, title, response, latency, arrayToIterate}) => {
const base64ToJson = response.content.mimeType === "application/json; charset=utf-8" ? JSON.parse(Buffer.from(response.content.text, "base64").toString()) : {};
return <React.Fragment>
{
arrayToIterate && arrayToIterate.length > 0 ?
<>
<HAREntrySectionContainer title={title}>
<table>
<tbody>
{arrayToIterate.map(({rule, matched}, index) => {
return (
<HAREntryPolicySectionContainer key={index} label={rule.Name} matched={matched && (rule.Type === 'latency' ? rule.Latency >= latency : true)? "Success" : "Failure"}>
{
<>
{
rule.Key != "" ?
<tr className={styles.dataValue}><td><b>Key</b>:</td><td>{rule.Key}</td></tr>
: null
}
{
rule.Latency != "" ?
<tr className={styles.dataValue}><td><b>Latency:</b></td> <td>{rule.Latency}</td></tr>
: null
}
{
rule.Method != "" ?
<tr className={styles.dataValue}><td><b>Method:</b></td> <td>{rule.Method}</td></tr>
: null
}
{
rule.Path != "" ?
<tr className={styles.dataValue}><td><b>Path:</b></td> <td>{rule.Path}</td></tr>
: null
}
{
rule.Service != "" ?
<tr className={styles.dataValue}><td><b>Service:</b></td> <td>{service}</td></tr>
: null
}
{
rule.Type != "" ?
<tr className={styles.dataValue}><td><b>Type:</b></td> <td>{rule.Type}</td></tr>
: null
}
{
rule.Value != "" ?
<tr className={styles.dataValue}><td><b>Value:</b></td> <td>{rule.Value}</td></tr>
: null
}
</>
}
</HAREntryPolicySectionContainer>
)
}
)
}
</tbody>
</table>
</HAREntrySectionContainer>
</> : <span/>
}
</React.Fragment>
}

View File

@@ -1,19 +1,22 @@
import React, {useState} from 'react';
import styles from './HAREntryViewer.module.sass';
import Tabs from "../Tabs";
import {HAREntryTableSection, HAREntryBodySection} from "./HAREntrySections";
import {HAREntryTableSection, HAREntryBodySection, HAREntryTablePolicySection} from "./HAREntrySections";
const MIME_TYPE_KEY = 'mimeType';
const HAREntryDisplay: React.FC<any> = ({entry, isCollapsed: initialIsCollapsed, isResponseMocked}) => {
const {request, response} = entry;
const HAREntryDisplay: React.FC<any> = ({har, entry, isCollapsed: initialIsCollapsed, isResponseMocked}) => {
const {request, response, timings: {receive}} = entry;
const rulesMatched = har.log.entries[0].rulesMatched
const TABS = [
{tab: 'request'},
{
tab: 'response',
badge: <>{isResponseMocked && <span className="smallBadge virtual mock">MOCK</span>}</>
},
{
tab: 'Rules',
},
];
const [currentTab, setCurrentTab] = useState(TABS[0].tab);
@@ -43,6 +46,9 @@ const HAREntryDisplay: React.FC<any> = ({entry, isCollapsed: initialIsCollapsed,
<HAREntryTableSection title={'Cookies'} arrayToIterate={response.cookies}/>
</React.Fragment>}
{currentTab === TABS[2].tab && <React.Fragment>
<HAREntryTablePolicySection service={har.log.entries[0].service} title={'Rule'} latency={receive} response={response} arrayToIterate={rulesMatched ? rulesMatched : []}/>
</React.Fragment>}
</div>}
</div>;
}
@@ -58,7 +64,7 @@ const HAREntryViewer: React.FC<Props> = ({harObject, className, isResponseMocked
const {log: {entries}} = harObject;
const isCollapsed = entries.length > 1;
return <div className={`${className ? className : ''}`}>
{Object.keys(entries).map((entry: any, index) => <HAREntryDisplay isCollapsed={isCollapsed} key={index} entry={entries[entry]} isResponseMocked={isResponseMocked} showTitle={showTitle}/>)}
{Object.keys(entries).map((entry: any, index) => <HAREntryDisplay har={harObject} isCollapsed={isCollapsed} key={index} entry={entries[entry].entry} isResponseMocked={isResponseMocked} showTitle={showTitle}/>)}
</div>
};

View File

@@ -72,7 +72,6 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus, onTLSDetected
ws.current.onmessage = e => {
if (!e?.data) return;
const message = JSON.parse(e.data);
switch (message.messageType) {
case "entry":
const entry = message.data

View File

@@ -23,6 +23,14 @@
margin-left: 10px
margin-right: 3px
.ruleSuccessRow
border: 1px $success-color solid
border-left: 5px $success-color solid
.ruleFailureRow
border: 1px $failure-color solid
border-left: 5px $failure-color solid
.service
text-overflow: ellipsis
overflow: hidden