From 413fb5b3f550c783bfefd08c6ebbff4a2eefd845 Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Mon, 9 Aug 2021 12:27:13 +0300 Subject: [PATCH] Add option to supply user agents to ignore via config (#173) --- README.md | 18 +++++++++++++++-- agent/go.sum | 3 ++- agent/main.go | 10 ++++------ cli/cmd/tap.go | 1 - cli/cmd/tapRunner.go | 6 +++++- cli/go.sum | 4 +--- cli/mizu/config.go | 1 + cli/mizu/configStructs/tapConfig.go | 30 ++++++++++++++--------------- shared/go.mod | 5 ++--- shared/go.sum | 8 ++++---- shared/models.go | 6 +++--- 11 files changed, 53 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 91734b4c0..cc3f053ae 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ To tap all pods in current namespace - To tap specific pod - -``` +```bash $ kubectl get pods NAME READY STATUS RESTARTS AGE front-end-649fc5fd6-kqbtn 2/2 Running 0 7m @@ -88,7 +88,7 @@ To tap specific pod - ``` To tap multiple pods using regex - -``` +```bash $ kubectl get pods NAME READY STATUS RESTARTS AGE carts-66c77f5fbb-fq65r 2/2 Running 0 20m @@ -133,3 +133,17 @@ to the namespace set by `mizu-resources-namespace`. The user must set the tapped using the `--namespace` flag or by setting `tap.namespaces` in the config file. Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior. + +### User agent filtering + +User-agent filtering (like health checks) - can be configured: + +Any request that contains one of those values in the user-agent header will not be captured + +```bash +$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus ++carts-66c77f5fbb-fq65r ++catalogue-5f4cb7cf5-7zrmn +Web interface is now available at http://localhost:8899 +^C +``` \ No newline at end of file diff --git a/agent/go.sum b/agent/go.sum index 0917a0fbb..a0b6f00fa 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -540,8 +540,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= diff --git a/agent/main.go b/agent/main.go index 381433832..f87cba902 100644 --- a/agent/main.go +++ b/agent/main.go @@ -149,15 +149,13 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { return &filteringOptions } -var userAgentsToFilter = []string{"kube-probe", "prometheus"} - func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { for message := range inChannel { if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { continue } // TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441 - if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) { + if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) { continue } @@ -169,11 +167,11 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta } } -func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool { +func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool { for _, header := range message.HarEntry.Request.Headers { if strings.ToLower(header.Name) == "user-agent" { - for _, userAgent := range userAgentsToFilter { - if strings.Contains(strings.ToLower(header.Value), userAgent) { + for _, userAgent := range userAgentsToIgnore { + if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) { return true } } diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 72e71f3fb..6e0da4a7a 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -64,7 +64,6 @@ func init() { tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)") tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces") tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies") - tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "Hides requests with kube-probe or prometheus user-agent headers") tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values") tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size") tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 401b7f77d..7325fcba5 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -207,7 +207,11 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) { } } - return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil + return &shared.TrafficFilteringOptions{ + PlainTextMaskingRegexes: compiledRegexSlice, + HealthChecksUserAgentHeaders: mizu.Config.Tap.HealthChecksUserAgentHeaders, + DisableRedaction: mizu.Config.Tap.DisableRedaction, + }, nil } func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error { diff --git a/cli/go.sum b/cli/go.sum index ae8e268fd..5f63d82b9 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -217,7 +217,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -411,10 +410,9 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= diff --git a/cli/mizu/config.go b/cli/mizu/config.go index 501173b01..e8f4dc537 100644 --- a/cli/mizu/config.go +++ b/cli/mizu/config.go @@ -31,6 +31,7 @@ var allowedSetFlags = []string{ KubeConfigPathName, configStructs.AnalysisDestinationTapName, configStructs.SleepIntervalSecTapName, + configStructs.IgnoredUserAgentsTapName, } var Config = ConfigStruct{} diff --git a/cli/mizu/configStructs/tapConfig.go b/cli/mizu/configStructs/tapConfig.go index 67582e467..553b1bf8c 100644 --- a/cli/mizu/configStructs/tapConfig.go +++ b/cli/mizu/configStructs/tapConfig.go @@ -17,8 +17,8 @@ const ( AnalysisTapName = "analysis" AllNamespacesTapName = "all-namespaces" PlainTextFilterRegexesTapName = "regex-masking" - HideHealthChecksTapName = "hide-healthchecks" DisableRedactionTapName = "no-redact" + IgnoredUserAgentsTapName = "ignored-user-agents" HumanMaxEntriesDBSizeTapName = "max-entries-db-size" DirectionTapName = "direction" DryRunTapName = "dry-run" @@ -26,20 +26,20 @@ const ( ) type TapConfig struct { - AnalysisDestination string `yaml:"dest" default:"up9.app"` - SleepIntervalSec int `yaml:"upload-interval" default:"10"` - PodRegexStr string `yaml:"regex" default:".*"` - GuiPort uint16 `yaml:"gui-port" default:"8899"` - Namespaces []string `yaml:"namespaces"` - Analysis bool `yaml:"analysis" default:"false"` - AllNamespaces bool `yaml:"all-namespaces" default:"false"` - PlainTextFilterRegexes []string `yaml:"regex-masking"` - HideHealthChecks bool `yaml:"hide-healthchecks" default:"false"` - DisableRedaction bool `yaml:"no-redact" default:"false"` - HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` - Direction string `yaml:"direction" default:"in"` - DryRun bool `yaml:"dry-run" default:"false"` - EnforcePolicyFile string `yaml:"test-rules"` + AnalysisDestination string `yaml:"dest" default:"up9.app"` + SleepIntervalSec int `yaml:"upload-interval" default:"10"` + PodRegexStr string `yaml:"regex" default:".*"` + GuiPort uint16 `yaml:"gui-port" default:"8899"` + Namespaces []string `yaml:"namespaces"` + Analysis bool `yaml:"analysis" default:"false"` + AllNamespaces bool `yaml:"all-namespaces" default:"false"` + PlainTextFilterRegexes []string `yaml:"regex-masking"` + HealthChecksUserAgentHeaders []string `yaml:"ignored-user-agents" default:"[]"` + DisableRedaction bool `yaml:"no-redact" default:"false"` + HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` + Direction string `yaml:"direction" default:"in"` + DryRun bool `yaml:"dry-run" default:"false"` + EnforcePolicyFile string `yaml:"test-rules"` } func (config *TapConfig) PodRegex() *regexp.Regexp { diff --git a/shared/go.mod b/shared/go.mod index 157d3e5fa..5e50ad375 100644 --- a/shared/go.mod +++ b/shared/go.mod @@ -3,8 +3,7 @@ module github.com/up9inc/mizu/shared go 1.16 require ( - github.com/google/martian v2.1.0+incompatible // indirect - github.com/gorilla/websocket v1.4.2 - github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect github.com/docker/go-units v0.4.0 + github.com/gorilla/websocket v1.4.2 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/shared/go.sum b/shared/go.sum index 498bce1d3..65c692cb5 100644 --- a/shared/go.sum +++ b/shared/go.sum @@ -1,8 +1,8 @@ -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/shared/models.go b/shared/models.go index 2a70fa212..ade4a9269 100644 --- a/shared/models.go +++ b/shared/models.go @@ -75,9 +75,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc } type TrafficFilteringOptions struct { - PlainTextMaskingRegexes []*SerializableRegexp - HideHealthChecks bool - DisableRedaction bool + HealthChecksUserAgentHeaders []string + PlainTextMaskingRegexes []*SerializableRegexp + DisableRedaction bool } type VersionResponse struct {