Merge pull request #332 - update download link fix

#minor

acceptanceTests/tap_test.go

@@ -762,6 +762,116 @@ func TestTapRegexMasking(t *testing.T) {
 	}
 }
 
+func TestTapIgnoredUserAgents(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	ignoredUserAgentValue := "ignore"
+	tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.ignored-user-agents=%v", ignoredUserAgentValue))
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+
+	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
+	headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	ignoredUserAgentsCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		if len(entries) == 0 {
+			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		}
+
+		for _, entryInterface := range entries {
+			entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
+			requestResult, requestErr = executeHttpGetRequest(entryUrl)
+			if requestErr != nil {
+				return fmt.Errorf("failed to get entry, err: %v", requestErr)
+			}
+
+			data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+			entryJson := data["entry"].(string)
+
+			var entry map[string]interface{}
+			if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+				return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+			}
+
+			entryRequest := entry["request"].(map[string]interface{})
+			entryPayload := entryRequest["payload"].(map[string]interface{})
+			entryDetails := entryPayload["details"].(map[string]interface{})
+
+			entryHeaders :=  entryDetails["headers"].([]interface{})
+			for _, headerInterface := range entryHeaders {
+				header := headerInterface.(map[string]interface{})
+				if header["name"].(string) != ignoredUserAgentCustomHeader {
+					continue
+				}
+
+				return fmt.Errorf("unexpected result - user agent is not ignored")
+			}
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, ignoredUserAgentsCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
+}
+
 func TestTapDumpLogs(t *testing.T) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")

acceptanceTests/testsUtils.go

@@ -172,6 +172,21 @@ func executeHttpRequest(response *http.Response, requestErr error) (interface{},
 	return jsonBytesToInterface(data)
 }
 
+func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
+	request, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for headerKey, headerValue := range headers {
+		request.Header.Add(headerKey, headerValue)
+	}
+
+	client := &http.Client{}
+	response, requestErr := client.Do(request)
+	return executeHttpRequest(response, requestErr)
+}
+
 func executeHttpGetRequest(url string) (interface{}, error) {
 	response, requestErr := http.Get(url)
 	return executeHttpRequest(response, requestErr)

agent/main.go

@@ -4,6 +4,13 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+	tapApi "github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
 	"log"
 	"mizuserver/pkg/api"
@@ -18,15 +25,6 @@ import (
 	"path/filepath"
 	"plugin"
 	"sort"
-	"strings"
-
-	"github.com/gin-contrib/static"
-	"github.com/gin-gonic/gin"
-	"github.com/gorilla/websocket"
-	"github.com/romana/rlog"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
-	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -59,7 +57,7 @@ func main() {
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
 
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel, filteringOptions)
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
 
 		hostApi(nil)
@@ -90,7 +88,7 @@ func main() {
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel, filteringOptions)
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
 
 		hostApi(outputItemsChannel)
@@ -98,7 +96,7 @@ func main() {
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
 		filteredHarChannel := make(chan *tapApi.OutputChannelItem)
 
-		go filterItems(outputItemsChannel, filteredHarChannel, filteringOptions)
+		go filterItems(outputItemsChannel, filteredHarChannel)
 		go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
 		hostApi(nil)
 	}
@@ -242,42 +240,16 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
-func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem, filterOptions *tapApi.TrafficFilteringOptions) {
+func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
 	for message := range inChannel {
 		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
 			continue
 		}
-		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
-		if isIgnoredUserAgent(message, filterOptions.IgnoredUserAgents) {
-			continue
-		}
 
 		outChannel <- message
 	}
 }
 
-func isIgnoredUserAgent(item *tapApi.OutputChannelItem, userAgentsToIgnore []string) bool {
-	if item.Protocol.Name != "http" {
-		return false
-	}
-
-	request := item.Pair.Request.Payload.(map[string]interface{})
-	reqDetails := request["details"].(map[string]interface{})
-
-	for _, header := range reqDetails["headers"].([]interface{}) {
-		h := header.(map[string]interface{})
-		if strings.ToLower(h["name"].(string)) == "user-agent" {
-			for _, userAgent := range userAgentsToIgnore {
-				if strings.Contains(strings.ToLower(h["value"].(string)), strings.ToLower(userAgent)) {
-					return true
-				}
-			}
-			return false
-		}
-	}
-	return false
-}
-
 func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")

cli/cmd/tap.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
 	"os"
 
 	"github.com/up9inc/mizu/cli/config"
@@ -68,7 +67,4 @@ func init() {
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
-
-	tapCmd.Flags().String(configStructs.EnforcePolicyFileDeprecated, defaultTapConfig.EnforcePolicyFileDeprecated, "Yaml file with policy rules")
-	tapCmd.Flags().MarkDeprecated(configStructs.EnforcePolicyFileDeprecated, fmt.Sprintf("Use --%s instead", configStructs.EnforcePolicyFile))
 }

cli/cmd/tapRunner.go

@@ -49,15 +49,8 @@ func RunMizuTap() {
 	}
 
 	var mizuValidationRules string
-	if config.Config.Tap.EnforcePolicyFile != "" || config.Config.Tap.EnforcePolicyFileDeprecated != "" {
-		var trafficValidation string
-		if config.Config.Tap.EnforcePolicyFile != "" {
-			trafficValidation = config.Config.Tap.EnforcePolicyFile
-		} else {
-			trafficValidation = config.Config.Tap.EnforcePolicyFileDeprecated
-		}
-
-		mizuValidationRules, err = readValidationRules(trafficValidation)
+	if config.Config.Tap.EnforcePolicyFile != "" {
+		mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
 		if err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
 			return

cli/config/configStructs/tapConfig.go

@@ -17,7 +17,6 @@ const (
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	DryRunTapName                 = "dry-run"
 	EnforcePolicyFile             = "traffic-validation-file"
-	EnforcePolicyFileDeprecated   = "test-rules"
 )
 
 type TapConfig struct {
@@ -34,7 +33,6 @@ type TapConfig struct {
 	HumanMaxEntriesDBSize       string    `yaml:"max-entries-db-size" default:"200MB"`
 	DryRun                      bool      `yaml:"dry-run" default:"false"`
 	EnforcePolicyFile           string    `yaml:"traffic-validation-file"`
-	EnforcePolicyFileDeprecated string    `yaml:"test-rules,omitempty" readonly:""`
 	ApiServerResources          Resources `yaml:"api-server-resources"`
 	TapperResources             Resources `yaml:"tapper-resources"`
 }

cli/mizu/version/versionCheck.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/google/go-github/v37/github"
@@ -77,7 +78,7 @@ func CheckNewerVersion(versionChan chan string) {
 	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
 
 	if gitHubVersionSemVer.GreaterThan(currentSemVer) {
-		versionChan <- fmt.Sprintf("Update available! %v -> %v (curl -Lo mizu %v/mizu_%s_amd64 && chmod 755 mizu)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL, runtime.GOOS)
+		versionChan <- fmt.Sprintf("Update available! %v -> %v (curl -Lo mizu %v/mizu_%s_amd64 && chmod 755 mizu)", mizu.SemVer, gitHubVersion, strings.Replace(*latestRelease.HTMLURL, "tag", "download", 1), runtime.GOOS)
 	} else {
 		versionChan <- ""
 	}

docs/POLICY_RULES.md

@@ -31,12 +31,12 @@ mizu tap --traffic-validation-file rules.yaml
 The structure of the traffic-validation-file is:
 
 * `name`: string, name of the rule
-* `type`: string, type of the rule, must be `json` or `header` or `latency`
+* `type`: string, type of the rule, must be `json` or `header` or `slo`
 * `key`: string, [jsonpath](https://code.google.com/archive/p/jsonpath/wikis/Javascript.wiki) used only in `json` or `header` type
 * `value`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) used only in `json` or `header` type
 * `service`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) service name to filter
 * `path`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) URL path to filter
-* `latency`: integer, time in ms of the expected latency.
+* `response-time`: integer, time in ms of the expected latency.
 
 
 ### For example:
@@ -54,8 +54,8 @@ rules:
   key: "Content-Le.*"
   value: "(\\d+(?:\\.\\d+)?)"
 - name: latency-test
-  type: latency
-  latency: 1
+  type: slo
+  response-time: 1
   service: "carts.*"
 ```
 

tap/extensions/http/handlers.go

@@ -14,9 +14,14 @@ import (
 )
 
 func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *api.TrafficFilteringOptions) {
+	if IsIgnoredUserAgent(item, options) {
+		return
+	}
+
 	if !options.DisableRedaction {
 		FilterSensitiveData(item, options)
 	}
+
 	emitter.Emit(item)
 }
 

tap/extensions/http/sensitive_data_cleaner.go

@@ -25,6 +25,30 @@ var personallyIdentifiableDataFields = []string{"token", "authorization", "authe
 	"zip", "zipcode", "address", "country", "firstname", "lastname",
 	"middlename", "fname", "lname", "birthdate"}
 
+func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool {
+	if item.Protocol.Name != "http" {
+		return false
+	}
+
+	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
+
+	for headerKey, headerValues := range request.Header {
+		if strings.ToLower(headerKey) == "user-agent" {
+			for _, userAgent := range options.IgnoredUserAgents {
+				for _, headerValue := range headerValues {
+					if strings.Contains(strings.ToLower(headerValue), strings.ToLower(userAgent)) {
+						return true
+					}
+				}
+			}
+
+			return false
+		}
+	}
+
+	return false
+}
+
 func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
 	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
 	response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)

ui/src/components/Filters.tsx

@@ -20,8 +20,8 @@ export const Filters: React.FC<FiltersProps> = ({methodsFilter, setMethodsFilter
     return <div className={styles.container}>
         <MethodFilter methodsFilter={methodsFilter} setMethodsFilter={setMethodsFilter}/>
         <StatusTypesFilter statusFilter={statusFilter} setStatusFilter={setStatusFilter}/>
-        <PathFilter pathFilter={pathFilter} setPathFilter={setPathFilter}/>
         <ServiceFilter serviceFilter={serviceFilter} setServiceFilter={setServiceFilter}/>
+        <PathFilter pathFilter={pathFilter} setPathFilter={setPathFilter}/>
     </div>;
 };