mizu release 2021-06-21 (#79)

* Show pod name and namespace (#61) * WIP * Update main.go, consts.go, and 2 more files... * Update messageSensitiveDataCleaner.go * Update consts.go and messageSensitiveDataCleaner.go * Update messageSensitiveDataCleaner.go * Update main.go, consts.go, and 3 more files... * WIP * Update main.go, messageSensitiveDataCleaner.go, and 6 more files... * Update main.go, messageSensitiveDataCleaner.go, and 3 more files... * Update consts.go, messageSensitiveDataCleaner.go, and tap.go * Update provider.go * Update serializableRegexp.go * Update tap.go * TRA-3234 fetch with _source + no hard limit (#64) * remove the HARD limit of 5000 * TRA-3299 Reduce footprint and Add Tolerances(#65) * Use lib const for DNSClusterFirstWithHostNet. * Whitespace. * Break lines. * Added affinity to pod names. * Added tolerations to NoExecute and NoSchedule taints. * Implementation of Mizu view command * . * . * Update main.go and messageSensitiveDataCleaner.go * Update main.go * String and not pointers (#68) * TRA-3318 - Cookies not null and fix har file names (#69) * no message * TRA-3212 Passive-Tapper and Mizu share code (#70) * Use log in tap package instead of fmt. * Moved api/pkg/tap to root. * Added go.mod and go.sum for tap. * Added replace for shared. * api uses tap module instead of tap package. * Removed dependency of tap in shared by moving env var out of tap. * Fixed compilation bugs. * Fixed: Forgot to export struct field HostMode. * Removed unused flag. * Close har output channel when done. * Moved websocket out of mizu and into passive-tapper. * Send connection details over har output channel. * Fixed compilation errors. * Removed unused info from request response cache. * Renamed connection -> connectionID. * Fixed rename bug. * Export setters and getters for filter ips and ports. * Added tap dependency to Dockerfile. * Uncomment error messages. * Renamed `filterIpAddresses` -> `filterAuthorities`. * Renamed ConnectionID -> ConnectionInfo. * Fixed: Missed one replace. * TRA-3342 Mizu/tap dump to har directory fails on Linux (#71) * Instead of saving incomplete temp har files in a temp dir, save them in the output dir with a *.har.tmp suffix. * API only loads har from *.har files (by extension). * Add export entries endpoint for better up9 connect funcionality (#72) * no message * no message * no message * Filter 'cookie' header * Release action (#73) * Create main.yml * Update main.yml * Update main.yml * Update main.yml * Update main.yml * trying new approach * no message * yaml error * no message * no message * no message * missing ) * no message * no message * remove main.yml and fix branches * Create tag-temp.yaml * Update tag-temp.yaml * Update tag-temp.yaml * no message * no message * no message * no message * no message * no message * no message * #minor * no message * no message * added checksum calc to CLI makefile * fixed build error - created bin directory upfront * using markdown for release text * use separate checksum files * fixed release readme * #minor * readme updated Co-authored-by: Alex Haiut <alex@up9.com> * TRA-3360 Fix: Mizu ignores -n namespace flag and records traffic from all pods (#75) Do not tap pods in namespaces which were not requested. * added apple/m1 binary, updated readme (#77) Co-authored-by: Alex Haiut <alex@up9.com> * Update README.md (#78) Co-authored-by: lirazyehezkel <61656597+lirazyehezkel@users.noreply.github.com> Co-authored-by: RamiBerm <rami.berman@up9.com> Co-authored-by: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com> Co-authored-by: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Co-authored-by: Igor Gov <igor.govorov1@gmail.com> Co-authored-by: Alex Haiut <alex@up9.com>
2026-08-19 04:16:35 +00:00 · 2021-06-21 15:17:31 +03:00
parent 5f2a4deb19
commit d3c023b3ba
46 changed files with 1250 additions and 883 deletions
@@ -0,0 +1,10 @@
+package sensitiveDataFiltering
+
+const maskedFieldPlaceholderValue = "[REDACTED]"
+
+//these values MUST be all lower case and contain no `-` or `_` characters
+var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
+	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
+	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
+	"zip", "zipcode", "address", "country", "firstname", "lastname",
+	"middlename", "fname", "lname", "birthdate"}
@@ -0,0 +1,198 @@
+package sensitiveDataFiltering
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/tap"
+	"net/url"
+	"strings"
+
+	"github.com/beevik/etree"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
+)
+
+func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) {
+	harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
+	harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
+
+	harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0)
+	harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0)
+
+	harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL)
+	for i, queryString := range harOutputItem.HarEntry.Request.QueryString {
+		if isFieldNameSensitive(queryString.Name) {
+			harOutputItem.HarEntry.Request.QueryString[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+
+	if harOutputItem.HarEntry.Request.PostData != nil {
+		requestContentType := getContentTypeHeaderValue(harOutputItem.HarEntry.Request.Headers)
+		filteredRequestBody, err := filterHttpBody([]byte(harOutputItem.HarEntry.Request.PostData.Text), requestContentType, options)
+		if err == nil {
+			harOutputItem.HarEntry.Request.PostData.Text = string(filteredRequestBody)
+		}
+	}
+	if harOutputItem.HarEntry.Response.Content != nil {
+		responseContentType := getContentTypeHeaderValue(harOutputItem.HarEntry.Response.Headers)
+		filteredResponseBody, err := filterHttpBody(harOutputItem.HarEntry.Response.Content.Text, responseContentType, options)
+		if err == nil {
+			harOutputItem.HarEntry.Response.Content.Text = filteredResponseBody
+		}
+	}
+}
+
+func filterHarHeaders(headers []har.Header) []har.Header {
+	newHeaders := make([]har.Header, 0)
+	for i, header := range headers {
+		if strings.ToLower(header.Name) == "cookie" {
+			continue
+		} else if isFieldNameSensitive(header.Name) {
+			newHeaders = append(newHeaders, har.Header{Name: header.Name, Value: maskedFieldPlaceholderValue})
+			headers[i].Value = maskedFieldPlaceholderValue
+		} else {
+			newHeaders = append(newHeaders, header)
+		}
+	}
+	return newHeaders
+}
+
+func getContentTypeHeaderValue(headers []har.Header) string {
+	for _, header := range headers {
+		if strings.ToLower(header.Name) == "content-type" {
+			return header.Value
+		}
+	}
+	return ""
+}
+
+func isFieldNameSensitive(fieldName string) bool {
+	name := strings.ToLower(fieldName)
+	name = strings.ReplaceAll(name, "_", "")
+	name = strings.ReplaceAll(name, "-", "")
+	name = strings.ReplaceAll(name, " ", "")
+
+	for _, sensitiveField := range personallyIdentifiableDataFields {
+		if strings.Contains(name, sensitiveField) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func filterHttpBody(bytes []byte, contentType string, options *shared.TrafficFilteringOptions) ([]byte, error) {
+	mimeType := strings.Split(contentType, ";")[0]
+	switch strings.ToLower(mimeType) {
+	case "application/json":
+		return filterJsonBody(bytes)
+	case "text/html":
+		fallthrough
+	case "application/xhtml+xml":
+		fallthrough
+	case "text/xml":
+		fallthrough
+	case "application/xml":
+		return filterXmlEtree(bytes)
+	case "text/plain":
+		if options != nil && options.PlainTextMaskingRegexes != nil {
+			return filterPlainText(bytes, options), nil
+		}
+	}
+	return bytes, nil
+}
+
+func filterPlainText(bytes []byte, options *shared.TrafficFilteringOptions) []byte {
+	for _, regex := range options.PlainTextMaskingRegexes {
+		bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
+	}
+	return bytes
+}
+
+func filterXmlEtree(bytes []byte) ([]byte, error) {
+	if !IsValidXML(bytes) {
+		return nil, errors.New("Invalid XML")
+	}
+	xmlDoc := etree.NewDocument()
+	err := xmlDoc.ReadFromBytes(bytes)
+	if err != nil {
+		return nil, err
+	} else {
+		filterXmlElement(xmlDoc.Root())
+	}
+	return xmlDoc.WriteToBytes()
+}
+
+func IsValidXML(data []byte) bool {
+	return xml.Unmarshal(data, new(interface{})) == nil
+}
+
+func filterXmlElement(element *etree.Element) {
+	for i, attribute := range element.Attr {
+		if isFieldNameSensitive(attribute.Key) {
+			element.Attr[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+	if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
+		if isFieldNameSensitive(element.Tag) {
+			element.SetText(maskedFieldPlaceholderValue)
+		}
+	} else {
+		for _, element := range element.ChildElements() {
+			filterXmlElement(element)
+		}
+	}
+}
+
+func filterJsonBody(bytes []byte) ([]byte, error) {
+	var bodyJsonMap map[string] interface{}
+	err := json.Unmarshal(bytes ,&bodyJsonMap)
+	if err != nil {
+		return nil, err
+	}
+	filterJsonMap(bodyJsonMap)
+	return json.Marshal(bodyJsonMap)
+}
+
+func filterJsonMap(jsonMap map[string] interface{}) {
+	for key, value := range jsonMap {
+		if value == nil {
+			return
+		}
+		nestedMap, isNested := value.(map[string] interface{})
+		if isNested {
+			filterJsonMap(nestedMap)
+		} else {
+			if isFieldNameSensitive(key) {
+				jsonMap[key] = maskedFieldPlaceholderValue
+			}
+		}
+	}
+}
+
+// receives string representing url, returns string url without sensitive query param values (http://service/api?userId=bob&password=123&type=login -> http://service/api?userId=[REDACTED]&password=[REDACTED]&type=login)
+func filterUrl(originalUrl string) string {
+	parsedUrl, err := url.Parse(originalUrl)
+	if err != nil {
+		return fmt.Sprintf("http://%s", maskedFieldPlaceholderValue)
+	} else {
+		if len(parsedUrl.RawQuery) > 0 {
+			newQueryArgs := make([]string, 0)
+			for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() {
+				newValues := urlQueryParamValues
+				if isFieldNameSensitive(urlQueryParamName) {
+					newValues = []string {maskedFieldPlaceholderValue}
+				}
+				for _, paramValue := range newValues {
+					newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
+				}
+			}
+
+			parsedUrl.RawQuery = strings.Join(newQueryArgs, "&")
+		}
+
+		return parsedUrl.String()
+	}
+}