feat(redact): use a scan regex for default redact rule of lines to improve cpu usage and reduce time cost (#1291)

pkg/collect/redact.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
@@ -17,97 +18,128 @@ import (
 )
 
 func RedactResult(bundlePath string, input CollectorResult, additionalRedactors []*troubleshootv1beta2.Redact) error {
+	wg := &sync.WaitGroup{}
+
+	// Error channel to capture errors from goroutines
+	errorCh := make(chan error, len(input))
+
 	for k, v := range input {
-		file := k
 
-		var reader io.Reader
-		if v == nil {
-			// Collected contents are in a file. Get a reader to the file.
-			info, err := os.Lstat(filepath.Join(bundlePath, file))
-			if err != nil {
-				if os.IsNotExist(errors.Cause(err)) {
-					// File not found, moving on.
-					continue
-				}
-				return errors.Wrap(err, "failed to stat file")
-			}
+		wg.Add(1)
 
-			// Redact the target file of a symlink
-			// There is an opportunity for improving performance here by skipping symlinks
-			// if a target has been redacted already, but that would require
-			// some extra logic to ensure that a spec filtering only symlinks still works.
-			if info.Mode().Type() == os.ModeSymlink {
-				symlink := file
-				target, err := os.Readlink(filepath.Join(bundlePath, symlink))
+		go func(file string, data []byte) {
+			defer wg.Done()
+			var reader io.Reader
+			if data == nil {
+
+				// Collected contents are in a file. Get a reader to the file.
+				info, err := os.Lstat(filepath.Join(bundlePath, file))
 				if err != nil {
-					return errors.Wrap(err, "failed to read symlink")
+					if os.IsNotExist(errors.Cause(err)) {
+						// File not found, moving on.
+						return
+					}
+					errorCh <- errors.Wrap(err, "failed to stat file")
+					return
 				}
 
-				// Get the relative path to the target file to conform with
-				// the path formats of the CollectorResult
-				file, err = filepath.Rel(bundlePath, target)
-				if err != nil {
-					return errors.Wrap(err, "failed to get relative path")
+				// Redact the target file of a symlink
+				// There is an opportunity for improving performance here by skipping symlinks
+				// if a target has been redacted already, but that would require
+				// some extra logic to ensure that a spec filtering only symlinks still works.
+				if info.Mode().Type() == os.ModeSymlink {
+					symlink := file
+					target, err := os.Readlink(filepath.Join(bundlePath, symlink))
+					if err != nil {
+						errorCh <- errors.Wrap(err, "failed to read symlink")
+						return
+					}
+					// Get the relative path to the target file to conform with
+					// the path formats of the CollectorResult
+					file, err = filepath.Rel(bundlePath, target)
+					if err != nil {
+						errorCh <- errors.Wrap(err, "failed to get relative path")
+						return
+					}
+					klog.V(2).Infof("Redacting %s (symlink => %s)\n", file, symlink)
+				} else {
+					klog.V(2).Infof("Redacting %s\n", file)
 				}
-				klog.V(2).Infof("Redacting %s (symlink => %s)\n", file, symlink)
+				r, err := input.GetReader(bundlePath, file)
+				if err != nil {
+					if os.IsNotExist(errors.Cause(err)) {
+						return
+					}
+					errorCh <- errors.Wrap(err, "failed to get reader")
+					return
+				}
+				defer r.Close()
+
+				reader = r
 			} else {
-				klog.V(2).Infof("Redacting %s\n", file)
+				// Collected contents are in memory. Get a reader to the memory buffer.
+				reader = bytes.NewBuffer(data)
 			}
-			r, err := input.GetReader(bundlePath, file)
-			if err != nil {
-				if os.IsNotExist(errors.Cause(err)) {
-					continue
+
+			// If the file is .tar, .tgz or .tar.gz, it must not be redacted. Instead it is
+			// decompressed and each file inside the tar redacted and compressed back into the archive.
+			if filepath.Ext(file) == ".tar" || filepath.Ext(file) == ".tgz" || strings.HasSuffix(file, ".tar.gz") {
+				tmpDir, err := ioutil.TempDir("", "troubleshoot-subresult-")
+				if err != nil {
+					errorCh <- errors.Wrap(err, "failed to create temp dir")
+					return
 				}
-				return errors.Wrap(err, "failed to get reader")
+				defer os.RemoveAll(tmpDir)
+
+				subResult, tarHeaders, err := decompressFile(tmpDir, reader, file)
+				if err != nil {
+					errorCh <- errors.Wrap(err, "failed to decompress file")
+					return
+				}
+				err = RedactResult(tmpDir, subResult, additionalRedactors)
+				if err != nil {
+					errorCh <- errors.Wrap(err, "failed to redact file")
+					return
+				}
+
+				dstFilename := filepath.Join(bundlePath, file)
+				err = compressFiles(tmpDir, subResult, tarHeaders, dstFilename)
+				if err != nil {
+					errorCh <- errors.Wrap(err, "failed to re-compress file")
+					return
+				}
+
+				os.RemoveAll(tmpDir) // ensure clean up on each iteration in addition to the defer
+
+				//Content of the tar file was redacted. return to next file.
+				return
 			}
-			defer r.Close()
 
-			reader = r
-		} else {
-			// Collected contents are in memory. Get a reader to the memory buffer.
-			reader = bytes.NewBuffer(v)
-		}
-
-		// If the file is .tar, .tgz or .tar.gz, it must not be redacted. Instead it is
-		// decompressed and each file inside the tar redacted and compressed back into the archive.
-		if filepath.Ext(file) == ".tar" || filepath.Ext(file) == ".tgz" || strings.HasSuffix(file, ".tar.gz") {
-			tmpDir, err := ioutil.TempDir("", "troubleshoot-subresult-")
+			redacted, err := redact.Redact(reader, file, additionalRedactors)
 			if err != nil {
-				return errors.Wrap(err, "failed to create temp dir")
+				errorCh <- errors.Wrap(err, "failed to redact io stream")
+				return
 			}
-			defer os.RemoveAll(tmpDir)
 
-			subResult, tarHeaders, err := decompressFile(tmpDir, reader, file)
+			err = input.ReplaceResult(bundlePath, file, redacted)
 			if err != nil {
-				return errors.Wrap(err, "failed to decompress file")
-			}
-			err = RedactResult(tmpDir, subResult, additionalRedactors)
-			if err != nil {
-				return errors.Wrap(err, "failed to redact file")
+				errorCh <- errors.Wrap(err, "failed to create redacted result")
+				return
 			}
+		}(k, v)
+	}
 
-			dstFilename := filepath.Join(bundlePath, file)
-			err = compressFiles(tmpDir, subResult, tarHeaders, dstFilename)
-			if err != nil {
-				return errors.Wrap(err, "failed to re-compress file")
-			}
+	go func() {
+		wg.Wait()
+		close(errorCh)
+	}()
 
-			os.RemoveAll(tmpDir) // ensure clean up on each iteration in addition to the defer
-
-			//Content of the tar file was redacted. Continue to next file.
-			continue
-		}
-
-		redacted, err := redact.Redact(reader, file, additionalRedactors)
+	for err := range errorCh {
 		if err != nil {
-			return errors.Wrap(err, "failed to redact io stream")
-		}
-
-		err = input.ReplaceResult(bundlePath, file, redacted)
-		if err != nil {
-			return errors.Wrap(err, "failed to create redacted result")
+			return err
 		}
 	}
+
 	return nil
 }
 

pkg/collect/result.go

@@ -127,7 +127,12 @@ func (r CollectorResult) SaveResult(bundlePath string, relativePath string, read
 		return errors.Wrap(err, "failed to copy data")
 	}
 
-	klog.V(2).Infof("Added %q to bundle output", relativePath)
+	fileInfo, err := f.Stat()
+	if err != nil {
+		return errors.Wrap(err, "failed to stat file")
+	}
+
+	klog.V(2).Infof("Added %q (%d MB) to bundle output", relativePath, fileInfo.Size()/(1024*1024))
 	return nil
 }
 

pkg/constants/constants.go

@@ -82,4 +82,7 @@ const (
 	// TermUI Display Constants
 	MESSAGE_TEXT_PADDING                = 4
 	MESSAGE_TEXT_LINES_MARGIN_TO_BOTTOM = 4
+
+	// Bufio Reader Constants
+	MAX_BUFFER_CAPACITY = 1024 * 1024
 )

pkg/redact/multi_line.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"io"
 	"regexp"
+	"strings"
 )
 
 type MultiLineRedactor struct {
+	scan       *regexp.Regexp
 	re1        *regexp.Regexp
 	re2        *regexp.Regexp
 	maskText   string
@@ -16,16 +18,26 @@ type MultiLineRedactor struct {
 	isDefault  bool
 }
 
-func NewMultiLineRedactor(re1, re2, maskText, path, name string, isDefault bool) (*MultiLineRedactor, error) {
-	compiled1, err := regexp.Compile(re1)
+func NewMultiLineRedactor(re1 LineRedactor, re2 string, maskText, path, name string, isDefault bool) (*MultiLineRedactor, error) {
+	var scanCompiled *regexp.Regexp
+	compiled1, err := regexp.Compile(re1.regex)
 	if err != nil {
 		return nil, err
 	}
+
+	if re1.scan != "" {
+		scanCompiled, err = regexp.Compile(re1.scan)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	compiled2, err := regexp.Compile(re2)
 	if err != nil {
 		return nil, err
 	}
-	return &MultiLineRedactor{re1: compiled1, re2: compiled2, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil
+
+	return &MultiLineRedactor{scan: scanCompiled, re1: compiled1, re2: compiled2, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil
 }
 
 func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader {
@@ -52,6 +64,17 @@ func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader {
 		for err == nil {
 			lineNum++ // the first line that can be redacted is line 2
 
+			// is scan is not nil, then check if line1 matches scan by lowercasing it
+			if r.scan != nil {
+				lowerLine1 := strings.ToLower(line1)
+				if !r.scan.MatchString(lowerLine1) {
+					fmt.Fprintf(writer, "%s\n", line1)
+					line1, line2, err = getNextTwoLines(reader, &line2)
+					flushLastLine = true
+					continue
+				}
+			}
+
 			// If line1 matches re1, then transform line2 using re2
 			if !r.re1.MatchString(line1) {
 				fmt.Fprintf(writer, "%s\n", line1)
@@ -60,7 +83,6 @@ func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader {
 				continue
 			}
 			flushLastLine = false
-
 			clean := r.re2.ReplaceAllString(line2, substStr)
 
 			// io.WriteString would be nicer, but reader strips new lines

pkg/redact/multi_line_test.go

@@ -0,0 +1,111 @@
+package redact
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_NewMultiLineRedactorr(t *testing.T) {
+	tests := []struct {
+		name        string
+		selector    LineRedactor
+		scan        string
+		redactor    string
+		inputString string
+		wantString  string
+	}{
+		{
+			name: "Redact multiline with AWS secret access key",
+			selector: LineRedactor{
+				regex: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			inputString: `"name": "secret_access_key"
+"value": "dfeadsfsdfe"`,
+			wantString: `"name": "secret_access_key"
+"value": "***HIDDEN***"
+`,
+		},
+		{
+			name: "Redact multiline with AWS secret id",
+			selector: LineRedactor{
+				regex: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			inputString: `"name": "ACCESS_KEY_ID"
+"value": "dfeadsfsdfe"`,
+			wantString: `"name": "ACCESS_KEY_ID"
+"value": "***HIDDEN***"
+`,
+		},
+		{
+			name: "Redact multiline with OSD",
+			selector: LineRedactor{
+				regex: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`,
+			},
+			redactor: `(?i)("key": *")(?P<mask>.{38}==[^\"]*)(")`,
+			inputString: `"entity": "osd.1abcdef"
+"key": "Gjt8s0WkfPtxZUo7gI8a0awbQGHgzuprdaedfb=="`,
+			wantString: `"entity": "osd.1abcdef"
+"key": "***HIDDEN***"
+`,
+		},
+		{
+			name: "Redact multiline with AWS secret access key and scan regex",
+			selector: LineRedactor{
+				regex: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`,
+				scan:  `secret_?access_?key\"`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			inputString: `"name": "secret_access_key"
+"value": "dfeadsfsdfe"`,
+			wantString: `"name": "secret_access_key"
+"value": "***HIDDEN***"
+`,
+		},
+		{
+			name: "Redact multiline with AWS secret id and scan regex",
+			selector: LineRedactor{
+				regex: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`,
+				scan:  `access_?key_?id\"`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			inputString: `"name": "ACCESS_KEY_ID"
+"value": "dfeadsfsdfe"`,
+			wantString: `"name": "ACCESS_KEY_ID"
+"value": "***HIDDEN***"
+`,
+		},
+		{
+			name: "Redact multiline with OSD and scan regex",
+			selector: LineRedactor{
+				regex: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`,
+				scan:  `(osd|client|mgr)`,
+			},
+			redactor: `(?i)("key": *")(?P<mask>.{38}==[^\"]*)(")`,
+			inputString: `"entity": "osd.1abcdef"
+"key": "Gjt8s0WkfPtxZUo7gI8a0awbQGHgzuprdaedfb=="`,
+			wantString: `"entity": "osd.1abcdef"
+"key": "***HIDDEN***"
+`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := require.New(t)
+
+			reRunner, err := NewMultiLineRedactor(tt.selector, tt.redactor, MASK_TEXT, "testfile", tt.name, true)
+			req.NoError(err)
+			outReader := reRunner.Redact(bytes.NewReader([]byte(tt.inputString)), "")
+
+			gotBytes, err := ioutil.ReadAll(outReader)
+			req.NoError(err)
+			req.Equal(tt.wantString, string(gotBytes))
+			GetRedactionList()
+			ResetRedactionList()
+		})
+	}
+}

pkg/redact/redact.go

@@ -46,6 +46,11 @@ type Redaction struct {
 	IsDefaultRedactor bool   `json:"isDefaultRedactor" yaml:"isDefaultRedactor"`
 }
 
+type LineRedactor struct {
+	regex string
+	scan  string
+}
+
 func Redact(input io.Reader, path string, additionalRedactors []*troubleshootv1beta2.Redact) (io.Reader, error) {
 	redactors, err := getRedactors(path)
 	if err != nil {
@@ -105,12 +110,16 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta2.Redact
 		for j, re := range redact.Removals.Regex {
 			var newRedactor Redactor
 			if re.Selector != "" {
-				newRedactor, err = NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT, path, redactorName(i, j, redact.Name, "multiLine"), false)
+				newRedactor, err = NewMultiLineRedactor(LineRedactor{
+					regex: re.Selector,
+				}, re.Redactor, MASK_TEXT, path, redactorName(i, j, redact.Name, "multiLine"), false)
 				if err != nil {
 					return nil, errors.Wrapf(err, "multiline redactor %+v", re)
 				}
 			} else {
-				newRedactor, err = NewSingleLineRedactor(re.Redactor, MASK_TEXT, path, redactorName(i, j, redact.Name, "regex"), false)
+				newRedactor, err = NewSingleLineRedactor(LineRedactor{
+					regex: re.Redactor,
+				}, MASK_TEXT, path, redactorName(i, j, redact.Name, "regex"), false)
 				if err != nil {
 					return nil, errors.Wrapf(err, "redactor %q", re)
 				}
@@ -165,88 +174,143 @@ func getRedactors(path string) ([]Redactor, error) {
 	// groups named with `?P<mask>` will be masked
 	// groups named with `?P<drop>` will be removed (replaced with empty strings)
 	singleLines := []struct {
-		regex string
+		regex LineRedactor
 		name  string
 	}{
 		// aws secrets
 		{
-			regex: `(?i)(\\\"name\\\":\\\"[^\"]*SECRET_?ACCESS_?KEY\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
-			name:  "Redact values for environment variables that look like AWS Secret Access Keys",
+			regex: LineRedactor{
+				regex: `(?i)(\\\"name\\\":\\\"[^\"]*SECRET_?ACCESS_?KEY\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
+				scan:  `secret_?access_?key`,
+			},
+			name: "Redact values for environment variables that look like AWS Secret Access Keys",
 		},
 		{
-			regex: `(?i)(\\\"name\\\":\\\"[^\"]*ACCESS_?KEY_?ID\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
-			name:  "Redact values for environment variables that look like AWS Access Keys",
+			regex: LineRedactor{
+				regex: `(?i)(\\\"name\\\":\\\"[^\"]*ACCESS_?KEY_?ID\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
+				scan:  `access_?key_?id`,
+			},
+			name: "Redact values for environment variables that look like AWS Access Keys",
 		},
 		{
-			regex: `(?i)(\\\"name\\\":\\\"[^\"]*OWNER_?ACCOUNT\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
-			name:  "Redact values for environment variables that look like AWS Owner or Account numbers",
+			regex: LineRedactor{
+				regex: `(?i)(\\\"name\\\":\\\"[^\"]*OWNER_?ACCOUNT\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
+				scan:  `owner_?account`,
+			},
+			name: "Redact values for environment variables that look like AWS Owner or Account numbers",
 		},
 		// passwords in general
 		{
-			regex: `(?i)(\\\"name\\\":\\\"[^\"]*password[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
-			name:  "Redact values for environment variables with names beginning with 'password'",
+			regex: LineRedactor{
+				regex: `(?i)(\\\"name\\\":\\\"[^\"]*password[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
+				scan:  `password`,
+			},
+			name: "Redact values for environment variables with names beginning with 'password'",
 		},
 		// tokens in general
 		{
-			regex: `(?i)(\\\"name\\\":\\\"[^\"]*token[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
-			name:  "Redact values for environment variables with names beginning with 'token'",
+
+			regex: LineRedactor{
+				regex: `(?i)(\\\"name\\\":\\\"[^\"]*token[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
+				scan:  `token`,
+			},
+			name: "Redact values for environment variables with names beginning with 'token'",
 		},
 		{
-			regex: `(?i)(\\\"name\\\":\\\"[^\"]*database[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
-			name:  "Redact values for environment variables with names beginning with 'database'",
+			regex: LineRedactor{
+				regex: `(?i)(\\\"name\\\":\\\"[^\"]*database[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
+				scan:  `database`,
+			},
+			name: "Redact values for environment variables with names beginning with 'database'",
 		},
 		{
-			regex: `(?i)(\\\"name\\\":\\\"[^\"]*user[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
-			name:  "Redact values for environment variables with names beginning with 'user'",
+			regex: LineRedactor{
+				regex: `(?i)(\\\"name\\\":\\\"[^\"]*user[^\"]*\\\",\\\"value\\\":\\\")(?P<mask>[^\"]*)(\\\")`,
+				scan:  `user`,
+			},
+			name: "Redact values for environment variables with names beginning with 'user'",
 		},
 		// connection strings with username and password
 		// http://user:password@host:8888
 		{
-			regex: `(?i)(https?|ftp)(:\/\/)(?P<mask>[^:\"\/]+){1}(:)(?P<mask>[^@\"\/]+){1}(?P<host>@[^:\/\s\"]+){1}(?P<port>:[\d]+)?`,
-			name:  "Redact connection strings with username and password",
+			regex: LineRedactor{
+				regex: `(?i)(https?|ftp)(:\/\/)(?P<mask>[^:\"\/]+){1}(:)(?P<mask>[^@\"\/]+){1}(?P<host>@[^:\/\s\"]+){1}(?P<port>:[\d]+)?`,
+				scan:  `https?|ftp`,
+			},
+			name: "Redact connection strings with username and password",
 		},
 		// user:password@tcp(host:3309)/db-name
 		{
-			regex: `\b(?P<mask>[^:\"\/]*){1}(:)(?P<mask>[^:\"\/]*){1}(@tcp\()(?P<mask>[^:\"\/]*){1}(?P<port>:[\d]*)?(\)\/)(?P<mask>[\w\d\S-_]+){1}\b`,
-			name:  "Redact database connection strings that contain username and password",
+			regex: LineRedactor{
+				regex: `\b(?P<mask>[^:\"\/]*){1}(:)(?P<mask>[^:\"\/]*){1}(@tcp\()(?P<mask>[^:\"\/]*){1}(?P<port>:[\d]*)?(\)\/)(?P<mask>[\w\d\S-_]+){1}\b`,
+				scan:  `@tcp`,
+			},
+			name: "Redact database connection strings that contain username and password",
 		},
 		// standard postgres and mysql connection strings
 		// protocol://user:password@host:5432/db
 		{
-			regex: `\b(\w*:\/\/)(?P<mask>[^:\"\/]*){1}(:)(?P<mask>[^:\"\/]*){1}(@)(?P<mask>[^:\"\/]*){1}(?P<port>:[\d]*)?(\/)(?P<mask>[\w\d\S-_]+){1}\b`,
-			name:  "Redact database connection strings that contain username and password",
+			regex: LineRedactor{
+				regex: `\b(\w*:\/\/)(?P<mask>[^:\"\/]*){1}(:)(?P<mask>[^:\"\/]*){1}(@)(?P<mask>[^:\"\/]*){1}(?P<port>:[\d]*)?(\/)(?P<mask>[\w\d\S-_]+){1}\b`,
+				scan:  `\b(\w*:\/\/)([^:\"\/]*)(:)([^@\"\/]*)(@)([^:\"\/]*)(:[\d]*)?(\/)([\w\d\S-_]+)\b`,
+			},
+			name: "Redact database connection strings that contain username and password",
 		},
 		{
-			regex: `(?i)(Data Source *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'Data Source' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(Data Source *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `data source`,
+			},
+			name: "Redact 'Data Source' values commonly found in database connection strings",
 		},
 		{
-			regex: `(?i)(location *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'location' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(location *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `location`,
+			},
+			name: "Redact 'location' values commonly found in database connection strings",
 		},
 		{
-			regex: `(?i)(User ID *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'User ID' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(User ID *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `user id`,
+			},
+			name: "Redact 'User ID' values commonly found in database connection strings",
 		},
 		{
-			regex: `(?i)(password *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'password' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(password *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `password`,
+			},
+			name: "Redact 'password' values commonly found in database connection strings",
 		},
 		{
-			regex: `(?i)(Server *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'Server' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(Server *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `server`,
+			},
+			name: "Redact 'Server' values commonly found in database connection strings",
 		},
 		{
-			regex: `(?i)(Database *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'Database' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(Database *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `database`,
+			},
+			name: "Redact 'Database' values commonly found in database connection strings",
 		},
 		{
-			regex: `(?i)(Uid *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'UID' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(Uid *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `uid`,
+			},
+			name: "Redact 'UID' values commonly found in database connection strings",
 		},
 		{
-			regex: `(?i)(Pwd *= *)(?P<mask>[^\;]+)(;)`,
-			name:  "Redact 'Pwd' values commonly found in database connection strings",
+			regex: LineRedactor{
+				regex: `(?i)(Pwd *= *)(?P<mask>[^\;]+)(;)`,
+				scan:  `pwd`,
+			},
+			name: "Redact 'Pwd' values commonly found in database connection strings",
 		},
 	}
 
@@ -260,54 +324,78 @@ func getRedactors(path string) ([]Redactor, error) {
 	}
 
 	doubleLines := []struct {
-		line1 string
-		line2 string
-		name  string
+		selector LineRedactor
+		redactor string
+		name     string
 	}{
 		{
-			line1: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`,
-			line2: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
-			name:  "Redact AWS Secret Access Key values in multiline JSON",
+			selector: LineRedactor{
+				regex: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`,
+				scan:  `secret_?access_?key`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			name:     "Redact AWS Secret Access Key values in multiline JSON",
 		},
 		{
-			line1: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`,
-			line2: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
-			name:  "Redact AWS Access Key ID values in multiline JSON",
+			selector: LineRedactor{
+				regex: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`,
+				scan:  `access_?key_?id`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			name:     "Redact AWS Access Key ID values in multiline JSON",
 		},
 		{
-			line1: `(?i)"name": *"[^\"]*OWNER_?ACCOUNT[^\"]*"`,
-			line2: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
-			name:  "Redact AWS Owner and Account Numbers in multiline JSON",
+			selector: LineRedactor{
+				regex: `(?i)"name": *"[^\"]*OWNER_?ACCOUNT[^\"]*"`,
+				scan:  `owner_?account`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			name:     "Redact AWS Owner and Account Numbers in multiline JSON",
 		},
 		{
-			line1: `(?i)"name": *".*password[^\"]*"`,
-			line2: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
-			name:  "Redact password environment variables in multiline JSON",
+			selector: LineRedactor{
+				regex: `(?i)"name": *".*password[^\"]*"`,
+				scan:  `password`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			name:     "Redact password environment variables in multiline JSON",
 		},
 		{
-			line1: `(?i)"name": *".*token[^\"]*"`,
-			line2: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
-			name:  "Redact values that look like API tokens in multiline JSON",
+			selector: LineRedactor{
+				regex: `(?i)"name": *".*token[^\"]*"`,
+				scan:  `token`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			name:     "Redact values that look like API tokens in multiline JSON",
 		},
 		{
-			line1: `(?i)"name": *".*database[^\"]*"`,
-			line2: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
-			name:  "Redact database connection strings in multiline JSON",
+			selector: LineRedactor{
+				regex: `(?i)"name": *".*database[^\"]*"`,
+				scan:  `database`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			name:     "Redact database connection strings in multiline JSON",
 		},
 		{
-			line1: `(?i)"name": *".*user[^\"]*"`,
-			line2: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
-			name:  "Redact usernames in multiline JSON",
+			selector: LineRedactor{
+				regex: `(?i)"name": *".*user[^\"]*"`,
+				scan:  `user`,
+			},
+			redactor: `(?i)("value": *")(?P<mask>.*[^\"]*)(")`,
+			name:     "Redact usernames in multiline JSON",
 		},
 		{
-			line1: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`,
-			line2: `(?i)("key": *")(?P<mask>.{38}==[^\"]*)(")`,
-			name:  "Redact 'key' values found in Ceph auth lists",
+			selector: LineRedactor{
+				regex: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`,
+				scan:  `(osd|client|mgr)`,
+			},
+			redactor: `(?i)("key": *")(?P<mask>.{38}==[^\"]*)(")`,
+			name:     "Redact 'key' values found in Ceph auth lists",
 		},
 	}
 
 	for _, l := range doubleLines {
-		r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT, path, l.name, true)
+		r, err := NewMultiLineRedactor(l.selector, l.redactor, MASK_TEXT, path, l.name, true)
 		if err != nil {
 			return nil, err // maybe skip broken ones?
 		}

pkg/redact/redact_test.go

@@ -1741,6 +1741,7 @@ func Test_Redactors(t *testing.T) {
 		ResetRedactionList()
 		req.Len(actualRedactions.ByFile["testpath"], wantRedactionsLen)
 		req.Len(actualRedactions.ByRedactor, wantRedactionsCount)
+		ResetRedactionList()
 	})
 }
 

pkg/redact/single_line.go

@@ -5,9 +5,13 @@ import (
 	"fmt"
 	"io"
 	"regexp"
+	"strings"
+
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
 )
 
 type SingleLineRedactor struct {
+	scan       *regexp.Regexp
 	re         *regexp.Regexp
 	maskText   string
 	filePath   string
@@ -15,12 +19,21 @@ type SingleLineRedactor struct {
 	isDefault  bool
 }
 
-func NewSingleLineRedactor(re, maskText, path, name string, isDefault bool) (*SingleLineRedactor, error) {
-	compiled, err := regexp.Compile(re)
+func NewSingleLineRedactor(re LineRedactor, maskText, path, name string, isDefault bool) (*SingleLineRedactor, error) {
+	var scanCompiled *regexp.Regexp
+	compiled, err := regexp.Compile(re.regex)
 	if err != nil {
 		return nil, err
 	}
-	return &SingleLineRedactor{re: compiled, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil
+
+	if re.scan != "" {
+		scanCompiled, err = regexp.Compile(re.scan)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &SingleLineRedactor{scan: scanCompiled, re: compiled, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil
 }
 
 func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader {
@@ -38,16 +51,25 @@ func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader {
 
 		substStr := getReplacementPattern(r.re, r.maskText)
 
-		reader := bufio.NewReader(input)
+		buf := make([]byte, constants.MAX_BUFFER_CAPACITY)
+		scanner := bufio.NewScanner(input)
+		scanner.Buffer(buf, constants.MAX_BUFFER_CAPACITY)
+
 		lineNum := 0
-		for {
+		for scanner.Scan() {
 			lineNum++
-			var line string
-			line, err = readLine(reader)
-			if err != nil {
-				return
+			line := scanner.Text()
+
+			// is scan is not nil, then check if line matches scan by lowercasing it
+			if r.scan != nil {
+				lowerLine := strings.ToLower(line)
+				if !r.scan.MatchString(lowerLine) {
+					fmt.Fprintf(writer, "%s\n", line)
+					continue
+				}
 			}
 
+			// if scan matches, but re does not, do not redact
 			if !r.re.MatchString(line) {
 				fmt.Fprintf(writer, "%s\n", line)
 				continue
@@ -57,6 +79,7 @@ func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader {
 
 			// io.WriteString would be nicer, but scanner strips new lines
 			fmt.Fprintf(writer, "%s\n", clean)
+
 			if err != nil {
 				return
 			}
@@ -72,6 +95,9 @@ func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader {
 				})
 			}
 		}
+		if scanErr := scanner.Err(); scanErr != nil {
+			err = scanErr
+		}
 	}()
 	return out
 }

pkg/redact/single_line_test.go

@@ -12,6 +12,7 @@ func TestNewSingleLineRedactor(t *testing.T) {
 	tests := []struct {
 		name           string
 		re             string
+		scan           string
 		inputString    string
 		wantString     string
 		wantRedactions RedactionList
@@ -100,12 +101,222 @@ func TestNewSingleLineRedactor(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:        "Redact values for environment variables that look like AWS Secret Access Keys",
+			re:          `(?i)("name":"[^\"]*SECRET_?ACCESS_?KEY","value":")(?P<mask>[^\"]*)(")`,
+			inputString: `{"name":"SECRET_ACCESS_KEY","value":"123"}`,
+			wantString: `{"name":"SECRET_ACCESS_KEY","value":"***HIDDEN***"}
+`,
+			wantRedactions: RedactionList{
+				ByRedactor: map[string][]Redaction{
+					"Redact values for environment variables that look like AWS Secret Access Keys": []Redaction{
+						{
+							RedactorName:      "Redact values for environment variables that look like AWS Secret Access Keys",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+				ByFile: map[string][]Redaction{
+					"testfile": []Redaction{
+						{
+							RedactorName:      "Redact values for environment variables that look like AWS Secret Access Keys",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+			},
+		},
+		{
+			name:        "Redact connection strings with username and password",
+			re:          `(?i)(https?|ftp)(:\/\/)(?P<mask>[^:\"\/]+){1}(:)(?P<mask>[^@\"\/]+){1}(?P<host>@[^:\/\s\"]+){1}(?P<port>:[\d]+)?`,
+			inputString: `http://user:password@host:8888`,
+			wantString:  "http://***HIDDEN***:***HIDDEN***@host:8888\n",
+			wantRedactions: RedactionList{
+				ByRedactor: map[string][]Redaction{
+					"Redact connection strings with username and password": []Redaction{
+						{
+							RedactorName:      "Redact connection strings with username and password",
+							CharactersRemoved: -12,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+				ByFile: map[string][]Redaction{
+					"testfile": []Redaction{
+						{
+							RedactorName:      "Redact connection strings with username and password",
+							CharactersRemoved: -12,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+			},
+		},
+		{
+			name:        "Redact values for environment variables that look like AWS Secret Access Keys With Scan",
+			re:          `(?i)("name":"[^\"]*SECRET_?ACCESS_?KEY","value":")(?P<mask>[^\"]*)(")`,
+			scan:        `secret_?access_?key`,
+			inputString: `{"name":"SECRET_ACCESS_KEY","value":"123"}`,
+			wantString: `{"name":"SECRET_ACCESS_KEY","value":"***HIDDEN***"}
+`,
+			wantRedactions: RedactionList{
+				ByRedactor: map[string][]Redaction{
+					"Redact values for environment variables that look like AWS Secret Access Keys With Scan": {
+						{
+							RedactorName:      "Redact values for environment variables that look like AWS Secret Access Keys With Scan",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+				ByFile: map[string][]Redaction{
+					"testfile": {
+						{
+							RedactorName:      "Redact values for environment variables that look like AWS Secret Access Keys With Scan",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+			},
+		},
+		{
+			name:        "Redact values for environment variables that look like Access Keys ID With Scan",
+			re:          `(?i)("name":"[^\"]*ACCESS_?KEY_?ID","value":")(?P<mask>[^\"]*)(")`,
+			scan:        `access_?key_?id`,
+			inputString: `{"name":"ACCESS_KEY_ID","value":"123"}`,
+			wantString: `{"name":"ACCESS_KEY_ID","value":"***HIDDEN***"}
+`,
+			wantRedactions: RedactionList{
+				ByRedactor: map[string][]Redaction{
+					"Redact values for environment variables that look like Access Keys ID With Scan": {
+						{
+							RedactorName:      "Redact values for environment variables that look like Access Keys ID With Scan",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+				ByFile: map[string][]Redaction{
+					"testfile": {
+						{
+							RedactorName:      "Redact values for environment variables that look like Access Keys ID With Scan",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+			},
+		},
+		{
+			name:        "Redact values for environment variables that look like Owner Account With Scan",
+			re:          `(?i)("name":"[^\"]*OWNER_?ACCOUNT","value":")(?P<mask>[^\"]*)(")`,
+			scan:        `owner_?account`,
+			inputString: `{"name":"OWNER_ACCOUNT","value":"123"}`,
+			wantString: `{"name":"OWNER_ACCOUNT","value":"***HIDDEN***"}
+`,
+			wantRedactions: RedactionList{
+				ByRedactor: map[string][]Redaction{
+					"Redact values for environment variables that look like Owner Account With Scan": {
+						{
+							RedactorName:      "Redact values for environment variables that look like Owner Account With Scan",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+				ByFile: map[string][]Redaction{
+					"testfile": {
+						{
+							RedactorName:      "Redact values for environment variables that look like Owner Account With Scan",
+							CharactersRemoved: -9,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+			},
+		},
+		{
+			name:        "Redact 'Data Source' values With Scan",
+			re:          `(?i)(Data Source *= *)(?P<mask>[^\;]+)(;)`,
+			scan:        `data source`,
+			inputString: `Data Source = abcdef;`,
+			wantString: `Data Source = ***HIDDEN***;
+`,
+			wantRedactions: RedactionList{
+				ByRedactor: map[string][]Redaction{
+					"Redact 'Data Source' values With Scan": {
+						{
+							RedactorName:      "Redact 'Data Source' values With Scan",
+							CharactersRemoved: -6,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+				ByFile: map[string][]Redaction{
+					"testfile": {
+						{
+							RedactorName:      "Redact 'Data Source' values With Scan",
+							CharactersRemoved: -6,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+			},
+		},
+		{
+			name:        "Redact connection strings With Scan",
+			re:          `(?i)(https?|ftp)(:\/\/)(?P<mask>[^:\"\/]+){1}(:)(?P<mask>[^@\"\/]+){1}(?P<host>@[^:\/\s\"]+){1}(?P<port>:[\d]+)?`,
+			scan:        `https?|ftp`,
+			inputString: `http://user:password@host:8888;`,
+			wantString: `http://***HIDDEN***:***HIDDEN***@host:8888;
+`,
+			wantRedactions: RedactionList{
+				ByRedactor: map[string][]Redaction{
+					"Redact connection strings With Scan": {
+						{
+							RedactorName:      "Redact connection strings With Scan",
+							CharactersRemoved: -12,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+				ByFile: map[string][]Redaction{
+					"testfile": {
+						{
+							RedactorName:      "Redact connection strings With Scan",
+							CharactersRemoved: -12,
+							Line:              1,
+							File:              "testfile",
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			req := require.New(t)
 			ResetRedactionList()
-			reRunner, err := NewSingleLineRedactor(tt.re, MASK_TEXT, "testfile", tt.name, false)
+			reRunner, err := NewSingleLineRedactor(LineRedactor{
+				regex: tt.re,
+				scan:  tt.scan,
+			}, MASK_TEXT, "testfile", tt.name, false)
 			req.NoError(err)
 
 			outReader := reRunner.Redact(bytes.NewReader([]byte(tt.inputString)), "")