From d730be0fae43f316a6799b6afdf6876d933ee884 Mon Sep 17 00:00:00 2001 From: Dexter Yan Date: Fri, 25 Aug 2023 15:11:04 +1200 Subject: [PATCH] feat(redact): use a scan regex for default redact rule of lines to improve cpu usage and reduce time cost (#1291) --- pkg/collect/redact.go | 170 +++++++++++++++---------- pkg/collect/result.go | 7 +- pkg/constants/constants.go | 3 + pkg/redact/multi_line.go | 30 ++++- pkg/redact/multi_line_test.go | 111 +++++++++++++++++ pkg/redact/redact.go | 222 +++++++++++++++++++++++---------- pkg/redact/redact_test.go | 1 + pkg/redact/single_line.go | 44 +++++-- pkg/redact/single_line_test.go | 213 ++++++++++++++++++++++++++++++- 9 files changed, 650 insertions(+), 151 deletions(-) create mode 100644 pkg/redact/multi_line_test.go diff --git a/pkg/collect/redact.go b/pkg/collect/redact.go index e9fdd74b..18b47764 100644 --- a/pkg/collect/redact.go +++ b/pkg/collect/redact.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" @@ -17,97 +18,128 @@ import ( ) func RedactResult(bundlePath string, input CollectorResult, additionalRedactors []*troubleshootv1beta2.Redact) error { + wg := &sync.WaitGroup{} + + // Error channel to capture errors from goroutines + errorCh := make(chan error, len(input)) + for k, v := range input { - file := k - var reader io.Reader - if v == nil { - // Collected contents are in a file. Get a reader to the file. - info, err := os.Lstat(filepath.Join(bundlePath, file)) - if err != nil { - if os.IsNotExist(errors.Cause(err)) { - // File not found, moving on. - continue - } - return errors.Wrap(err, "failed to stat file") - } + wg.Add(1) - // Redact the target file of a symlink - // There is an opportunity for improving performance here by skipping symlinks - // if a target has been redacted already, but that would require - // some extra logic to ensure that a spec filtering only symlinks still works. - if info.Mode().Type() == os.ModeSymlink { - symlink := file - target, err := os.Readlink(filepath.Join(bundlePath, symlink)) + go func(file string, data []byte) { + defer wg.Done() + var reader io.Reader + if data == nil { + + // Collected contents are in a file. Get a reader to the file. + info, err := os.Lstat(filepath.Join(bundlePath, file)) if err != nil { - return errors.Wrap(err, "failed to read symlink") + if os.IsNotExist(errors.Cause(err)) { + // File not found, moving on. + return + } + errorCh <- errors.Wrap(err, "failed to stat file") + return } - // Get the relative path to the target file to conform with - // the path formats of the CollectorResult - file, err = filepath.Rel(bundlePath, target) - if err != nil { - return errors.Wrap(err, "failed to get relative path") + // Redact the target file of a symlink + // There is an opportunity for improving performance here by skipping symlinks + // if a target has been redacted already, but that would require + // some extra logic to ensure that a spec filtering only symlinks still works. + if info.Mode().Type() == os.ModeSymlink { + symlink := file + target, err := os.Readlink(filepath.Join(bundlePath, symlink)) + if err != nil { + errorCh <- errors.Wrap(err, "failed to read symlink") + return + } + // Get the relative path to the target file to conform with + // the path formats of the CollectorResult + file, err = filepath.Rel(bundlePath, target) + if err != nil { + errorCh <- errors.Wrap(err, "failed to get relative path") + return + } + klog.V(2).Infof("Redacting %s (symlink => %s)\n", file, symlink) + } else { + klog.V(2).Infof("Redacting %s\n", file) } - klog.V(2).Infof("Redacting %s (symlink => %s)\n", file, symlink) + r, err := input.GetReader(bundlePath, file) + if err != nil { + if os.IsNotExist(errors.Cause(err)) { + return + } + errorCh <- errors.Wrap(err, "failed to get reader") + return + } + defer r.Close() + + reader = r } else { - klog.V(2).Infof("Redacting %s\n", file) + // Collected contents are in memory. Get a reader to the memory buffer. + reader = bytes.NewBuffer(data) } - r, err := input.GetReader(bundlePath, file) - if err != nil { - if os.IsNotExist(errors.Cause(err)) { - continue + + // If the file is .tar, .tgz or .tar.gz, it must not be redacted. Instead it is + // decompressed and each file inside the tar redacted and compressed back into the archive. + if filepath.Ext(file) == ".tar" || filepath.Ext(file) == ".tgz" || strings.HasSuffix(file, ".tar.gz") { + tmpDir, err := ioutil.TempDir("", "troubleshoot-subresult-") + if err != nil { + errorCh <- errors.Wrap(err, "failed to create temp dir") + return } - return errors.Wrap(err, "failed to get reader") + defer os.RemoveAll(tmpDir) + + subResult, tarHeaders, err := decompressFile(tmpDir, reader, file) + if err != nil { + errorCh <- errors.Wrap(err, "failed to decompress file") + return + } + err = RedactResult(tmpDir, subResult, additionalRedactors) + if err != nil { + errorCh <- errors.Wrap(err, "failed to redact file") + return + } + + dstFilename := filepath.Join(bundlePath, file) + err = compressFiles(tmpDir, subResult, tarHeaders, dstFilename) + if err != nil { + errorCh <- errors.Wrap(err, "failed to re-compress file") + return + } + + os.RemoveAll(tmpDir) // ensure clean up on each iteration in addition to the defer + + //Content of the tar file was redacted. return to next file. + return } - defer r.Close() - reader = r - } else { - // Collected contents are in memory. Get a reader to the memory buffer. - reader = bytes.NewBuffer(v) - } - - // If the file is .tar, .tgz or .tar.gz, it must not be redacted. Instead it is - // decompressed and each file inside the tar redacted and compressed back into the archive. - if filepath.Ext(file) == ".tar" || filepath.Ext(file) == ".tgz" || strings.HasSuffix(file, ".tar.gz") { - tmpDir, err := ioutil.TempDir("", "troubleshoot-subresult-") + redacted, err := redact.Redact(reader, file, additionalRedactors) if err != nil { - return errors.Wrap(err, "failed to create temp dir") + errorCh <- errors.Wrap(err, "failed to redact io stream") + return } - defer os.RemoveAll(tmpDir) - subResult, tarHeaders, err := decompressFile(tmpDir, reader, file) + err = input.ReplaceResult(bundlePath, file, redacted) if err != nil { - return errors.Wrap(err, "failed to decompress file") - } - err = RedactResult(tmpDir, subResult, additionalRedactors) - if err != nil { - return errors.Wrap(err, "failed to redact file") + errorCh <- errors.Wrap(err, "failed to create redacted result") + return } + }(k, v) + } - dstFilename := filepath.Join(bundlePath, file) - err = compressFiles(tmpDir, subResult, tarHeaders, dstFilename) - if err != nil { - return errors.Wrap(err, "failed to re-compress file") - } + go func() { + wg.Wait() + close(errorCh) + }() - os.RemoveAll(tmpDir) // ensure clean up on each iteration in addition to the defer - - //Content of the tar file was redacted. Continue to next file. - continue - } - - redacted, err := redact.Redact(reader, file, additionalRedactors) + for err := range errorCh { if err != nil { - return errors.Wrap(err, "failed to redact io stream") - } - - err = input.ReplaceResult(bundlePath, file, redacted) - if err != nil { - return errors.Wrap(err, "failed to create redacted result") + return err } } + return nil } diff --git a/pkg/collect/result.go b/pkg/collect/result.go index 45e6d299..997de610 100644 --- a/pkg/collect/result.go +++ b/pkg/collect/result.go @@ -127,7 +127,12 @@ func (r CollectorResult) SaveResult(bundlePath string, relativePath string, read return errors.Wrap(err, "failed to copy data") } - klog.V(2).Infof("Added %q to bundle output", relativePath) + fileInfo, err := f.Stat() + if err != nil { + return errors.Wrap(err, "failed to stat file") + } + + klog.V(2).Infof("Added %q (%d MB) to bundle output", relativePath, fileInfo.Size()/(1024*1024)) return nil } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index d49b81b0..c6037630 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -82,4 +82,7 @@ const ( // TermUI Display Constants MESSAGE_TEXT_PADDING = 4 MESSAGE_TEXT_LINES_MARGIN_TO_BOTTOM = 4 + + // Bufio Reader Constants + MAX_BUFFER_CAPACITY = 1024 * 1024 ) diff --git a/pkg/redact/multi_line.go b/pkg/redact/multi_line.go index dfdbb4f5..e4476a93 100644 --- a/pkg/redact/multi_line.go +++ b/pkg/redact/multi_line.go @@ -5,9 +5,11 @@ import ( "fmt" "io" "regexp" + "strings" ) type MultiLineRedactor struct { + scan *regexp.Regexp re1 *regexp.Regexp re2 *regexp.Regexp maskText string @@ -16,16 +18,26 @@ type MultiLineRedactor struct { isDefault bool } -func NewMultiLineRedactor(re1, re2, maskText, path, name string, isDefault bool) (*MultiLineRedactor, error) { - compiled1, err := regexp.Compile(re1) +func NewMultiLineRedactor(re1 LineRedactor, re2 string, maskText, path, name string, isDefault bool) (*MultiLineRedactor, error) { + var scanCompiled *regexp.Regexp + compiled1, err := regexp.Compile(re1.regex) if err != nil { return nil, err } + + if re1.scan != "" { + scanCompiled, err = regexp.Compile(re1.scan) + if err != nil { + return nil, err + } + } + compiled2, err := regexp.Compile(re2) if err != nil { return nil, err } - return &MultiLineRedactor{re1: compiled1, re2: compiled2, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil + + return &MultiLineRedactor{scan: scanCompiled, re1: compiled1, re2: compiled2, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil } func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader { @@ -52,6 +64,17 @@ func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader { for err == nil { lineNum++ // the first line that can be redacted is line 2 + // is scan is not nil, then check if line1 matches scan by lowercasing it + if r.scan != nil { + lowerLine1 := strings.ToLower(line1) + if !r.scan.MatchString(lowerLine1) { + fmt.Fprintf(writer, "%s\n", line1) + line1, line2, err = getNextTwoLines(reader, &line2) + flushLastLine = true + continue + } + } + // If line1 matches re1, then transform line2 using re2 if !r.re1.MatchString(line1) { fmt.Fprintf(writer, "%s\n", line1) @@ -60,7 +83,6 @@ func (r *MultiLineRedactor) Redact(input io.Reader, path string) io.Reader { continue } flushLastLine = false - clean := r.re2.ReplaceAllString(line2, substStr) // io.WriteString would be nicer, but reader strips new lines diff --git a/pkg/redact/multi_line_test.go b/pkg/redact/multi_line_test.go new file mode 100644 index 00000000..984cec7a --- /dev/null +++ b/pkg/redact/multi_line_test.go @@ -0,0 +1,111 @@ +package redact + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_NewMultiLineRedactorr(t *testing.T) { + tests := []struct { + name string + selector LineRedactor + scan string + redactor string + inputString string + wantString string + }{ + { + name: "Redact multiline with AWS secret access key", + selector: LineRedactor{ + regex: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + inputString: `"name": "secret_access_key" +"value": "dfeadsfsdfe"`, + wantString: `"name": "secret_access_key" +"value": "***HIDDEN***" +`, + }, + { + name: "Redact multiline with AWS secret id", + selector: LineRedactor{ + regex: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + inputString: `"name": "ACCESS_KEY_ID" +"value": "dfeadsfsdfe"`, + wantString: `"name": "ACCESS_KEY_ID" +"value": "***HIDDEN***" +`, + }, + { + name: "Redact multiline with OSD", + selector: LineRedactor{ + regex: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`, + }, + redactor: `(?i)("key": *")(?P.{38}==[^\"]*)(")`, + inputString: `"entity": "osd.1abcdef" +"key": "Gjt8s0WkfPtxZUo7gI8a0awbQGHgzuprdaedfb=="`, + wantString: `"entity": "osd.1abcdef" +"key": "***HIDDEN***" +`, + }, + { + name: "Redact multiline with AWS secret access key and scan regex", + selector: LineRedactor{ + regex: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`, + scan: `secret_?access_?key\"`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + inputString: `"name": "secret_access_key" +"value": "dfeadsfsdfe"`, + wantString: `"name": "secret_access_key" +"value": "***HIDDEN***" +`, + }, + { + name: "Redact multiline with AWS secret id and scan regex", + selector: LineRedactor{ + regex: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`, + scan: `access_?key_?id\"`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + inputString: `"name": "ACCESS_KEY_ID" +"value": "dfeadsfsdfe"`, + wantString: `"name": "ACCESS_KEY_ID" +"value": "***HIDDEN***" +`, + }, + { + name: "Redact multiline with OSD and scan regex", + selector: LineRedactor{ + regex: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`, + scan: `(osd|client|mgr)`, + }, + redactor: `(?i)("key": *")(?P.{38}==[^\"]*)(")`, + inputString: `"entity": "osd.1abcdef" +"key": "Gjt8s0WkfPtxZUo7gI8a0awbQGHgzuprdaedfb=="`, + wantString: `"entity": "osd.1abcdef" +"key": "***HIDDEN***" +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + + reRunner, err := NewMultiLineRedactor(tt.selector, tt.redactor, MASK_TEXT, "testfile", tt.name, true) + req.NoError(err) + outReader := reRunner.Redact(bytes.NewReader([]byte(tt.inputString)), "") + + gotBytes, err := ioutil.ReadAll(outReader) + req.NoError(err) + req.Equal(tt.wantString, string(gotBytes)) + GetRedactionList() + ResetRedactionList() + }) + } +} diff --git a/pkg/redact/redact.go b/pkg/redact/redact.go index 920a3756..499c1009 100644 --- a/pkg/redact/redact.go +++ b/pkg/redact/redact.go @@ -46,6 +46,11 @@ type Redaction struct { IsDefaultRedactor bool `json:"isDefaultRedactor" yaml:"isDefaultRedactor"` } +type LineRedactor struct { + regex string + scan string +} + func Redact(input io.Reader, path string, additionalRedactors []*troubleshootv1beta2.Redact) (io.Reader, error) { redactors, err := getRedactors(path) if err != nil { @@ -105,12 +110,16 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta2.Redact for j, re := range redact.Removals.Regex { var newRedactor Redactor if re.Selector != "" { - newRedactor, err = NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT, path, redactorName(i, j, redact.Name, "multiLine"), false) + newRedactor, err = NewMultiLineRedactor(LineRedactor{ + regex: re.Selector, + }, re.Redactor, MASK_TEXT, path, redactorName(i, j, redact.Name, "multiLine"), false) if err != nil { return nil, errors.Wrapf(err, "multiline redactor %+v", re) } } else { - newRedactor, err = NewSingleLineRedactor(re.Redactor, MASK_TEXT, path, redactorName(i, j, redact.Name, "regex"), false) + newRedactor, err = NewSingleLineRedactor(LineRedactor{ + regex: re.Redactor, + }, MASK_TEXT, path, redactorName(i, j, redact.Name, "regex"), false) if err != nil { return nil, errors.Wrapf(err, "redactor %q", re) } @@ -165,88 +174,143 @@ func getRedactors(path string) ([]Redactor, error) { // groups named with `?P` will be masked // groups named with `?P` will be removed (replaced with empty strings) singleLines := []struct { - regex string + regex LineRedactor name string }{ // aws secrets { - regex: `(?i)(\\\"name\\\":\\\"[^\"]*SECRET_?ACCESS_?KEY\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "Redact values for environment variables that look like AWS Secret Access Keys", + regex: LineRedactor{ + regex: `(?i)(\\\"name\\\":\\\"[^\"]*SECRET_?ACCESS_?KEY\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + scan: `secret_?access_?key`, + }, + name: "Redact values for environment variables that look like AWS Secret Access Keys", }, { - regex: `(?i)(\\\"name\\\":\\\"[^\"]*ACCESS_?KEY_?ID\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "Redact values for environment variables that look like AWS Access Keys", + regex: LineRedactor{ + regex: `(?i)(\\\"name\\\":\\\"[^\"]*ACCESS_?KEY_?ID\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + scan: `access_?key_?id`, + }, + name: "Redact values for environment variables that look like AWS Access Keys", }, { - regex: `(?i)(\\\"name\\\":\\\"[^\"]*OWNER_?ACCOUNT\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "Redact values for environment variables that look like AWS Owner or Account numbers", + regex: LineRedactor{ + regex: `(?i)(\\\"name\\\":\\\"[^\"]*OWNER_?ACCOUNT\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + scan: `owner_?account`, + }, + name: "Redact values for environment variables that look like AWS Owner or Account numbers", }, // passwords in general { - regex: `(?i)(\\\"name\\\":\\\"[^\"]*password[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "Redact values for environment variables with names beginning with 'password'", + regex: LineRedactor{ + regex: `(?i)(\\\"name\\\":\\\"[^\"]*password[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + scan: `password`, + }, + name: "Redact values for environment variables with names beginning with 'password'", }, // tokens in general { - regex: `(?i)(\\\"name\\\":\\\"[^\"]*token[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "Redact values for environment variables with names beginning with 'token'", + + regex: LineRedactor{ + regex: `(?i)(\\\"name\\\":\\\"[^\"]*token[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + scan: `token`, + }, + name: "Redact values for environment variables with names beginning with 'token'", }, { - regex: `(?i)(\\\"name\\\":\\\"[^\"]*database[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "Redact values for environment variables with names beginning with 'database'", + regex: LineRedactor{ + regex: `(?i)(\\\"name\\\":\\\"[^\"]*database[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + scan: `database`, + }, + name: "Redact values for environment variables with names beginning with 'database'", }, { - regex: `(?i)(\\\"name\\\":\\\"[^\"]*user[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "Redact values for environment variables with names beginning with 'user'", + regex: LineRedactor{ + regex: `(?i)(\\\"name\\\":\\\"[^\"]*user[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + scan: `user`, + }, + name: "Redact values for environment variables with names beginning with 'user'", }, // connection strings with username and password // http://user:password@host:8888 { - regex: `(?i)(https?|ftp)(:\/\/)(?P[^:\"\/]+){1}(:)(?P[^@\"\/]+){1}(?P@[^:\/\s\"]+){1}(?P:[\d]+)?`, - name: "Redact connection strings with username and password", + regex: LineRedactor{ + regex: `(?i)(https?|ftp)(:\/\/)(?P[^:\"\/]+){1}(:)(?P[^@\"\/]+){1}(?P@[^:\/\s\"]+){1}(?P:[\d]+)?`, + scan: `https?|ftp`, + }, + name: "Redact connection strings with username and password", }, // user:password@tcp(host:3309)/db-name { - regex: `\b(?P[^:\"\/]*){1}(:)(?P[^:\"\/]*){1}(@tcp\()(?P[^:\"\/]*){1}(?P:[\d]*)?(\)\/)(?P[\w\d\S-_]+){1}\b`, - name: "Redact database connection strings that contain username and password", + regex: LineRedactor{ + regex: `\b(?P[^:\"\/]*){1}(:)(?P[^:\"\/]*){1}(@tcp\()(?P[^:\"\/]*){1}(?P:[\d]*)?(\)\/)(?P[\w\d\S-_]+){1}\b`, + scan: `@tcp`, + }, + name: "Redact database connection strings that contain username and password", }, // standard postgres and mysql connection strings // protocol://user:password@host:5432/db { - regex: `\b(\w*:\/\/)(?P[^:\"\/]*){1}(:)(?P[^:\"\/]*){1}(@)(?P[^:\"\/]*){1}(?P:[\d]*)?(\/)(?P[\w\d\S-_]+){1}\b`, - name: "Redact database connection strings that contain username and password", + regex: LineRedactor{ + regex: `\b(\w*:\/\/)(?P[^:\"\/]*){1}(:)(?P[^:\"\/]*){1}(@)(?P[^:\"\/]*){1}(?P:[\d]*)?(\/)(?P[\w\d\S-_]+){1}\b`, + scan: `\b(\w*:\/\/)([^:\"\/]*)(:)([^@\"\/]*)(@)([^:\"\/]*)(:[\d]*)?(\/)([\w\d\S-_]+)\b`, + }, + name: "Redact database connection strings that contain username and password", }, { - regex: `(?i)(Data Source *= *)(?P[^\;]+)(;)`, - name: "Redact 'Data Source' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(Data Source *= *)(?P[^\;]+)(;)`, + scan: `data source`, + }, + name: "Redact 'Data Source' values commonly found in database connection strings", }, { - regex: `(?i)(location *= *)(?P[^\;]+)(;)`, - name: "Redact 'location' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(location *= *)(?P[^\;]+)(;)`, + scan: `location`, + }, + name: "Redact 'location' values commonly found in database connection strings", }, { - regex: `(?i)(User ID *= *)(?P[^\;]+)(;)`, - name: "Redact 'User ID' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(User ID *= *)(?P[^\;]+)(;)`, + scan: `user id`, + }, + name: "Redact 'User ID' values commonly found in database connection strings", }, { - regex: `(?i)(password *= *)(?P[^\;]+)(;)`, - name: "Redact 'password' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(password *= *)(?P[^\;]+)(;)`, + scan: `password`, + }, + name: "Redact 'password' values commonly found in database connection strings", }, { - regex: `(?i)(Server *= *)(?P[^\;]+)(;)`, - name: "Redact 'Server' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(Server *= *)(?P[^\;]+)(;)`, + scan: `server`, + }, + name: "Redact 'Server' values commonly found in database connection strings", }, { - regex: `(?i)(Database *= *)(?P[^\;]+)(;)`, - name: "Redact 'Database' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(Database *= *)(?P[^\;]+)(;)`, + scan: `database`, + }, + name: "Redact 'Database' values commonly found in database connection strings", }, { - regex: `(?i)(Uid *= *)(?P[^\;]+)(;)`, - name: "Redact 'UID' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(Uid *= *)(?P[^\;]+)(;)`, + scan: `uid`, + }, + name: "Redact 'UID' values commonly found in database connection strings", }, { - regex: `(?i)(Pwd *= *)(?P[^\;]+)(;)`, - name: "Redact 'Pwd' values commonly found in database connection strings", + regex: LineRedactor{ + regex: `(?i)(Pwd *= *)(?P[^\;]+)(;)`, + scan: `pwd`, + }, + name: "Redact 'Pwd' values commonly found in database connection strings", }, } @@ -260,54 +324,78 @@ func getRedactors(path string) ([]Redactor, error) { } doubleLines := []struct { - line1 string - line2 string - name string + selector LineRedactor + redactor string + name string }{ { - line1: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`, - line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "Redact AWS Secret Access Key values in multiline JSON", + selector: LineRedactor{ + regex: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`, + scan: `secret_?access_?key`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "Redact AWS Secret Access Key values in multiline JSON", }, { - line1: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`, - line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "Redact AWS Access Key ID values in multiline JSON", + selector: LineRedactor{ + regex: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`, + scan: `access_?key_?id`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "Redact AWS Access Key ID values in multiline JSON", }, { - line1: `(?i)"name": *"[^\"]*OWNER_?ACCOUNT[^\"]*"`, - line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "Redact AWS Owner and Account Numbers in multiline JSON", + selector: LineRedactor{ + regex: `(?i)"name": *"[^\"]*OWNER_?ACCOUNT[^\"]*"`, + scan: `owner_?account`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "Redact AWS Owner and Account Numbers in multiline JSON", }, { - line1: `(?i)"name": *".*password[^\"]*"`, - line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "Redact password environment variables in multiline JSON", + selector: LineRedactor{ + regex: `(?i)"name": *".*password[^\"]*"`, + scan: `password`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "Redact password environment variables in multiline JSON", }, { - line1: `(?i)"name": *".*token[^\"]*"`, - line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "Redact values that look like API tokens in multiline JSON", + selector: LineRedactor{ + regex: `(?i)"name": *".*token[^\"]*"`, + scan: `token`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "Redact values that look like API tokens in multiline JSON", }, { - line1: `(?i)"name": *".*database[^\"]*"`, - line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "Redact database connection strings in multiline JSON", + selector: LineRedactor{ + regex: `(?i)"name": *".*database[^\"]*"`, + scan: `database`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "Redact database connection strings in multiline JSON", }, { - line1: `(?i)"name": *".*user[^\"]*"`, - line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "Redact usernames in multiline JSON", + selector: LineRedactor{ + regex: `(?i)"name": *".*user[^\"]*"`, + scan: `user`, + }, + redactor: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "Redact usernames in multiline JSON", }, { - line1: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`, - line2: `(?i)("key": *")(?P.{38}==[^\"]*)(")`, - name: "Redact 'key' values found in Ceph auth lists", + selector: LineRedactor{ + regex: `(?i)"entity": *"(osd|client|mgr)\..*[^\"]*"`, + scan: `(osd|client|mgr)`, + }, + redactor: `(?i)("key": *")(?P.{38}==[^\"]*)(")`, + name: "Redact 'key' values found in Ceph auth lists", }, } for _, l := range doubleLines { - r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT, path, l.name, true) + r, err := NewMultiLineRedactor(l.selector, l.redactor, MASK_TEXT, path, l.name, true) if err != nil { return nil, err // maybe skip broken ones? } diff --git a/pkg/redact/redact_test.go b/pkg/redact/redact_test.go index 72f2ff9d..24eedc80 100644 --- a/pkg/redact/redact_test.go +++ b/pkg/redact/redact_test.go @@ -1741,6 +1741,7 @@ func Test_Redactors(t *testing.T) { ResetRedactionList() req.Len(actualRedactions.ByFile["testpath"], wantRedactionsLen) req.Len(actualRedactions.ByRedactor, wantRedactionsCount) + ResetRedactionList() }) } diff --git a/pkg/redact/single_line.go b/pkg/redact/single_line.go index 58ab2bae..7fb7d493 100644 --- a/pkg/redact/single_line.go +++ b/pkg/redact/single_line.go @@ -5,9 +5,13 @@ import ( "fmt" "io" "regexp" + "strings" + + "github.com/replicatedhq/troubleshoot/pkg/constants" ) type SingleLineRedactor struct { + scan *regexp.Regexp re *regexp.Regexp maskText string filePath string @@ -15,12 +19,21 @@ type SingleLineRedactor struct { isDefault bool } -func NewSingleLineRedactor(re, maskText, path, name string, isDefault bool) (*SingleLineRedactor, error) { - compiled, err := regexp.Compile(re) +func NewSingleLineRedactor(re LineRedactor, maskText, path, name string, isDefault bool) (*SingleLineRedactor, error) { + var scanCompiled *regexp.Regexp + compiled, err := regexp.Compile(re.regex) if err != nil { return nil, err } - return &SingleLineRedactor{re: compiled, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil + + if re.scan != "" { + scanCompiled, err = regexp.Compile(re.scan) + if err != nil { + return nil, err + } + } + + return &SingleLineRedactor{scan: scanCompiled, re: compiled, maskText: maskText, filePath: path, redactName: name, isDefault: isDefault}, nil } func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader { @@ -38,16 +51,25 @@ func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader { substStr := getReplacementPattern(r.re, r.maskText) - reader := bufio.NewReader(input) + buf := make([]byte, constants.MAX_BUFFER_CAPACITY) + scanner := bufio.NewScanner(input) + scanner.Buffer(buf, constants.MAX_BUFFER_CAPACITY) + lineNum := 0 - for { + for scanner.Scan() { lineNum++ - var line string - line, err = readLine(reader) - if err != nil { - return + line := scanner.Text() + + // is scan is not nil, then check if line matches scan by lowercasing it + if r.scan != nil { + lowerLine := strings.ToLower(line) + if !r.scan.MatchString(lowerLine) { + fmt.Fprintf(writer, "%s\n", line) + continue + } } + // if scan matches, but re does not, do not redact if !r.re.MatchString(line) { fmt.Fprintf(writer, "%s\n", line) continue @@ -57,6 +79,7 @@ func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader { // io.WriteString would be nicer, but scanner strips new lines fmt.Fprintf(writer, "%s\n", clean) + if err != nil { return } @@ -72,6 +95,9 @@ func (r *SingleLineRedactor) Redact(input io.Reader, path string) io.Reader { }) } } + if scanErr := scanner.Err(); scanErr != nil { + err = scanErr + } }() return out } diff --git a/pkg/redact/single_line_test.go b/pkg/redact/single_line_test.go index 96c98a3a..1968334b 100644 --- a/pkg/redact/single_line_test.go +++ b/pkg/redact/single_line_test.go @@ -12,6 +12,7 @@ func TestNewSingleLineRedactor(t *testing.T) { tests := []struct { name string re string + scan string inputString string wantString string wantRedactions RedactionList @@ -100,12 +101,222 @@ func TestNewSingleLineRedactor(t *testing.T) { }, }, }, + { + name: "Redact values for environment variables that look like AWS Secret Access Keys", + re: `(?i)("name":"[^\"]*SECRET_?ACCESS_?KEY","value":")(?P[^\"]*)(")`, + inputString: `{"name":"SECRET_ACCESS_KEY","value":"123"}`, + wantString: `{"name":"SECRET_ACCESS_KEY","value":"***HIDDEN***"} +`, + wantRedactions: RedactionList{ + ByRedactor: map[string][]Redaction{ + "Redact values for environment variables that look like AWS Secret Access Keys": []Redaction{ + { + RedactorName: "Redact values for environment variables that look like AWS Secret Access Keys", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + ByFile: map[string][]Redaction{ + "testfile": []Redaction{ + { + RedactorName: "Redact values for environment variables that look like AWS Secret Access Keys", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + }, + }, + { + name: "Redact connection strings with username and password", + re: `(?i)(https?|ftp)(:\/\/)(?P[^:\"\/]+){1}(:)(?P[^@\"\/]+){1}(?P@[^:\/\s\"]+){1}(?P:[\d]+)?`, + inputString: `http://user:password@host:8888`, + wantString: "http://***HIDDEN***:***HIDDEN***@host:8888\n", + wantRedactions: RedactionList{ + ByRedactor: map[string][]Redaction{ + "Redact connection strings with username and password": []Redaction{ + { + RedactorName: "Redact connection strings with username and password", + CharactersRemoved: -12, + Line: 1, + File: "testfile", + }, + }, + }, + ByFile: map[string][]Redaction{ + "testfile": []Redaction{ + { + RedactorName: "Redact connection strings with username and password", + CharactersRemoved: -12, + Line: 1, + File: "testfile", + }, + }, + }, + }, + }, + { + name: "Redact values for environment variables that look like AWS Secret Access Keys With Scan", + re: `(?i)("name":"[^\"]*SECRET_?ACCESS_?KEY","value":")(?P[^\"]*)(")`, + scan: `secret_?access_?key`, + inputString: `{"name":"SECRET_ACCESS_KEY","value":"123"}`, + wantString: `{"name":"SECRET_ACCESS_KEY","value":"***HIDDEN***"} +`, + wantRedactions: RedactionList{ + ByRedactor: map[string][]Redaction{ + "Redact values for environment variables that look like AWS Secret Access Keys With Scan": { + { + RedactorName: "Redact values for environment variables that look like AWS Secret Access Keys With Scan", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + ByFile: map[string][]Redaction{ + "testfile": { + { + RedactorName: "Redact values for environment variables that look like AWS Secret Access Keys With Scan", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + }, + }, + { + name: "Redact values for environment variables that look like Access Keys ID With Scan", + re: `(?i)("name":"[^\"]*ACCESS_?KEY_?ID","value":")(?P[^\"]*)(")`, + scan: `access_?key_?id`, + inputString: `{"name":"ACCESS_KEY_ID","value":"123"}`, + wantString: `{"name":"ACCESS_KEY_ID","value":"***HIDDEN***"} +`, + wantRedactions: RedactionList{ + ByRedactor: map[string][]Redaction{ + "Redact values for environment variables that look like Access Keys ID With Scan": { + { + RedactorName: "Redact values for environment variables that look like Access Keys ID With Scan", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + ByFile: map[string][]Redaction{ + "testfile": { + { + RedactorName: "Redact values for environment variables that look like Access Keys ID With Scan", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + }, + }, + { + name: "Redact values for environment variables that look like Owner Account With Scan", + re: `(?i)("name":"[^\"]*OWNER_?ACCOUNT","value":")(?P[^\"]*)(")`, + scan: `owner_?account`, + inputString: `{"name":"OWNER_ACCOUNT","value":"123"}`, + wantString: `{"name":"OWNER_ACCOUNT","value":"***HIDDEN***"} +`, + wantRedactions: RedactionList{ + ByRedactor: map[string][]Redaction{ + "Redact values for environment variables that look like Owner Account With Scan": { + { + RedactorName: "Redact values for environment variables that look like Owner Account With Scan", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + ByFile: map[string][]Redaction{ + "testfile": { + { + RedactorName: "Redact values for environment variables that look like Owner Account With Scan", + CharactersRemoved: -9, + Line: 1, + File: "testfile", + }, + }, + }, + }, + }, + { + name: "Redact 'Data Source' values With Scan", + re: `(?i)(Data Source *= *)(?P[^\;]+)(;)`, + scan: `data source`, + inputString: `Data Source = abcdef;`, + wantString: `Data Source = ***HIDDEN***; +`, + wantRedactions: RedactionList{ + ByRedactor: map[string][]Redaction{ + "Redact 'Data Source' values With Scan": { + { + RedactorName: "Redact 'Data Source' values With Scan", + CharactersRemoved: -6, + Line: 1, + File: "testfile", + }, + }, + }, + ByFile: map[string][]Redaction{ + "testfile": { + { + RedactorName: "Redact 'Data Source' values With Scan", + CharactersRemoved: -6, + Line: 1, + File: "testfile", + }, + }, + }, + }, + }, + { + name: "Redact connection strings With Scan", + re: `(?i)(https?|ftp)(:\/\/)(?P[^:\"\/]+){1}(:)(?P[^@\"\/]+){1}(?P@[^:\/\s\"]+){1}(?P:[\d]+)?`, + scan: `https?|ftp`, + inputString: `http://user:password@host:8888;`, + wantString: `http://***HIDDEN***:***HIDDEN***@host:8888; +`, + wantRedactions: RedactionList{ + ByRedactor: map[string][]Redaction{ + "Redact connection strings With Scan": { + { + RedactorName: "Redact connection strings With Scan", + CharactersRemoved: -12, + Line: 1, + File: "testfile", + }, + }, + }, + ByFile: map[string][]Redaction{ + "testfile": { + { + RedactorName: "Redact connection strings With Scan", + CharactersRemoved: -12, + Line: 1, + File: "testfile", + }, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := require.New(t) ResetRedactionList() - reRunner, err := NewSingleLineRedactor(tt.re, MASK_TEXT, "testfile", tt.name, false) + reRunner, err := NewSingleLineRedactor(LineRedactor{ + regex: tt.re, + scan: tt.scan, + }, MASK_TEXT, "testfile", tt.name, false) req.NoError(err) outReader := reRunner.Redact(bytes.NewReader([]byte(tt.inputString)), "")