From 6c61040f745057595294e461d920636014b92fcc Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 26 May 2020 17:15:30 -0400 Subject: [PATCH 1/4] default redactor name improvements for instance, defaultRegex.SECRET_ACCESS_KEY instead of the raw regex --- pkg/redact/redact.go | 131 ++++++++++++++++++++++++++++++++----------- 1 file changed, 98 insertions(+), 33 deletions(-) diff --git a/pkg/redact/redact.go b/pkg/redact/redact.go index 789704f6..3597c38a 100644 --- a/pkg/redact/redact.go +++ b/pkg/redact/redact.go @@ -106,7 +106,7 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact withinRedactNum := 0 // give unique redaction names for _, re := range redact.Regex { - r, err := NewSingleLineRedactor(re, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "regex", "")) + r, err := NewSingleLineRedactor(re, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "regex")) if err != nil { return nil, errors.Wrapf(err, "redactor %q", re) } @@ -114,11 +114,11 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact } for _, literal := range redact.Values { - additionalRedactors = append(additionalRedactors, literalString(literal, path, redactorName(i, withinRedactNum, redact.Name, "literal", ""))) + additionalRedactors = append(additionalRedactors, literalString(literal, path, redactorName(i, withinRedactNum, redact.Name, "literal"))) } for _, re := range redact.MultiLine { - r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "multiLine", "")) + r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "multiLine")) if err != nil { return nil, errors.Wrapf(err, "multiline redactor %+v", re) } @@ -126,7 +126,7 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact } for _, yaml := range redact.Yaml { - r := NewYamlRedactor(yaml, path, redactorName(i, withinRedactNum, redact.Name, "yaml", "")) + r := NewYamlRedactor(yaml, path, redactorName(i, withinRedactNum, redact.Name, "yaml")) additionalRedactors = append(additionalRedactors, r) } } @@ -171,39 +171,96 @@ func getRedactors(path string) ([]Redactor, error) { // (?i) makes it case insensitive // groups named with `?P` will be masked // groups named with `?P` will be removed (replaced with empty strings) - singleLines := []string{ + singleLines := []struct { + regex string + name string + }{ // ipv4 - `(?P\b(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b)`, + { + regex: `(?P\b(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b)`, + name: "ipv4", + }, // TODO: ipv6 // aws secrets - `(?i)(\\\"name\\\":\\\"[^\"]*SECRET_?ACCESS_?KEY\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - `(?i)(\\\"name\\\":\\\"[^\"]*ACCESS_?KEY_?ID\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - `(?i)(\\\"name\\\":\\\"[^\"]*OWNER_?ACCOUNT\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + { + regex: `(?i)(\\\"name\\\":\\\"[^\"]*SECRET_?ACCESS_?KEY\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + name: "SECRET_ACCESS_KEY", + }, + { + regex: `(?i)(\\\"name\\\":\\\"[^\"]*ACCESS_?KEY_?ID\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + name: "ACCESS_KEY_ID", + }, + { + regex: `(?i)(\\\"name\\\":\\\"[^\"]*OWNER_?ACCOUNT\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + name: "OWNER_ACCOUNT", + }, // passwords in general - `(?i)(\\\"name\\\":\\\"[^\"]*password[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + { + regex: `(?i)(\\\"name\\\":\\\"[^\"]*password[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + name: "password", + }, // tokens in general - `(?i)(\\\"name\\\":\\\"[^\"]*token[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - `(?i)(\\\"name\\\":\\\"[^\"]*database[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - `(?i)(\\\"name\\\":\\\"[^\"]*user[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + { + regex: `(?i)(\\\"name\\\":\\\"[^\"]*token[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + name: "token", + }, + { + regex: `(?i)(\\\"name\\\":\\\"[^\"]*database[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + name: "database", + }, + { + regex: `(?i)(\\\"name\\\":\\\"[^\"]*user[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, + name: "user", + }, // connection strings with username and password // http://user:password@host:8888 - `(?i)(https?|ftp)(:\/\/)(?P[^:\"\/]+){1}(:)(?P[^@\"\/]+){1}(?P@[^:\/\s\"]+){1}(?P:[\d]+)?`, + { + regex: `(?i)(https?|ftp)(:\/\/)(?P[^:\"\/]+){1}(:)(?P[^@\"\/]+){1}(?P@[^:\/\s\"]+){1}(?P:[\d]+)?`, + name: "http://user:password@host:8888", + }, // user:password@tcp(host:3309)/db-name - `\b(?P[^:\"\/]*){1}(:)(?P[^:\"\/]*){1}(@tcp\()(?P[^:\"\/]*){1}(?P:[\d]*)?(\)\/)(?P[\w\d\S-_]+){1}\b`, - // standard postgres and mysql connnection strings - `(?i)(Data Source *= *)(?P[^\;]+)(;)`, - `(?i)(location *= *)(?P[^\;]+)(;)`, - `(?i)(User ID *= *)(?P[^\;]+)(;)`, - `(?i)(password *= *)(?P[^\;]+)(;)`, - `(?i)(Server *= *)(?P[^\;]+)(;)`, - `(?i)(Database *= *)(?P[^\;]+)(;)`, - `(?i)(Uid *= *)(?P[^\;]+)(;)`, - `(?i)(Pwd *= *)(?P[^\;]+)(;)`, + { + regex: `\b(?P[^:\"\/]*){1}(:)(?P[^:\"\/]*){1}(@tcp\()(?P[^:\"\/]*){1}(?P:[\d]*)?(\)\/)(?P[\w\d\S-_]+){1}\b`, + name: "user:password@tcp(host:3309)/db-name", + }, + // standard postgres and mysql connection strings + { + regex: `(?i)(Data Source *= *)(?P[^\;]+)(;)`, + name: "Data Source", + }, + { + regex: `(?i)(location *= *)(?P[^\;]+)(;)`, + name: "location", + }, + { + regex: `(?i)(User ID *= *)(?P[^\;]+)(;)`, + name: "User ID", + }, + { + regex: `(?i)(password *= *)(?P[^\;]+)(;)`, + name: "db-password", + }, + { + regex: `(?i)(Server *= *)(?P[^\;]+)(;)`, + name: "server", + }, + { + regex: `(?i)(Database *= *)(?P[^\;]+)(;)`, + name: "db-database", + }, + { + regex: `(?i)(Uid *= *)(?P[^\;]+)(;)`, + name: "Uid", + }, + { + regex: `(?i)(Pwd *= *)(?P[^\;]+)(;)`, + name: "Pwd", + }, } redactors := make([]Redactor, 0) - for i, re := range singleLines { - r, err := NewSingleLineRedactor(re, MASK_TEXT, path, redactorName(-1, i, "", "defaultRegex", re)) + for _, re := range singleLines { + r, err := NewSingleLineRedactor(re.regex, MASK_TEXT, path, redactorName(-1, -1, re.name, "defaultRegex")) if err != nil { return nil, err // maybe skip broken ones? } @@ -213,39 +270,47 @@ func getRedactors(path string) ([]Redactor, error) { doubleLines := []struct { line1 string line2 string + name string }{ { line1: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "SECRET_ACCESS_KEY", }, { line1: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "ACCESS_KEY_ID", }, { line1: `(?i)"name": *"[^\"]*OWNER_?ACCOUNT[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "OWNER_ACCOUNT", }, { line1: `(?i)"name": *".*password[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "password", }, { line1: `(?i)"name": *".*token[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "token", }, { line1: `(?i)"name": *".*database[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "database", }, { line1: `(?i)"name": *".*user[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, + name: "user", }, } - for i, l := range doubleLines { - r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT, path, redactorName(-1, i, "", "defaultMultiLine", l.line1)) + for _, l := range doubleLines { + r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT, path, redactorName(-1, -1, l.name, "defaultMultiLine")) if err != nil { return nil, err // maybe skip broken ones? } @@ -302,12 +367,12 @@ func addRedaction(redaction Redaction) { }(redaction) } -func redactorName(redactorNum, withinRedactorNum int, redactorName, redactorType, redactorLiteral string) string { +func redactorName(redactorNum, withinRedactorNum int, redactorName, redactorType string) string { + if withinRedactorNum == -1 { + return fmt.Sprintf("%s.%q", redactorType, redactorName) + } if redactorName != "" { return fmt.Sprintf("%s-%d", redactorName, withinRedactorNum) } - if redactorLiteral == "" { - return fmt.Sprintf("unnamed-%d.%d-%s", redactorNum, withinRedactorNum, redactorType) - } - return fmt.Sprintf("%s.%d-%q", redactorType, withinRedactorNum, redactorLiteral) + return fmt.Sprintf("unnamed-%d.%d-%s", redactorNum, withinRedactorNum, redactorType) } From 869a9eafd31a22431ad33c109c9648dcbb3b7508 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 26 May 2020 18:07:23 -0400 Subject: [PATCH 2/4] actually increment withinRedactNum --- pkg/redact/redact.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/redact/redact.go b/pkg/redact/redact.go index 3597c38a..3deda340 100644 --- a/pkg/redact/redact.go +++ b/pkg/redact/redact.go @@ -111,10 +111,12 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact return nil, errors.Wrapf(err, "redactor %q", re) } additionalRedactors = append(additionalRedactors, r) + withinRedactNum++ } for _, literal := range redact.Values { additionalRedactors = append(additionalRedactors, literalString(literal, path, redactorName(i, withinRedactNum, redact.Name, "literal"))) + withinRedactNum++ } for _, re := range redact.MultiLine { @@ -123,11 +125,13 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact return nil, errors.Wrapf(err, "multiline redactor %+v", re) } additionalRedactors = append(additionalRedactors, r) + withinRedactNum++ } for _, yaml := range redact.Yaml { r := NewYamlRedactor(yaml, path, redactorName(i, withinRedactNum, redact.Name, "yaml")) additionalRedactors = append(additionalRedactors, r) + withinRedactNum++ } } return additionalRedactors, nil From 171c440f27447b71f796e1f9e7b89f3d12d09026 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 26 May 2020 18:13:48 -0400 Subject: [PATCH 3/4] improve generated names for custom redactors --- pkg/redact/redact.go | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/pkg/redact/redact.go b/pkg/redact/redact.go index 3deda340..0dac32a1 100644 --- a/pkg/redact/redact.go +++ b/pkg/redact/redact.go @@ -103,35 +103,29 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact continue } - withinRedactNum := 0 // give unique redaction names - - for _, re := range redact.Regex { - r, err := NewSingleLineRedactor(re, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "regex")) + for j, re := range redact.Regex { + r, err := NewSingleLineRedactor(re, MASK_TEXT, path, redactorName(i, j, redact.Name, "regex")) if err != nil { return nil, errors.Wrapf(err, "redactor %q", re) } additionalRedactors = append(additionalRedactors, r) - withinRedactNum++ } - for _, literal := range redact.Values { - additionalRedactors = append(additionalRedactors, literalString(literal, path, redactorName(i, withinRedactNum, redact.Name, "literal"))) - withinRedactNum++ + for j, literal := range redact.Values { + additionalRedactors = append(additionalRedactors, literalString(literal, path, redactorName(i, j, redact.Name, "literal"))) } - for _, re := range redact.MultiLine { - r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "multiLine")) + for j, re := range redact.MultiLine { + r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT, path, redactorName(i, j, redact.Name, "multiLine")) if err != nil { return nil, errors.Wrapf(err, "multiline redactor %+v", re) } additionalRedactors = append(additionalRedactors, r) - withinRedactNum++ } - for _, yaml := range redact.Yaml { - r := NewYamlRedactor(yaml, path, redactorName(i, withinRedactNum, redact.Name, "yaml")) + for j, yaml := range redact.Yaml { + r := NewYamlRedactor(yaml, path, redactorName(i, j, redact.Name, "yaml")) additionalRedactors = append(additionalRedactors, r) - withinRedactNum++ } } return additionalRedactors, nil @@ -376,7 +370,7 @@ func redactorName(redactorNum, withinRedactorNum int, redactorName, redactorType return fmt.Sprintf("%s.%q", redactorType, redactorName) } if redactorName != "" { - return fmt.Sprintf("%s-%d", redactorName, withinRedactorNum) + return fmt.Sprintf("%s.%s.%d", redactorName, redactorType, withinRedactorNum) } - return fmt.Sprintf("unnamed-%d.%d-%s", redactorNum, withinRedactorNum, redactorType) + return fmt.Sprintf("unnamed-%d.%s.%d", redactorNum, redactorType, withinRedactorNum) } From 553718eada2575b3cefa8bcd471a35c72fd2e696 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 26 May 2020 19:33:40 -0400 Subject: [PATCH 4/4] fully spell out default redactor names --- pkg/redact/redact.go | 57 +++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/pkg/redact/redact.go b/pkg/redact/redact.go index 0dac32a1..e8765f3f 100644 --- a/pkg/redact/redact.go +++ b/pkg/redact/redact.go @@ -176,89 +176,89 @@ func getRedactors(path string) ([]Redactor, error) { // ipv4 { regex: `(?P\b(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?P25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b)`, - name: "ipv4", + name: "default ipv4 redactor", }, // TODO: ipv6 // aws secrets { regex: `(?i)(\\\"name\\\":\\\"[^\"]*SECRET_?ACCESS_?KEY\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "SECRET_ACCESS_KEY", + name: "default SECRET_ACCESS_KEY redactor", }, { regex: `(?i)(\\\"name\\\":\\\"[^\"]*ACCESS_?KEY_?ID\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "ACCESS_KEY_ID", + name: "default ACCESS_KEY_ID redactor", }, { regex: `(?i)(\\\"name\\\":\\\"[^\"]*OWNER_?ACCOUNT\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "OWNER_ACCOUNT", + name: "default OWNER_ACCOUNT redactor", }, // passwords in general { regex: `(?i)(\\\"name\\\":\\\"[^\"]*password[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "password", + name: "default password redactor", }, // tokens in general { regex: `(?i)(\\\"name\\\":\\\"[^\"]*token[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "token", + name: "default token redactor", }, { regex: `(?i)(\\\"name\\\":\\\"[^\"]*database[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "database", + name: "default database redactor", }, { regex: `(?i)(\\\"name\\\":\\\"[^\"]*user[^\"]*\\\",\\\"value\\\":\\\")(?P[^\"]*)(\\\")`, - name: "user", + name: "default user redactor", }, // connection strings with username and password // http://user:password@host:8888 { regex: `(?i)(https?|ftp)(:\/\/)(?P[^:\"\/]+){1}(:)(?P[^@\"\/]+){1}(?P@[^:\/\s\"]+){1}(?P:[\d]+)?`, - name: "http://user:password@host:8888", + name: "default connection string redactor", }, // user:password@tcp(host:3309)/db-name { regex: `\b(?P[^:\"\/]*){1}(:)(?P[^:\"\/]*){1}(@tcp\()(?P[^:\"\/]*){1}(?P:[\d]*)?(\)\/)(?P[\w\d\S-_]+){1}\b`, - name: "user:password@tcp(host:3309)/db-name", + name: "default db connection string redactor", }, // standard postgres and mysql connection strings { regex: `(?i)(Data Source *= *)(?P[^\;]+)(;)`, - name: "Data Source", + name: "default Data Source redactor", }, { regex: `(?i)(location *= *)(?P[^\;]+)(;)`, - name: "location", + name: "default location redactor", }, { regex: `(?i)(User ID *= *)(?P[^\;]+)(;)`, - name: "User ID", + name: "default User ID redactor", }, { regex: `(?i)(password *= *)(?P[^\;]+)(;)`, - name: "db-password", + name: "default db-password redactor", }, { regex: `(?i)(Server *= *)(?P[^\;]+)(;)`, - name: "server", + name: "default server redactor", }, { regex: `(?i)(Database *= *)(?P[^\;]+)(;)`, - name: "db-database", + name: "default db-database redactor", }, { regex: `(?i)(Uid *= *)(?P[^\;]+)(;)`, - name: "Uid", + name: "default Uid redactor", }, { regex: `(?i)(Pwd *= *)(?P[^\;]+)(;)`, - name: "Pwd", + name: "default Pwd redactor", }, } redactors := make([]Redactor, 0) for _, re := range singleLines { - r, err := NewSingleLineRedactor(re.regex, MASK_TEXT, path, redactorName(-1, -1, re.name, "defaultRegex")) + r, err := NewSingleLineRedactor(re.regex, MASK_TEXT, path, re.name) if err != nil { return nil, err // maybe skip broken ones? } @@ -273,42 +273,42 @@ func getRedactors(path string) ([]Redactor, error) { { line1: `(?i)"name": *"[^\"]*SECRET_?ACCESS_?KEY[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "SECRET_ACCESS_KEY", + name: "default multiline SECRET_ACCESS_KEY redactor", }, { line1: `(?i)"name": *"[^\"]*ACCESS_?KEY_?ID[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "ACCESS_KEY_ID", + name: "default multiline ACCESS_KEY_ID redactor", }, { line1: `(?i)"name": *"[^\"]*OWNER_?ACCOUNT[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "OWNER_ACCOUNT", + name: "default multiline OWNER_ACCOUNT redactor", }, { line1: `(?i)"name": *".*password[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "password", + name: "default multiline password redactor", }, { line1: `(?i)"name": *".*token[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "token", + name: "default multiline token redactor", }, { line1: `(?i)"name": *".*database[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "database", + name: "default multiline database redactor", }, { line1: `(?i)"name": *".*user[^\"]*"`, line2: `(?i)("value": *")(?P.*[^\"]*)(")`, - name: "user", + name: "default multiline user redactor", }, } for _, l := range doubleLines { - r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT, path, redactorName(-1, -1, l.name, "defaultMultiLine")) + r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT, path, l.name) if err != nil { return nil, err // maybe skip broken ones? } @@ -366,9 +366,6 @@ func addRedaction(redaction Redaction) { } func redactorName(redactorNum, withinRedactorNum int, redactorName, redactorType string) string { - if withinRedactorNum == -1 { - return fmt.Sprintf("%s.%q", redactorType, redactorName) - } if redactorName != "" { return fmt.Sprintf("%s.%s.%d", redactorName, redactorType, withinRedactorNum) }