diff --git a/pkg/apis/troubleshoot/v1beta1/redact_shared.go b/pkg/apis/troubleshoot/v1beta1/redact_shared.go index d6937b8b..fe48c059 100644 --- a/pkg/apis/troubleshoot/v1beta1/redact_shared.go +++ b/pkg/apis/troubleshoot/v1beta1/redact_shared.go @@ -1,9 +1,16 @@ package v1beta1 -type Redact struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - File string `json:"file,omitempty" yaml:"file,omitempty"` - Files []string `json:"files,omitempty" yaml:"files,omitempty"` - Values []string `json:"values,omitempty" yaml:"values,omitempty"` - Regex []string `json:"regex,omitempty" yaml:"regex,omitempty"` +type MultiLine struct { + Selector string `json:"selector,omitempty" yaml:"selector,omitempty"` + Redactor string `json:"redactor,omitempty" yaml:"redactor,omitempty"` +} + +type Redact struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + File string `json:"file,omitempty" yaml:"file,omitempty"` + Files []string `json:"files,omitempty" yaml:"files,omitempty"` + Values []string `json:"values,omitempty" yaml:"values,omitempty"` + Regex []string `json:"regex,omitempty" yaml:"regex,omitempty"` + MultiLine []MultiLine `json:"multiLine,omitempty" yaml:"multiLine,omitempty"` + Yaml []string `json:"yaml,omitempty" yaml:"yaml,omitempty"` } diff --git a/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go index 95e8c460..6b41a37e 100644 --- a/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go @@ -915,6 +915,21 @@ func (in *Logs) DeepCopy() *Logs { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultiLine) DeepCopyInto(out *MultiLine) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiLine. +func (in *MultiLine) DeepCopy() *MultiLine { + if in == nil { + return nil + } + out := new(MultiLine) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeResourceFilters) DeepCopyInto(out *NodeResourceFilters) { *out = *in @@ -1165,6 +1180,16 @@ func (in *Redact) DeepCopyInto(out *Redact) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.MultiLine != nil { + in, out := &in.MultiLine, &out.MultiLine + *out = make([]MultiLine, len(*in)) + copy(*out, *in) + } + if in.Yaml != nil { + in, out := &in.Yaml, &out.Yaml + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Redact. diff --git a/pkg/collect/collector_test.go b/pkg/collect/collector_test.go index b932f8fd..9142336a 100644 --- a/pkg/collect/collector_test.go +++ b/pkg/collect/collector_test.go @@ -176,6 +176,67 @@ pwd=somethinggoeshere;`, "data/data/collectorname": `***HIDDEN*** ***HIDDEN*** ***HIDDEN*** line here pwd=***HIDDEN***; +`, + }, + }, + { + name: "data with custom yaml redactor", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "datacollectorname", + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `abc 123 +another line here`, + }, + }, + Redactors: []*troubleshootv1beta1.Redact{ + { + Yaml: []string{ + `abc`, + }, + }, + }, + want: map[string]string{ + "data/datacollectorname": `abc 123 +another line here +`, + }, + }, + { + name: "custom multiline redactor", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "datacollectorname", + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `xyz123 +abc +xyz123 +xyz123 +abc`, + }, + }, + Redactors: []*troubleshootv1beta1.Redact{ + { + MultiLine: []troubleshootv1beta1.MultiLine{ + { + Selector: "abc", + Redactor: "xyz(123)", + }, + }, + }, + }, + want: map[string]string{ + "data/datacollectorname": `xyz123 +abc +123 +xyz123 +abc `, }, }, diff --git a/pkg/redact/literal.go b/pkg/redact/literal.go index ba270a3d..dfdbdfda 100644 --- a/pkg/redact/literal.go +++ b/pkg/redact/literal.go @@ -16,7 +16,7 @@ func literalString(matchString string) Redactor { } func (r literalRedactor) Redact(input io.Reader) io.Reader { - reader, writer := io.Pipe() + out, writer := io.Pipe() go func() { var err error @@ -43,5 +43,5 @@ func (r literalRedactor) Redact(input io.Reader) io.Reader { } } }() - return reader + return out } diff --git a/pkg/redact/multi_line.go b/pkg/redact/multi_line.go index 6e294e39..9c9f87a5 100644 --- a/pkg/redact/multi_line.go +++ b/pkg/redact/multi_line.go @@ -26,7 +26,7 @@ func NewMultiLineRedactor(re1, re2, maskText string) (*MultiLineRedactor, error) } func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader { - reader, writer := io.Pipe() + out, writer := io.Pipe() go func() { var err error defer func() { @@ -70,7 +70,7 @@ func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader { fmt.Fprintf(writer, "%s\n", line1) } }() - return reader + return out } func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line2 string, err error) { diff --git a/pkg/redact/redact.go b/pkg/redact/redact.go index 16a5df59..702638f5 100644 --- a/pkg/redact/redact.go +++ b/pkg/redact/redact.go @@ -65,7 +65,7 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact for _, re := range redact.Regex { r, err := NewSingleLineRedactor(re, MASK_TEXT) if err != nil { - return nil, err // maybe skip broken ones? + return nil, errors.Wrapf(err, "redactor %q", re) } additionalRedactors = append(additionalRedactors, r) } @@ -73,6 +73,19 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact for _, literal := range redact.Values { additionalRedactors = append(additionalRedactors, literalString(literal)) } + + for _, re := range redact.MultiLine { + r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT) + if err != nil { + return nil, errors.Wrapf(err, "multiline redactor %+v", re) + } + additionalRedactors = append(additionalRedactors, r) + } + + for _, yaml := range redact.Yaml { + r := NewYamlRedactor(yaml) + additionalRedactors = append(additionalRedactors, r) + } } return additionalRedactors, nil } diff --git a/pkg/redact/single_line.go b/pkg/redact/single_line.go index d9d37c4c..cdfc0788 100644 --- a/pkg/redact/single_line.go +++ b/pkg/redact/single_line.go @@ -21,7 +21,7 @@ func NewSingleLineRedactor(re, maskText string) (*SingleLineRedactor, error) { } func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader { - reader, writer := io.Pipe() + out, writer := io.Pipe() go func() { var err error @@ -57,5 +57,5 @@ func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader { } } }() - return reader + return out } diff --git a/pkg/redact/yaml.go b/pkg/redact/yaml.go new file mode 100644 index 00000000..59e93f80 --- /dev/null +++ b/pkg/redact/yaml.go @@ -0,0 +1,106 @@ +package redact + +import ( + "bufio" + "bytes" + "io" + "io/ioutil" + "strconv" + "strings" + + "gopkg.in/yaml.v2" +) + +type YamlRedactor struct { + maskPath []string + foundMatch bool +} + +func NewYamlRedactor(yamlPath string) *YamlRedactor { + pathComponents := strings.Split(yamlPath, ".") + return &YamlRedactor{maskPath: pathComponents} +} + +func (r *YamlRedactor) Redact(input io.Reader) io.Reader { + reader, writer := io.Pipe() + go func() { + var err error + defer func() { + if err == io.EOF { + writer.Close() + } else { + writer.CloseWithError(err) + } + }() + reader := bufio.NewReader(input) + + var doc []byte + doc, err = ioutil.ReadAll(reader) + var yamlInterface interface{} + err = yaml.Unmarshal(doc, &yamlInterface) + if err != nil { + buf := bytes.NewBuffer(doc) + buf.WriteTo(writer) + err = nil // this is not a fatal error + return + } + + newYaml := r.redactYaml(yamlInterface, r.maskPath) + if !r.foundMatch { + // no match found, so make no changes + buf := bytes.NewBuffer(doc) + buf.WriteTo(writer) + return + } + + var newBytes []byte + newBytes, err = yaml.Marshal(newYaml) + if err != nil { + return + } + + buf := bytes.NewBuffer(newBytes) + buf.WriteTo(writer) + return + }() + return reader +} + +func (r *YamlRedactor) redactYaml(in interface{}, path []string) interface{} { + if len(path) == 0 { + r.foundMatch = true + return MASK_TEXT + } + switch typed := in.(type) { + case []interface{}: + // check if first path element is * - if it is, run redact on all children + if path[0] == "*" { + var newArr []interface{} + for _, child := range typed { + newChild := r.redactYaml(child, path[1:]) + newArr = append(newArr, newChild) + } + return newArr + } + // check if first path element is an integer - if it is, run redact on that child + pathIdx, err := strconv.Atoi(path[0]) + if err != nil { + return typed + } + if len(typed) > pathIdx { + child := typed[pathIdx] + typed[pathIdx] = r.redactYaml(child, path[1:]) + return typed + } + return typed + case map[interface{}]interface{}: + child, ok := typed[path[0]] + if ok { + newChild := r.redactYaml(child, path[1:]) + typed[path[0]] = newChild + } + return typed + default: + return typed + } +} diff --git a/pkg/redact/yaml_test.go b/pkg/redact/yaml_test.go new file mode 100644 index 00000000..617bb4e8 --- /dev/null +++ b/pkg/redact/yaml_test.go @@ -0,0 +1,170 @@ +package redact + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/require" + "go.undefinedlabs.com/scopeagent" +) + +func TestNewYamlRedactor(t *testing.T) { + tests := []struct { + name string + path []string + inputString string + wantString string + }{ + { + name: "object paths", + path: []string{"abc", "xyz"}, + inputString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + wantString: `abc: + xyz: '***HIDDEN***' +xyz: + hello: {} +`, + }, + { + name: "one index in array", + path: []string{"abc", "xyz", "0"}, + inputString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + wantString: `abc: + xyz: + - '***HIDDEN***' + - b +xyz: + hello: {} +`, + }, + { + name: "index after end of array", + path: []string{"abc", "xyz", "10"}, + inputString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + wantString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + }, + { + name: "non-integer index", + path: []string{"abc", "xyz", "non-int"}, + inputString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + wantString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + }, + { + name: "object paths, no matches", + path: []string{"notexist", "xyz"}, + inputString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + wantString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + }, + { + name: "star index in array", + path: []string{"abc", "xyz", "*"}, + inputString: ` +abc: + xyz: + - a + - b +xyz: + hello: {}`, + wantString: `abc: + xyz: + - '***HIDDEN***' + - '***HIDDEN***' +xyz: + hello: {} +`, + }, + { + name: "objects within array index in array", + path: []string{"abc", "xyz", "0", "a"}, + inputString: ` +abc: + xyz: + - a: hello + - b +xyz: + hello: {}`, + wantString: `abc: + xyz: + - a: '***HIDDEN***' + - b +xyz: + hello: {} +`, + }, + { + name: "non-yaml file", + path: []string{""}, + inputString: `hello world, this is not valid yaml: {`, + wantString: `hello world, this is not valid yaml: {`, + }, + { + name: "no matches", + path: []string{"abc"}, + inputString: `improperly-formatted: yaml`, + wantString: `improperly-formatted: yaml`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scopetest := scopeagent.StartTest(t) + defer scopetest.End() + + req := require.New(t) + yamlRunner := YamlRedactor{maskPath: tt.path} + + outReader := yamlRunner.Redact(bytes.NewReader([]byte(tt.inputString))) + gotBytes, err := ioutil.ReadAll(outReader) + req.NoError(err) + req.Equal(tt.wantString, string(gotBytes)) + }) + } +}