Merge pull request #184 from replicatedhq/laverya/custom-multiline-and-yaml-redactors

custom multiline and yaml redactors

pkg/apis/troubleshoot/v1beta1/redact_shared.go

@@ -1,9 +1,16 @@
 package v1beta1
 
-type Redact struct {
-	Name   string   `json:"name,omitempty" yaml:"name,omitempty"`
-	File   string   `json:"file,omitempty" yaml:"file,omitempty"`
-	Files  []string `json:"files,omitempty" yaml:"files,omitempty"`
-	Values []string `json:"values,omitempty" yaml:"values,omitempty"`
-	Regex  []string `json:"regex,omitempty" yaml:"regex,omitempty"`
+type MultiLine struct {
+	Selector string `json:"selector,omitempty" yaml:"selector,omitempty"`
+	Redactor string `json:"redactor,omitempty" yaml:"redactor,omitempty"`
+}
+
+type Redact struct {
+	Name      string      `json:"name,omitempty" yaml:"name,omitempty"`
+	File      string      `json:"file,omitempty" yaml:"file,omitempty"`
+	Files     []string    `json:"files,omitempty" yaml:"files,omitempty"`
+	Values    []string    `json:"values,omitempty" yaml:"values,omitempty"`
+	Regex     []string    `json:"regex,omitempty" yaml:"regex,omitempty"`
+	MultiLine []MultiLine `json:"multiLine,omitempty" yaml:"multiLine,omitempty"`
+	Yaml      []string    `json:"yaml,omitempty" yaml:"yaml,omitempty"`
 }

pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go

@@ -915,6 +915,21 @@ func (in *Logs) DeepCopy() *Logs {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MultiLine) DeepCopyInto(out *MultiLine) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiLine.
+func (in *MultiLine) DeepCopy() *MultiLine {
+	if in == nil {
+		return nil
+	}
+	out := new(MultiLine)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NodeResourceFilters) DeepCopyInto(out *NodeResourceFilters) {
 	*out = *in
@@ -1165,6 +1180,16 @@ func (in *Redact) DeepCopyInto(out *Redact) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.MultiLine != nil {
+		in, out := &in.MultiLine, &out.MultiLine
+		*out = make([]MultiLine, len(*in))
+		copy(*out, *in)
+	}
+	if in.Yaml != nil {
+		in, out := &in.Yaml, &out.Yaml
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Redact.

pkg/collect/collector_test.go

@@ -176,6 +176,67 @@ pwd=somethinggoeshere;`,
 				"data/data/collectorname": `***HIDDEN*** ***HIDDEN***
 ***HIDDEN*** line here
 pwd=***HIDDEN***;
+`,
+			},
+		},
+		{
+			name: "data with custom yaml redactor",
+			Collect: &troubleshootv1beta1.Collect{
+				Data: &troubleshootv1beta1.Data{
+					CollectorMeta: troubleshootv1beta1.CollectorMeta{
+						CollectorName: "datacollectorname",
+						Exclude:       multitype.BoolOrString{},
+					},
+					Name: "data",
+					Data: `abc 123
+another line here`,
+				},
+			},
+			Redactors: []*troubleshootv1beta1.Redact{
+				{
+					Yaml: []string{
+						`abc`,
+					},
+				},
+			},
+			want: map[string]string{
+				"data/datacollectorname": `abc 123
+another line here
+`,
+			},
+		},
+		{
+			name: "custom multiline redactor",
+			Collect: &troubleshootv1beta1.Collect{
+				Data: &troubleshootv1beta1.Data{
+					CollectorMeta: troubleshootv1beta1.CollectorMeta{
+						CollectorName: "datacollectorname",
+						Exclude:       multitype.BoolOrString{},
+					},
+					Name: "data",
+					Data: `xyz123
+abc
+xyz123
+xyz123
+abc`,
+				},
+			},
+			Redactors: []*troubleshootv1beta1.Redact{
+				{
+					MultiLine: []troubleshootv1beta1.MultiLine{
+						{
+							Selector: "abc",
+							Redactor: "xyz(123)",
+						},
+					},
+				},
+			},
+			want: map[string]string{
+				"data/datacollectorname": `xyz123
+abc
+123
+xyz123
+abc
 `,
 			},
 		},

pkg/redact/literal.go

@@ -16,7 +16,7 @@ func literalString(matchString string) Redactor {
 }
 
 func (r literalRedactor) Redact(input io.Reader) io.Reader {
-	reader, writer := io.Pipe()
+	out, writer := io.Pipe()
 
 	go func() {
 		var err error
@@ -43,5 +43,5 @@ func (r literalRedactor) Redact(input io.Reader) io.Reader {
 			}
 		}
 	}()
-	return reader
+	return out
 }

pkg/redact/multi_line.go

@@ -26,7 +26,7 @@ func NewMultiLineRedactor(re1, re2, maskText string) (*MultiLineRedactor, error)
 }
 
 func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
-	reader, writer := io.Pipe()
+	out, writer := io.Pipe()
 	go func() {
 		var err error
 		defer func() {
@@ -70,7 +70,7 @@ func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
 			fmt.Fprintf(writer, "%s\n", line1)
 		}
 	}()
-	return reader
+	return out
 }
 
 func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line2 string, err error) {

pkg/redact/redact.go

@@ -65,7 +65,7 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact
 		for _, re := range redact.Regex {
 			r, err := NewSingleLineRedactor(re, MASK_TEXT)
 			if err != nil {
-				return nil, err // maybe skip broken ones?
+				return nil, errors.Wrapf(err, "redactor %q", re)
 			}
 			additionalRedactors = append(additionalRedactors, r)
 		}
@@ -73,6 +73,19 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact
 		for _, literal := range redact.Values {
 			additionalRedactors = append(additionalRedactors, literalString(literal))
 		}
+
+		for _, re := range redact.MultiLine {
+			r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT)
+			if err != nil {
+				return nil, errors.Wrapf(err, "multiline redactor %+v", re)
+			}
+			additionalRedactors = append(additionalRedactors, r)
+		}
+
+		for _, yaml := range redact.Yaml {
+			r := NewYamlRedactor(yaml)
+			additionalRedactors = append(additionalRedactors, r)
+		}
 	}
 	return additionalRedactors, nil
 }

pkg/redact/single_line.go

@@ -21,7 +21,7 @@ func NewSingleLineRedactor(re, maskText string) (*SingleLineRedactor, error) {
 }
 
 func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader {
-	reader, writer := io.Pipe()
+	out, writer := io.Pipe()
 
 	go func() {
 		var err error
@@ -57,5 +57,5 @@ func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader {
 			}
 		}
 	}()
-	return reader
+	return out
 }

pkg/redact/yaml.go

@@ -0,0 +1,106 @@
+package redact
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"strconv"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+type YamlRedactor struct {
+	maskPath   []string
+	foundMatch bool
+}
+
+func NewYamlRedactor(yamlPath string) *YamlRedactor {
+	pathComponents := strings.Split(yamlPath, ".")
+	return &YamlRedactor{maskPath: pathComponents}
+}
+
+func (r *YamlRedactor) Redact(input io.Reader) io.Reader {
+	reader, writer := io.Pipe()
+	go func() {
+		var err error
+		defer func() {
+			if err == io.EOF {
+				writer.Close()
+			} else {
+				writer.CloseWithError(err)
+			}
+		}()
+		reader := bufio.NewReader(input)
+
+		var doc []byte
+		doc, err = ioutil.ReadAll(reader)
+		var yamlInterface interface{}
+		err = yaml.Unmarshal(doc, &yamlInterface)
+		if err != nil {
+			buf := bytes.NewBuffer(doc)
+			buf.WriteTo(writer)
+			err = nil // this is not a fatal error
+			return
+		}
+
+		newYaml := r.redactYaml(yamlInterface, r.maskPath)
+		if !r.foundMatch {
+			// no match found, so make no changes
+			buf := bytes.NewBuffer(doc)
+			buf.WriteTo(writer)
+			return
+		}
+
+		var newBytes []byte
+		newBytes, err = yaml.Marshal(newYaml)
+		if err != nil {
+			return
+		}
+
+		buf := bytes.NewBuffer(newBytes)
+		buf.WriteTo(writer)
+		return
+	}()
+	return reader
+}
+
+func (r *YamlRedactor) redactYaml(in interface{}, path []string) interface{} {
+	if len(path) == 0 {
+		r.foundMatch = true
+		return MASK_TEXT
+	}
+	switch typed := in.(type) {
+	case []interface{}:
+		// check if first path element is * - if it is, run redact on all children
+		if path[0] == "*" {
+			var newArr []interface{}
+			for _, child := range typed {
+				newChild := r.redactYaml(child, path[1:])
+				newArr = append(newArr, newChild)
+			}
+			return newArr
+		}
+		// check if first path element is an integer - if it is, run redact on that child
+		pathIdx, err := strconv.Atoi(path[0])
+		if err != nil {
+			return typed
+		}
+		if len(typed) > pathIdx {
+			child := typed[pathIdx]
+			typed[pathIdx] = r.redactYaml(child, path[1:])
+			return typed
+		}
+		return typed
+	case map[interface{}]interface{}:
+		child, ok := typed[path[0]]
+		if ok {
+			newChild := r.redactYaml(child, path[1:])
+			typed[path[0]] = newChild
+		}
+		return typed
+	default:
+		return typed
+	}
+}

pkg/redact/yaml_test.go

@@ -0,0 +1,170 @@
+package redact
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.undefinedlabs.com/scopeagent"
+)
+
+func TestNewYamlRedactor(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        []string
+		inputString string
+		wantString  string
+	}{
+		{
+			name: "object paths",
+			path: []string{"abc", "xyz"},
+			inputString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+			wantString: `abc:
+  xyz: '***HIDDEN***'
+xyz:
+  hello: {}
+`,
+		},
+		{
+			name: "one index in array",
+			path: []string{"abc", "xyz", "0"},
+			inputString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+			wantString: `abc:
+  xyz:
+  - '***HIDDEN***'
+  - b
+xyz:
+  hello: {}
+`,
+		},
+		{
+			name: "index after end of array",
+			path: []string{"abc", "xyz", "10"},
+			inputString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+			wantString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+		},
+		{
+			name: "non-integer index",
+			path: []string{"abc", "xyz", "non-int"},
+			inputString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+			wantString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+		},
+		{
+			name: "object paths, no matches",
+			path: []string{"notexist", "xyz"},
+			inputString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+			wantString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+		},
+		{
+			name: "star index in array",
+			path: []string{"abc", "xyz", "*"},
+			inputString: `
+abc:
+  xyz:
+  - a
+  - b
+xyz:
+  hello: {}`,
+			wantString: `abc:
+  xyz:
+  - '***HIDDEN***'
+  - '***HIDDEN***'
+xyz:
+  hello: {}
+`,
+		},
+		{
+			name: "objects within array index in array",
+			path: []string{"abc", "xyz", "0", "a"},
+			inputString: `
+abc:
+  xyz:
+  - a: hello
+  - b
+xyz:
+  hello: {}`,
+			wantString: `abc:
+  xyz:
+  - a: '***HIDDEN***'
+  - b
+xyz:
+  hello: {}
+`,
+		},
+		{
+			name:        "non-yaml file",
+			path:        []string{""},
+			inputString: `hello world, this is not valid yaml: {`,
+			wantString:  `hello world, this is not valid yaml: {`,
+		},
+		{
+			name:        "no matches",
+			path:        []string{"abc"},
+			inputString: `improperly-formatted: yaml`,
+			wantString:  `improperly-formatted: yaml`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scopetest := scopeagent.StartTest(t)
+			defer scopetest.End()
+
+			req := require.New(t)
+			yamlRunner := YamlRedactor{maskPath: tt.path}
+
+			outReader := yamlRunner.Redact(bytes.NewReader([]byte(tt.inputString)))
+			gotBytes, err := ioutil.ReadAll(outReader)
+			req.NoError(err)
+			req.Equal(tt.wantString, string(gotBytes))
+		})
+	}
+}