remove per-collector redactors

This commit is contained in:
Andrew Lavery
2020-04-20 14:59:15 -04:00
parent 7be40ddf16
commit 8b54c28b4b
7 changed files with 106 additions and 142 deletions

View File

@@ -262,7 +262,6 @@ func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, addi
if additionalRedactors != nil {
globalRedactors = additionalRedactors.Spec.Redactors
}
globalRedactors = append(globalRedactors, collector.Spec.GlobalRedactors...)
// Run preflights collectors synchronously
for _, collector := range collectors {

View File

@@ -9,8 +9,7 @@ import (
)
type CollectorMeta struct {
CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"`
Redactors []*Redact `json:"redactors,omitempty" yaml:"redactors,omitempty"`
CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"`
// +optional
Exclude multitype.BoolOrString `json:"exclude,omitempty" yaml:"exclude,omitempty"`
}

View File

@@ -34,7 +34,6 @@ type AfterCollection struct {
type CollectorSpec struct {
Collectors []*Collect `json:"collectors,omitempty" yaml:"collectors,omitempty"`
AfterCollection []*AfterCollection `json:"afterCollection,omitempty" yaml:"afterCollection,omitempty"`
GlobalRedactors []*Redact `json:"globalRedactors,omitempty" yaml:"globalRedactors,omitempty"`
}
// CollectorStatus defines the observed state of Collector

View File

@@ -22,7 +22,7 @@ import (
// RedactorSpec defines the desired state of Redactor
type RedactorSpec struct {
Redactors []*Redact `json:"redacts,omitempty"`
Redactors []*Redact `json:"redactors,omitempty"`
}
// RedactorStatus defines the observed state of Redactor

View File

@@ -285,7 +285,7 @@ func (in *AnalyzerStatus) DeepCopy() *AnalyzerStatus {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterInfo) DeepCopyInto(out *ClusterInfo) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterInfo.
@@ -301,7 +301,7 @@ func (in *ClusterInfo) DeepCopy() *ClusterInfo {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterResources) DeepCopyInto(out *ClusterResources) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResources.
@@ -347,17 +347,17 @@ func (in *Collect) DeepCopyInto(out *Collect) {
if in.ClusterInfo != nil {
in, out := &in.ClusterInfo, &out.ClusterInfo
*out = new(ClusterInfo)
(*in).DeepCopyInto(*out)
**out = **in
}
if in.ClusterResources != nil {
in, out := &in.ClusterResources, &out.ClusterResources
*out = new(ClusterResources)
(*in).DeepCopyInto(*out)
**out = **in
}
if in.Secret != nil {
in, out := &in.Secret, &out.Secret
*out = new(Secret)
(*in).DeepCopyInto(*out)
**out = **in
}
if in.Logs != nil {
in, out := &in.Logs, &out.Logs
@@ -377,7 +377,7 @@ func (in *Collect) DeepCopyInto(out *Collect) {
if in.Data != nil {
in, out := &in.Data, &out.Data
*out = new(Data)
(*in).DeepCopyInto(*out)
**out = **in
}
if in.Copy != nil {
in, out := &in.Copy, &out.Copy
@@ -392,17 +392,17 @@ func (in *Collect) DeepCopyInto(out *Collect) {
if in.Postgres != nil {
in, out := &in.Postgres, &out.Postgres
*out = new(Database)
(*in).DeepCopyInto(*out)
**out = **in
}
if in.Mysql != nil {
in, out := &in.Mysql, &out.Mysql
*out = new(Database)
(*in).DeepCopyInto(*out)
**out = **in
}
if in.Redis != nil {
in, out := &in.Redis, &out.Redis
*out = new(Database)
(*in).DeepCopyInto(*out)
**out = **in
}
}
@@ -478,17 +478,6 @@ func (in *CollectorList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CollectorMeta) DeepCopyInto(out *CollectorMeta) {
*out = *in
if in.Redactors != nil {
in, out := &in.Redactors, &out.Redactors
*out = make([]*Redact, len(*in))
for i := range *in {
if (*in)[i] != nil {
in, out := &(*in)[i], &(*out)[i]
*out = new(Redact)
(*in).DeepCopyInto(*out)
}
}
}
out.Exclude = in.Exclude
}
@@ -527,17 +516,6 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) {
}
}
}
if in.GlobalRedactors != nil {
in, out := &in.GlobalRedactors, &out.GlobalRedactors
*out = make([]*Redact, len(*in))
for i := range *in {
if (*in)[i] != nil {
in, out := &(*in)[i], &(*out)[i]
*out = new(Redact)
(*in).DeepCopyInto(*out)
}
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollectorSpec.
@@ -595,7 +573,7 @@ func (in *ContainerRuntime) DeepCopy() *ContainerRuntime {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Copy) DeepCopyInto(out *Copy) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
if in.Selector != nil {
in, out := &in.Selector, &out.Selector
*out = make([]string, len(*in))
@@ -643,7 +621,7 @@ func (in *CustomResourceDefinition) DeepCopy() *CustomResourceDefinition {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Data) DeepCopyInto(out *Data) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Data.
@@ -659,7 +637,7 @@ func (in *Data) DeepCopy() *Data {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Database) DeepCopyInto(out *Database) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Database.
@@ -756,7 +734,7 @@ func (in *Distribution) DeepCopy() *Distribution {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Exec) DeepCopyInto(out *Exec) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
if in.Selector != nil {
in, out := &in.Selector, &out.Selector
*out = make([]string, len(*in))
@@ -809,7 +787,7 @@ func (in *Get) DeepCopy() *Get {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *HTTP) DeepCopyInto(out *HTTP) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
if in.Get != nil {
in, out := &in.Get, &out.Get
*out = new(Get)
@@ -909,7 +887,7 @@ func (in *LogLimits) DeepCopy() *LogLimits {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Logs) DeepCopyInto(out *Logs) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
if in.Selector != nil {
in, out := &in.Selector, &out.Selector
*out = make([]string, len(*in))
@@ -1317,7 +1295,7 @@ func (in *ResultRequest) DeepCopy() *ResultRequest {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Run) DeepCopyInto(out *Run) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
if in.Command != nil {
in, out := &in.Command, &out.Command
*out = make([]string, len(*in))
@@ -1343,7 +1321,7 @@ func (in *Run) DeepCopy() *Run {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Secret) DeepCopyInto(out *Secret) {
*out = *in
in.CollectorMeta.DeepCopyInto(&out.CollectorMeta)
out.CollectorMeta = in.CollectorMeta
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secret.

View File

@@ -44,7 +44,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
var unRedacted map[string][]byte
var isExcludedResult bool
var err error
var redactors []*troubleshootv1beta1.Redact
if c.Collect.ClusterInfo != nil {
isExcludedResult, err = isExcluded(c.Collect.ClusterInfo.Exclude)
if err != nil {
@@ -54,7 +53,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = ClusterInfo(c.GetContext())
redactors = c.Collect.ClusterInfo.Redactors
} else if c.Collect.ClusterResources != nil {
isExcludedResult, err = isExcluded(c.Collect.ClusterResources.Exclude)
if err != nil {
@@ -64,7 +62,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = ClusterResources(c.GetContext())
redactors = c.Collect.ClusterResources.Redactors
} else if c.Collect.Secret != nil {
isExcludedResult, err = isExcluded(c.Collect.Secret.Exclude)
if err != nil {
@@ -74,7 +71,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Secret(c.GetContext(), c.Collect.Secret)
redactors = c.Collect.Secret.Redactors
} else if c.Collect.Logs != nil {
isExcludedResult, err = isExcluded(c.Collect.Logs.Exclude)
if err != nil {
@@ -84,7 +80,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Logs(c.GetContext(), c.Collect.Logs)
redactors = c.Collect.Logs.Redactors
} else if c.Collect.Run != nil {
isExcludedResult, err = isExcluded(c.Collect.Run.Exclude)
if err != nil {
@@ -94,7 +89,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Run(c.GetContext(), c.Collect.Run)
redactors = c.Collect.Run.Redactors
} else if c.Collect.Exec != nil {
isExcludedResult, err = isExcluded(c.Collect.Exec.Exclude)
if err != nil {
@@ -104,7 +98,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Exec(c.GetContext(), c.Collect.Exec)
redactors = c.Collect.Exec.Redactors
} else if c.Collect.Data != nil {
isExcludedResult, err = isExcluded(c.Collect.Data.Exclude)
if err != nil {
@@ -114,7 +107,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Data(c.GetContext(), c.Collect.Data)
redactors = c.Collect.Data.Redactors
} else if c.Collect.Copy != nil {
isExcludedResult, err = isExcluded(c.Collect.Copy.Exclude)
if err != nil {
@@ -124,7 +116,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Copy(c.GetContext(), c.Collect.Copy)
redactors = c.Collect.Copy.Redactors
} else if c.Collect.HTTP != nil {
isExcludedResult, err = isExcluded(c.Collect.HTTP.Exclude)
if err != nil {
@@ -134,7 +125,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = HTTP(c.GetContext(), c.Collect.HTTP)
redactors = c.Collect.HTTP.Redactors
} else if c.Collect.Postgres != nil {
isExcludedResult, err = isExcluded(c.Collect.Postgres.Exclude)
if err != nil {
@@ -144,7 +134,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Postgres(c.GetContext(), c.Collect.Postgres)
redactors = c.Collect.Postgres.Redactors
} else if c.Collect.Mysql != nil {
isExcludedResult, err = isExcluded(c.Collect.Mysql.Exclude)
if err != nil {
@@ -154,7 +143,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Mysql(c.GetContext(), c.Collect.Mysql)
redactors = c.Collect.Mysql.Redactors
} else if c.Collect.Redis != nil {
isExcludedResult, err = isExcluded(c.Collect.Redis.Exclude)
if err != nil {
@@ -164,7 +152,6 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, nil
}
unRedacted, err = Redis(c.GetContext(), c.Collect.Redis)
redactors = c.Collect.Redis.Redactors
} else {
return nil, errors.New("no spec found to run")
}
@@ -173,7 +160,7 @@ func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Reda
return nil, err
}
if c.Redact {
return redactMap(unRedacted, append(redactors, globalRedactors...))
return redactMap(unRedacted, globalRedactors)
}
return unRedacted, nil
}

View File

@@ -11,9 +11,10 @@ import (
func TestCollector_RunCollectorSyncNoRedact(t *testing.T) {
tests := []struct {
name string
Collect *troubleshootv1beta1.Collect
want map[string]string
name string
Collect *troubleshootv1beta1.Collect
Redactors []*troubleshootv1beta1.Redact
want map[string]string
}{
{
name: "data with custom redactor",
@@ -21,19 +22,7 @@ func TestCollector_RunCollectorSyncNoRedact(t *testing.T) {
Data: &troubleshootv1beta1.Data{
CollectorMeta: troubleshootv1beta1.CollectorMeta{
CollectorName: "datacollectorname",
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "",
Files: nil,
Values: nil,
Regex: []string{
`abc`,
`(another)(?P<mask>.*)(here)`,
},
},
},
Exclude: multitype.BoolOrString{},
Exclude: multitype.BoolOrString{},
},
Name: "data",
Data: `abc 123
@@ -41,6 +30,18 @@ another line here
pwd=somethinggoeshere;`,
},
},
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "",
Files: nil,
Values: nil,
Regex: []string{
`abc`,
`(another)(?P<mask>.*)(here)`,
},
},
},
want: map[string]string{
"data/datacollectorname": ` 123
another***HIDDEN***here
@@ -54,17 +55,7 @@ pwd=***HIDDEN***;
Data: &troubleshootv1beta1.Data{
CollectorMeta: troubleshootv1beta1.CollectorMeta{
CollectorName: "datacollectorname",
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "data/*",
Values: nil,
Regex: []string{
`(another)(?P<mask>.*)(here)`,
},
},
},
Exclude: multitype.BoolOrString{},
Exclude: multitype.BoolOrString{},
},
Name: "data",
Data: `abc 123
@@ -72,6 +63,16 @@ another line here
pwd=somethinggoeshere;`,
},
},
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "data/*",
Values: nil,
Regex: []string{
`(another)(?P<mask>.*)(here)`,
},
},
},
want: map[string]string{
"data/datacollectorname": `abc 123
another***HIDDEN***here
@@ -85,17 +86,7 @@ pwd=***HIDDEN***;
Data: &troubleshootv1beta1.Data{
CollectorMeta: troubleshootv1beta1.CollectorMeta{
CollectorName: "datacollectorname",
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "notdata/*",
Values: nil,
Regex: []string{
`(another)(?P<mask>.*)(here)`,
},
},
},
Exclude: multitype.BoolOrString{},
Exclude: multitype.BoolOrString{},
},
Name: "data",
Data: `abc 123
@@ -103,6 +94,16 @@ another line here
pwd=somethinggoeshere;`,
},
},
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "notdata/*",
Values: nil,
Regex: []string{
`(another)(?P<mask>.*)(here)`,
},
},
},
want: map[string]string{
"data/datacollectorname": `abc 123
another line here
@@ -116,20 +117,7 @@ pwd=***HIDDEN***;
Data: &troubleshootv1beta1.Data{
CollectorMeta: troubleshootv1beta1.CollectorMeta{
CollectorName: "datacollectorname",
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
Files: []string{
"notData/*",
"data/*",
},
Values: nil,
Regex: []string{
`(another)(?P<mask>.*)(here)`,
},
},
},
Exclude: multitype.BoolOrString{},
Exclude: multitype.BoolOrString{},
},
Name: "data",
Data: `abc 123
@@ -137,6 +125,19 @@ another line here
pwd=somethinggoeshere;`,
},
},
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
Files: []string{
"notData/*",
"data/*",
},
Values: nil,
Regex: []string{
`(another)(?P<mask>.*)(here)`,
},
},
},
want: map[string]string{
"data/datacollectorname": `abc 123
another***HIDDEN***here
@@ -150,20 +151,7 @@ pwd=***HIDDEN***;
Data: &troubleshootv1beta1.Data{
CollectorMeta: troubleshootv1beta1.CollectorMeta{
CollectorName: "data/collectorname",
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
Files: []string{
"data/*/*",
},
Values: []string{
`abc`,
`123`,
`another`,
},
},
},
Exclude: multitype.BoolOrString{},
Exclude: multitype.BoolOrString{},
},
Name: "data",
Data: `abc 123
@@ -171,6 +159,19 @@ another line here
pwd=somethinggoeshere;`,
},
},
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
Files: []string{
"data/*/*",
},
Values: []string{
`abc`,
`123`,
`another`,
},
},
},
want: map[string]string{
"data/data/collectorname": `***HIDDEN*** ***HIDDEN***
***HIDDEN*** line here
@@ -189,7 +190,7 @@ pwd=***HIDDEN***;
Collect: tt.Collect,
Redact: true,
}
got, err := c.RunCollectorSync(nil)
got, err := c.RunCollectorSync(tt.Redactors)
req.NoError(err)
// convert to string to make differences easier to see
@@ -204,9 +205,10 @@ pwd=***HIDDEN***;
func TestCollector_RunCollectorSync(t *testing.T) {
tests := []struct {
name string
Collect *troubleshootv1beta1.Collect
want map[string]string
name string
Collect *troubleshootv1beta1.Collect
Redactors []*troubleshootv1beta1.Redact
want map[string]string
}{
{
name: "data with custom redactor - but redaction disabled",
@@ -214,19 +216,7 @@ func TestCollector_RunCollectorSync(t *testing.T) {
Data: &troubleshootv1beta1.Data{
CollectorMeta: troubleshootv1beta1.CollectorMeta{
CollectorName: "datacollectorname",
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "",
Files: nil,
Values: nil,
Regex: []string{
`abc`,
`(another)(?P<mask>.*)(here)`,
},
},
},
Exclude: multitype.BoolOrString{},
Exclude: multitype.BoolOrString{},
},
Name: "data",
Data: `abc 123
@@ -234,6 +224,18 @@ another line here
pwd=somethinggoeshere;`,
},
},
Redactors: []*troubleshootv1beta1.Redact{
{
Name: "",
File: "",
Files: nil,
Values: nil,
Regex: []string{
`abc`,
`(another)(?P<mask>.*)(here)`,
},
},
},
want: map[string]string{
"data/datacollectorname": `abc 123
another line here
@@ -251,7 +253,7 @@ pwd=somethinggoeshere;`,
Collect: tt.Collect,
Redact: false,
}
got, err := c.RunCollectorSync(nil)
got, err := c.RunCollectorSync(tt.Redactors)
req.NoError(err)
// convert to string to make differences easier to see