diff --git a/cmd/troubleshoot/cli/root.go b/cmd/troubleshoot/cli/root.go index 8c93fb82..3a61bbb9 100644 --- a/cmd/troubleshoot/cli/root.go +++ b/cmd/troubleshoot/cli/root.go @@ -43,6 +43,7 @@ from a server that can be used to assist when troubleshooting a server.`, cmd.Flags().String("collectors", "", "name of the collectors to use") cmd.Flags().String("image", "", "the full name of the collector image to use") cmd.Flags().String("pullpolicy", "", "the pull policy of the collector image") + cmd.Flags().String("redactors", "", "name of the additional redactors to use") cmd.Flags().Bool("redact", true, "enable/disable default redactions") cmd.Flags().Bool("collect-without-permissions", false, "always run troubleshoot collectors even if some require permissions that troubleshoot does not have") diff --git a/cmd/troubleshoot/cli/run.go b/cmd/troubleshoot/cli/run.go index fde67998..ad458378 100644 --- a/cmd/troubleshoot/cli/run.go +++ b/cmd/troubleshoot/cli/run.go @@ -56,6 +56,23 @@ func runTroubleshoot(v *viper.Viper, arg string) error { collector := obj.(*troubleshootv1beta1.Collector) + var additionalRedactors *troubleshootv1beta1.Redactor + if v.GetString("redactors") != "" { + redactorContent, err := loadSpec(v, v.GetString("redactors")) + if err != nil { + return errors.Wrap(err, "failed to load redactor spec") + } + obj, _, err := decode([]byte(redactorContent), nil, nil) + if err != nil { + return errors.Wrapf(err, "failed to parse redactors %s", v.GetString("redactors")) + } + var ok bool + additionalRedactors, ok = obj.(*troubleshootv1beta1.Redactor) + if !ok { + return fmt.Errorf("%s is not a troubleshootv1beta1 redactor type", v.GetString("redactors")) + } + } + s := spin.New() finishedCh := make(chan bool, 1) progressChan := make(chan interface{}, 0) // non-zero buffer can result in missed messages @@ -87,7 +104,7 @@ func runTroubleshoot(v *viper.Viper, arg string) error { close(finishedCh) }() - archivePath, err := runCollectors(v, *collector, progressChan) + archivePath, err := runCollectors(v, *collector, additionalRedactors, progressChan) if err != nil { return errors.Wrap(err, "run collectors") } @@ -193,7 +210,7 @@ func canTryInsecure(v *viper.Viper) bool { return true } -func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, progressChan chan interface{}) (string, error) { +func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, additionalRedactors *troubleshootv1beta1.Redactor, progressChan chan interface{}) (string, error) { bundlePath, err := ioutil.TempDir("", "troubleshoot") if err != nil { return "", errors.Wrap(err, "create temp dir") @@ -241,6 +258,11 @@ func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, prog return "", errors.New("insufficient permissions to run all collectors") } + globalRedactors := []*troubleshootv1beta1.Redact{} + if additionalRedactors != nil { + globalRedactors = additionalRedactors.Spec.Redactors + } + // Run preflights collectors synchronously for _, collector := range collectors { if len(collector.RBACErrors) > 0 { @@ -253,7 +275,7 @@ func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, prog progressChan <- collector.GetDisplayName() - result, err := collector.RunCollectorSync() + result, err := collector.RunCollectorSync(globalRedactors) if err != nil { progressChan <- fmt.Errorf("failed to run collector %q: %v", collector.GetDisplayName(), err) continue diff --git a/pkg/apis/troubleshoot/v1beta1/collector_shared.go b/pkg/apis/troubleshoot/v1beta1/collector_shared.go index aeaf7735..418573d3 100644 --- a/pkg/apis/troubleshoot/v1beta1/collector_shared.go +++ b/pkg/apis/troubleshoot/v1beta1/collector_shared.go @@ -9,7 +9,8 @@ import ( ) type CollectorMeta struct { - CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` + CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` + Redactors []*Redact `json:"redactors,omitempty" yaml:"redactors,omitempty"` // +optional Exclude multitype.BoolOrString `json:"exclude,omitempty" yaml:"exclude,omitempty"` } diff --git a/pkg/apis/troubleshoot/v1beta1/collector_types.go b/pkg/apis/troubleshoot/v1beta1/collector_types.go index f193b3b7..161578f2 100644 --- a/pkg/apis/troubleshoot/v1beta1/collector_types.go +++ b/pkg/apis/troubleshoot/v1beta1/collector_types.go @@ -34,6 +34,7 @@ type AfterCollection struct { type CollectorSpec struct { Collectors []*Collect `json:"collectors,omitempty" yaml:"collectors,omitempty"` AfterCollection []*AfterCollection `json:"afterCollection,omitempty" yaml:"afterCollection,omitempty"` + GlobalRedactors []*Redact `json:"globalRedactors,omitempty" yaml:"globalRedactors,omitempty"` } // CollectorStatus defines the observed state of Collector diff --git a/pkg/apis/troubleshoot/v1beta1/redact_shared.go b/pkg/apis/troubleshoot/v1beta1/redact_shared.go new file mode 100644 index 00000000..d6937b8b --- /dev/null +++ b/pkg/apis/troubleshoot/v1beta1/redact_shared.go @@ -0,0 +1,9 @@ +package v1beta1 + +type Redact struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + File string `json:"file,omitempty" yaml:"file,omitempty"` + Files []string `json:"files,omitempty" yaml:"files,omitempty"` + Values []string `json:"values,omitempty" yaml:"values,omitempty"` + Regex []string `json:"regex,omitempty" yaml:"regex,omitempty"` +} diff --git a/pkg/apis/troubleshoot/v1beta1/redact_types.go b/pkg/apis/troubleshoot/v1beta1/redact_types.go new file mode 100644 index 00000000..dad57aa1 --- /dev/null +++ b/pkg/apis/troubleshoot/v1beta1/redact_types.go @@ -0,0 +1,56 @@ +/* +Copyright 2019 Replicated, Inc.. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RedactorSpec defines the desired state of Redactor +type RedactorSpec struct { + Redactors []*Redact `json:"redacts,omitempty"` +} + +// RedactorStatus defines the observed state of Redactor +type RedactorStatus struct { +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Redactor is the Schema for the redaction API +// +k8s:openapi-gen=true +type Redactor struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RedactorSpec `json:"spec,omitempty"` + Status RedactorStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// RedactorList contains a list of Redactor +type RedactorList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Redactor `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Redactor{}, &RedactorList{}) +} diff --git a/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go index e1c81238..f6986e32 100644 --- a/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go @@ -285,7 +285,7 @@ func (in *AnalyzerStatus) DeepCopy() *AnalyzerStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterInfo) DeepCopyInto(out *ClusterInfo) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterInfo. @@ -301,7 +301,7 @@ func (in *ClusterInfo) DeepCopy() *ClusterInfo { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterResources) DeepCopyInto(out *ClusterResources) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterResources. @@ -347,17 +347,17 @@ func (in *Collect) DeepCopyInto(out *Collect) { if in.ClusterInfo != nil { in, out := &in.ClusterInfo, &out.ClusterInfo *out = new(ClusterInfo) - **out = **in + (*in).DeepCopyInto(*out) } if in.ClusterResources != nil { in, out := &in.ClusterResources, &out.ClusterResources *out = new(ClusterResources) - **out = **in + (*in).DeepCopyInto(*out) } if in.Secret != nil { in, out := &in.Secret, &out.Secret *out = new(Secret) - **out = **in + (*in).DeepCopyInto(*out) } if in.Logs != nil { in, out := &in.Logs, &out.Logs @@ -377,7 +377,7 @@ func (in *Collect) DeepCopyInto(out *Collect) { if in.Data != nil { in, out := &in.Data, &out.Data *out = new(Data) - **out = **in + (*in).DeepCopyInto(*out) } if in.Copy != nil { in, out := &in.Copy, &out.Copy @@ -392,17 +392,17 @@ func (in *Collect) DeepCopyInto(out *Collect) { if in.Postgres != nil { in, out := &in.Postgres, &out.Postgres *out = new(Database) - **out = **in + (*in).DeepCopyInto(*out) } if in.Mysql != nil { in, out := &in.Mysql, &out.Mysql *out = new(Database) - **out = **in + (*in).DeepCopyInto(*out) } if in.Redis != nil { in, out := &in.Redis, &out.Redis *out = new(Database) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -478,6 +478,17 @@ func (in *CollectorList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CollectorMeta) DeepCopyInto(out *CollectorMeta) { *out = *in + if in.Redactors != nil { + in, out := &in.Redactors, &out.Redactors + *out = make([]*Redact, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Redact) + (*in).DeepCopyInto(*out) + } + } + } out.Exclude = in.Exclude } @@ -516,6 +527,17 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { } } } + if in.GlobalRedactors != nil { + in, out := &in.GlobalRedactors, &out.GlobalRedactors + *out = make([]*Redact, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Redact) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollectorSpec. @@ -573,7 +595,7 @@ func (in *ContainerRuntime) DeepCopy() *ContainerRuntime { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Copy) DeepCopyInto(out *Copy) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) if in.Selector != nil { in, out := &in.Selector, &out.Selector *out = make([]string, len(*in)) @@ -621,7 +643,7 @@ func (in *CustomResourceDefinition) DeepCopy() *CustomResourceDefinition { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Data) DeepCopyInto(out *Data) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Data. @@ -637,7 +659,7 @@ func (in *Data) DeepCopy() *Data { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Database) DeepCopyInto(out *Database) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Database. @@ -734,7 +756,7 @@ func (in *Distribution) DeepCopy() *Distribution { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Exec) DeepCopyInto(out *Exec) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) if in.Selector != nil { in, out := &in.Selector, &out.Selector *out = make([]string, len(*in)) @@ -787,7 +809,7 @@ func (in *Get) DeepCopy() *Get { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTP) DeepCopyInto(out *HTTP) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) if in.Get != nil { in, out := &in.Get, &out.Get *out = new(Get) @@ -887,7 +909,7 @@ func (in *LogLimits) DeepCopy() *LogLimits { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Logs) DeepCopyInto(out *Logs) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) if in.Selector != nil { in, out := &in.Selector, &out.Selector *out = make([]string, len(*in)) @@ -1147,6 +1169,136 @@ func (in *Put) DeepCopy() *Put { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Redact) DeepCopyInto(out *Redact) { + *out = *in + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Values != nil { + in, out := &in.Values, &out.Values + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Regex != nil { + in, out := &in.Regex, &out.Regex + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Redact. +func (in *Redact) DeepCopy() *Redact { + if in == nil { + return nil + } + out := new(Redact) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Redactor) DeepCopyInto(out *Redactor) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Redactor. +func (in *Redactor) DeepCopy() *Redactor { + if in == nil { + return nil + } + out := new(Redactor) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Redactor) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RedactorList) DeepCopyInto(out *RedactorList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Redactor, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedactorList. +func (in *RedactorList) DeepCopy() *RedactorList { + if in == nil { + return nil + } + out := new(RedactorList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RedactorList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RedactorSpec) DeepCopyInto(out *RedactorSpec) { + *out = *in + if in.Redactors != nil { + in, out := &in.Redactors, &out.Redactors + *out = make([]*Redact, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Redact) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedactorSpec. +func (in *RedactorSpec) DeepCopy() *RedactorSpec { + if in == nil { + return nil + } + out := new(RedactorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RedactorStatus) DeepCopyInto(out *RedactorStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedactorStatus. +func (in *RedactorStatus) DeepCopy() *RedactorStatus { + if in == nil { + return nil + } + out := new(RedactorStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResultRequest) DeepCopyInto(out *ResultRequest) { *out = *in @@ -1165,7 +1317,7 @@ func (in *ResultRequest) DeepCopy() *ResultRequest { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Run) DeepCopyInto(out *Run) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) if in.Command != nil { in, out := &in.Command, &out.Command *out = make([]string, len(*in)) @@ -1191,7 +1343,7 @@ func (in *Run) DeepCopy() *Run { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Secret) DeepCopyInto(out *Secret) { *out = *in - out.CollectorMeta = in.CollectorMeta + in.CollectorMeta.DeepCopyInto(&out.CollectorMeta) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secret. diff --git a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/fake/fake_redactor.go b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/fake/fake_redactor.go new file mode 100644 index 00000000..e914082a --- /dev/null +++ b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/fake/fake_redactor.go @@ -0,0 +1,139 @@ +/* +Copyright 2019 Replicated, Inc.. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeRedactors implements RedactorInterface +type FakeRedactors struct { + Fake *FakeTroubleshootV1beta1 + ns string +} + +var redactorsResource = schema.GroupVersionResource{Group: "troubleshoot.replicated.com", Version: "v1beta1", Resource: "redactors"} + +var redactorsKind = schema.GroupVersionKind{Group: "troubleshoot.replicated.com", Version: "v1beta1", Kind: "Redactor"} + +// Get takes name of the redactor, and returns the corresponding redactor object, and an error if there is any. +func (c *FakeRedactors) Get(name string, options v1.GetOptions) (result *v1beta1.Redactor, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(redactorsResource, c.ns, name), &v1beta1.Redactor{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.Redactor), err +} + +// List takes label and field selectors, and returns the list of Redactors that match those selectors. +func (c *FakeRedactors) List(opts v1.ListOptions) (result *v1beta1.RedactorList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(redactorsResource, redactorsKind, c.ns, opts), &v1beta1.RedactorList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1beta1.RedactorList{ListMeta: obj.(*v1beta1.RedactorList).ListMeta} + for _, item := range obj.(*v1beta1.RedactorList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested redactors. +func (c *FakeRedactors) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(redactorsResource, c.ns, opts)) + +} + +// Create takes the representation of a redactor and creates it. Returns the server's representation of the redactor, and an error, if there is any. +func (c *FakeRedactors) Create(redactor *v1beta1.Redactor) (result *v1beta1.Redactor, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(redactorsResource, c.ns, redactor), &v1beta1.Redactor{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.Redactor), err +} + +// Update takes the representation of a redactor and updates it. Returns the server's representation of the redactor, and an error, if there is any. +func (c *FakeRedactors) Update(redactor *v1beta1.Redactor) (result *v1beta1.Redactor, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(redactorsResource, c.ns, redactor), &v1beta1.Redactor{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.Redactor), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeRedactors) UpdateStatus(redactor *v1beta1.Redactor) (*v1beta1.Redactor, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(redactorsResource, "status", c.ns, redactor), &v1beta1.Redactor{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.Redactor), err +} + +// Delete takes name of the redactor and deletes it. Returns an error if one occurs. +func (c *FakeRedactors) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(redactorsResource, c.ns, name), &v1beta1.Redactor{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeRedactors) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(redactorsResource, c.ns, listOptions) + + _, err := c.Fake.Invokes(action, &v1beta1.RedactorList{}) + return err +} + +// Patch applies the patch and returns the patched redactor. +func (c *FakeRedactors) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.Redactor, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(redactorsResource, c.ns, name, pt, data, subresources...), &v1beta1.Redactor{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.Redactor), err +} diff --git a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/fake/fake_troubleshoot_client.go b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/fake/fake_troubleshoot_client.go index cabe34bb..bcea4dc7 100644 --- a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/fake/fake_troubleshoot_client.go +++ b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/fake/fake_troubleshoot_client.go @@ -39,6 +39,10 @@ func (c *FakeTroubleshootV1beta1) Preflights(namespace string) v1beta1.Preflight return &FakePreflights{c, namespace} } +func (c *FakeTroubleshootV1beta1) Redactors(namespace string) v1beta1.RedactorInterface { + return &FakeRedactors{c, namespace} +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeTroubleshootV1beta1) RESTClient() rest.Interface { diff --git a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/generated_expansion.go b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/generated_expansion.go index 5080ab7d..fa367f6e 100644 --- a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/generated_expansion.go +++ b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/generated_expansion.go @@ -22,3 +22,5 @@ type AnalyzerExpansion interface{} type CollectorExpansion interface{} type PreflightExpansion interface{} + +type RedactorExpansion interface{} diff --git a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/redactor.go b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/redactor.go new file mode 100644 index 00000000..5a244e4a --- /dev/null +++ b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/redactor.go @@ -0,0 +1,190 @@ +/* +Copyright 2019 Replicated, Inc.. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "time" + + v1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + scheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// RedactorsGetter has a method to return a RedactorInterface. +// A group's client should implement this interface. +type RedactorsGetter interface { + Redactors(namespace string) RedactorInterface +} + +// RedactorInterface has methods to work with Redactor resources. +type RedactorInterface interface { + Create(*v1beta1.Redactor) (*v1beta1.Redactor, error) + Update(*v1beta1.Redactor) (*v1beta1.Redactor, error) + UpdateStatus(*v1beta1.Redactor) (*v1beta1.Redactor, error) + Delete(name string, options *v1.DeleteOptions) error + DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error + Get(name string, options v1.GetOptions) (*v1beta1.Redactor, error) + List(opts v1.ListOptions) (*v1beta1.RedactorList, error) + Watch(opts v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.Redactor, err error) + RedactorExpansion +} + +// redactors implements RedactorInterface +type redactors struct { + client rest.Interface + ns string +} + +// newRedactors returns a Redactors +func newRedactors(c *TroubleshootV1beta1Client, namespace string) *redactors { + return &redactors{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the redactor, and returns the corresponding redactor object, and an error if there is any. +func (c *redactors) Get(name string, options v1.GetOptions) (result *v1beta1.Redactor, err error) { + result = &v1beta1.Redactor{} + err = c.client.Get(). + Namespace(c.ns). + Resource("redactors"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of Redactors that match those selectors. +func (c *redactors) List(opts v1.ListOptions) (result *v1beta1.RedactorList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1beta1.RedactorList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("redactors"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested redactors. +func (c *redactors) Watch(opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("redactors"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch() +} + +// Create takes the representation of a redactor and creates it. Returns the server's representation of the redactor, and an error, if there is any. +func (c *redactors) Create(redactor *v1beta1.Redactor) (result *v1beta1.Redactor, err error) { + result = &v1beta1.Redactor{} + err = c.client.Post(). + Namespace(c.ns). + Resource("redactors"). + Body(redactor). + Do(). + Into(result) + return +} + +// Update takes the representation of a redactor and updates it. Returns the server's representation of the redactor, and an error, if there is any. +func (c *redactors) Update(redactor *v1beta1.Redactor) (result *v1beta1.Redactor, err error) { + result = &v1beta1.Redactor{} + err = c.client.Put(). + Namespace(c.ns). + Resource("redactors"). + Name(redactor.Name). + Body(redactor). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *redactors) UpdateStatus(redactor *v1beta1.Redactor) (result *v1beta1.Redactor, err error) { + result = &v1beta1.Redactor{} + err = c.client.Put(). + Namespace(c.ns). + Resource("redactors"). + Name(redactor.Name). + SubResource("status"). + Body(redactor). + Do(). + Into(result) + return +} + +// Delete takes name of the redactor and deletes it. Returns an error if one occurs. +func (c *redactors) Delete(name string, options *v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("redactors"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *redactors) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + var timeout time.Duration + if listOptions.TimeoutSeconds != nil { + timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("redactors"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Timeout(timeout). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched redactor. +func (c *redactors) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.Redactor, err error) { + result = &v1beta1.Redactor{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("redactors"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/troubleshoot_client.go b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/troubleshoot_client.go index 1c967b09..1287254f 100644 --- a/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/troubleshoot_client.go +++ b/pkg/client/troubleshootclientset/typed/troubleshoot/v1beta1/troubleshoot_client.go @@ -28,6 +28,7 @@ type TroubleshootV1beta1Interface interface { AnalyzersGetter CollectorsGetter PreflightsGetter + RedactorsGetter } // TroubleshootV1beta1Client is used to interact with features provided by the troubleshoot.replicated.com group. @@ -47,6 +48,10 @@ func (c *TroubleshootV1beta1Client) Preflights(namespace string) PreflightInterf return newPreflights(c, namespace) } +func (c *TroubleshootV1beta1Client) Redactors(namespace string) RedactorInterface { + return newRedactors(c, namespace) +} + // NewForConfig creates a new TroubleshootV1beta1Client for the given config. func NewForConfig(c *rest.Config) (*TroubleshootV1beta1Client, error) { config := *c diff --git a/pkg/collect/cluster_info.go b/pkg/collect/cluster_info.go index e87a9578..3cd61090 100644 --- a/pkg/collect/cluster_info.go +++ b/pkg/collect/cluster_info.go @@ -13,11 +13,6 @@ type ClusterVersion struct { String string `json:"string"` } -type ClusterInfoOutput struct { - ClusterVersion []byte `json:"cluster-info/cluster_version.json,omitempty"` - Errors []byte `json:"cluster-info/errors.json,omitempty"` -} - func ClusterInfo(ctx *Context) (map[string][]byte, error) { client, err := kubernetes.NewForConfig(ctx.ClientConfig) if err != nil { diff --git a/pkg/collect/cluster_resources.go b/pkg/collect/cluster_resources.go index f421f6e8..7b49f2db 100644 --- a/pkg/collect/cluster_resources.go +++ b/pkg/collect/cluster_resources.go @@ -160,13 +160,6 @@ func ClusterResources(ctx *Context) (map[string][]byte, error) { return nil, err } - if ctx.Redact { - clusterResourcesOutput, err = redactMap(clusterResourcesOutput) - if err != nil { - return nil, err - } - } - return clusterResourcesOutput, nil } diff --git a/pkg/collect/collector.go b/pkg/collect/collector.go index 96986035..4d19c885 100644 --- a/pkg/collect/collector.go +++ b/pkg/collect/collector.go @@ -40,129 +40,142 @@ func isExcluded(excludeVal multitype.BoolOrString) (bool, error) { return parsed, nil } -func (c *Collector) RunCollectorSync() (map[string][]byte, error) { +func (c *Collector) RunCollectorSync(globalRedactors []*troubleshootv1beta1.Redact) (map[string][]byte, error) { + var unRedacted map[string][]byte + var isExcludedResult bool + var err error + var redactors []*troubleshootv1beta1.Redact if c.Collect.ClusterInfo != nil { - isExcluded, err := isExcluded(c.Collect.ClusterInfo.Exclude) + isExcludedResult, err = isExcluded(c.Collect.ClusterInfo.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return ClusterInfo(c.GetContext()) - } - if c.Collect.ClusterResources != nil { - isExcluded, err := isExcluded(c.Collect.ClusterResources.Exclude) + unRedacted, err = ClusterInfo(c.GetContext()) + redactors = c.Collect.ClusterInfo.Redactors + } else if c.Collect.ClusterResources != nil { + isExcludedResult, err = isExcluded(c.Collect.ClusterResources.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return ClusterResources(c.GetContext()) - } - if c.Collect.Secret != nil { - isExcluded, err := isExcluded(c.Collect.Secret.Exclude) + unRedacted, err = ClusterResources(c.GetContext()) + redactors = c.Collect.ClusterResources.Redactors + } else if c.Collect.Secret != nil { + isExcludedResult, err = isExcluded(c.Collect.Secret.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Secret(c.GetContext(), c.Collect.Secret) - } - if c.Collect.Logs != nil { - isExcluded, err := isExcluded(c.Collect.Logs.Exclude) + unRedacted, err = Secret(c.GetContext(), c.Collect.Secret) + redactors = c.Collect.Secret.Redactors + } else if c.Collect.Logs != nil { + isExcludedResult, err = isExcluded(c.Collect.Logs.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Logs(c.GetContext(), c.Collect.Logs) - } - if c.Collect.Run != nil { - isExcluded, err := isExcluded(c.Collect.Run.Exclude) + unRedacted, err = Logs(c.GetContext(), c.Collect.Logs) + redactors = c.Collect.Logs.Redactors + } else if c.Collect.Run != nil { + isExcludedResult, err = isExcluded(c.Collect.Run.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Run(c.GetContext(), c.Collect.Run) - } - if c.Collect.Exec != nil { - isExcluded, err := isExcluded(c.Collect.Exec.Exclude) + unRedacted, err = Run(c.GetContext(), c.Collect.Run) + redactors = c.Collect.Run.Redactors + } else if c.Collect.Exec != nil { + isExcludedResult, err = isExcluded(c.Collect.Exec.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Exec(c.GetContext(), c.Collect.Exec) - } - if c.Collect.Data != nil { - isExcluded, err := isExcluded(c.Collect.Data.Exclude) + unRedacted, err = Exec(c.GetContext(), c.Collect.Exec) + redactors = c.Collect.Exec.Redactors + } else if c.Collect.Data != nil { + isExcludedResult, err = isExcluded(c.Collect.Data.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Data(c.GetContext(), c.Collect.Data) - } - if c.Collect.Copy != nil { - isExcluded, err := isExcluded(c.Collect.Copy.Exclude) + unRedacted, err = Data(c.GetContext(), c.Collect.Data) + redactors = c.Collect.Data.Redactors + } else if c.Collect.Copy != nil { + isExcludedResult, err = isExcluded(c.Collect.Copy.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Copy(c.GetContext(), c.Collect.Copy) - } - if c.Collect.HTTP != nil { - isExcluded, err := isExcluded(c.Collect.HTTP.Exclude) + unRedacted, err = Copy(c.GetContext(), c.Collect.Copy) + redactors = c.Collect.Copy.Redactors + } else if c.Collect.HTTP != nil { + isExcludedResult, err = isExcluded(c.Collect.HTTP.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return HTTP(c.GetContext(), c.Collect.HTTP) - } - if c.Collect.Postgres != nil { - isExcluded, err := isExcluded(c.Collect.Postgres.Exclude) + unRedacted, err = HTTP(c.GetContext(), c.Collect.HTTP) + redactors = c.Collect.HTTP.Redactors + } else if c.Collect.Postgres != nil { + isExcludedResult, err = isExcluded(c.Collect.Postgres.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Postgres(c.GetContext(), c.Collect.Postgres) - } - if c.Collect.Mysql != nil { - isExcluded, err := isExcluded(c.Collect.Mysql.Exclude) + unRedacted, err = Postgres(c.GetContext(), c.Collect.Postgres) + redactors = c.Collect.Postgres.Redactors + } else if c.Collect.Mysql != nil { + isExcludedResult, err = isExcluded(c.Collect.Mysql.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Mysql(c.GetContext(), c.Collect.Mysql) - } - if c.Collect.Redis != nil { - isExcluded, err := isExcluded(c.Collect.Redis.Exclude) + unRedacted, err = Mysql(c.GetContext(), c.Collect.Mysql) + redactors = c.Collect.Mysql.Redactors + } else if c.Collect.Redis != nil { + isExcludedResult, err = isExcluded(c.Collect.Redis.Exclude) if err != nil { return nil, err } - if isExcluded { + if isExcludedResult { return nil, nil } - return Redis(c.GetContext(), c.Collect.Redis) + unRedacted, err = Redis(c.GetContext(), c.Collect.Redis) + redactors = c.Collect.Redis.Redactors + } else { + return nil, errors.New("no spec found to run") } - return nil, errors.New("no spec found to run") + if err != nil { + return nil, err + } + if c.Redact { + return redactMap(unRedacted, append(redactors, globalRedactors...)) + } + return unRedacted, nil } func (c *Collector) GetDisplayName() string { diff --git a/pkg/collect/collector_test.go b/pkg/collect/collector_test.go new file mode 100644 index 00000000..2bd11815 --- /dev/null +++ b/pkg/collect/collector_test.go @@ -0,0 +1,265 @@ +package collect + +import ( + "testing" + + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + "github.com/replicatedhq/troubleshoot/pkg/multitype" + "github.com/stretchr/testify/require" + "go.undefinedlabs.com/scopeagent" +) + +func TestCollector_RunCollectorSyncNoRedact(t *testing.T) { + tests := []struct { + name string + Collect *troubleshootv1beta1.Collect + want map[string]string + }{ + { + name: "data with custom redactor", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "datacollectorname", + Redactors: []*troubleshootv1beta1.Redact{ + { + Name: "", + File: "", + Files: nil, + Values: nil, + Regex: []string{ + `abc`, + `(another)(?P.*)(here)`, + }, + }, + }, + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `abc 123 +another line here +pwd=somethinggoeshere;`, + }, + }, + want: map[string]string{ + "data/datacollectorname": ` 123 +another***HIDDEN***here +pwd=***HIDDEN***; +`, + }, + }, + { + name: "data with custom redactor at a restricted path", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "datacollectorname", + Redactors: []*troubleshootv1beta1.Redact{ + { + Name: "", + File: "data/*", + Values: nil, + Regex: []string{ + `(another)(?P.*)(here)`, + }, + }, + }, + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `abc 123 +another line here +pwd=somethinggoeshere;`, + }, + }, + want: map[string]string{ + "data/datacollectorname": `abc 123 +another***HIDDEN***here +pwd=***HIDDEN***; +`, + }, + }, + { + name: "data with custom redactor at other path", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "datacollectorname", + Redactors: []*troubleshootv1beta1.Redact{ + { + Name: "", + File: "notdata/*", + Values: nil, + Regex: []string{ + `(another)(?P.*)(here)`, + }, + }, + }, + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `abc 123 +another line here +pwd=somethinggoeshere;`, + }, + }, + want: map[string]string{ + "data/datacollectorname": `abc 123 +another line here +pwd=***HIDDEN***; +`, + }, + }, + { + name: "data with custom redactor at second path", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "datacollectorname", + Redactors: []*troubleshootv1beta1.Redact{ + { + Name: "", + Files: []string{ + "notData/*", + "data/*", + }, + Values: nil, + Regex: []string{ + `(another)(?P.*)(here)`, + }, + }, + }, + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `abc 123 +another line here +pwd=somethinggoeshere;`, + }, + }, + want: map[string]string{ + "data/datacollectorname": `abc 123 +another***HIDDEN***here +pwd=***HIDDEN***; +`, + }, + }, + { + name: "data with literal string replacer", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "data/collectorname", + Redactors: []*troubleshootv1beta1.Redact{ + { + Name: "", + Files: []string{ + "data/*/*", + }, + Values: []string{ + `abc`, + `123`, + `another`, + }, + }, + }, + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `abc 123 +another line here +pwd=somethinggoeshere;`, + }, + }, + want: map[string]string{ + "data/data/collectorname": `***HIDDEN*** ***HIDDEN*** +***HIDDEN*** line here +pwd=***HIDDEN***; +`, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scopetest := scopeagent.StartTest(t) + defer scopetest.End() + + req := require.New(t) + c := &Collector{ + Collect: tt.Collect, + Redact: true, + } + got, err := c.RunCollectorSync(nil) + req.NoError(err) + + // convert to string to make differences easier to see + toString := map[string]string{} + for k, v := range got { + toString[k] = string(v) + } + req.EqualValues(tt.want, toString) + }) + } +} + +func TestCollector_RunCollectorSync(t *testing.T) { + tests := []struct { + name string + Collect *troubleshootv1beta1.Collect + want map[string]string + }{ + { + name: "data with custom redactor - but redaction disabled", + Collect: &troubleshootv1beta1.Collect{ + Data: &troubleshootv1beta1.Data{ + CollectorMeta: troubleshootv1beta1.CollectorMeta{ + CollectorName: "datacollectorname", + Redactors: []*troubleshootv1beta1.Redact{ + { + Name: "", + File: "", + Files: nil, + Values: nil, + Regex: []string{ + `abc`, + `(another)(?P.*)(here)`, + }, + }, + }, + Exclude: multitype.BoolOrString{}, + }, + Name: "data", + Data: `abc 123 +another line here +pwd=somethinggoeshere;`, + }, + }, + want: map[string]string{ + "data/datacollectorname": `abc 123 +another line here +pwd=somethinggoeshere;`, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scopetest := scopeagent.StartTest(t) + defer scopetest.End() + + req := require.New(t) + c := &Collector{ + Collect: tt.Collect, + Redact: false, + } + got, err := c.RunCollectorSync(nil) + req.NoError(err) + + // convert to string to make differences easier to see + toString := map[string]string{} + for k, v := range got { + toString[k] = string(v) + } + req.EqualValues(tt.want, toString) + }) + } +} diff --git a/pkg/collect/copy.go b/pkg/collect/copy.go index 2632fd35..d15fde03 100644 --- a/pkg/collect/copy.go +++ b/pkg/collect/copy.go @@ -12,15 +12,13 @@ import ( "k8s.io/client-go/tools/remotecommand" ) -type CopyOutput map[string][]byte - func Copy(ctx *Context, copyCollector *troubleshootv1beta1.Copy) (map[string][]byte, error) { client, err := kubernetes.NewForConfig(ctx.ClientConfig) if err != nil { return nil, err } - copyOutput := CopyOutput{} + copyOutput := map[string][]byte{} pods, podsErrors := listPodsInSelectors(client, copyCollector.Namespace, copyCollector.Selector) if len(podsErrors) > 0 { @@ -49,13 +47,6 @@ func Copy(ctx *Context, copyCollector *troubleshootv1beta1.Copy) (map[string][]b copyOutput[filepath.Join(bundlePath, k)] = v } } - - if ctx.Redact { - copyOutput, err = copyOutput.Redact() - if err != nil { - return nil, err - } - } } return copyOutput, nil @@ -120,15 +111,6 @@ func copyFiles(ctx *Context, client *kubernetes.Clientset, pod corev1.Pod, copyC }, nil } -func (c CopyOutput) Redact() (CopyOutput, error) { - results, err := redactMap(c) - if err != nil { - return nil, err - } - - return results, nil -} - func getCopyErrosFileName(copyCollector *troubleshootv1beta1.Copy) string { if len(copyCollector.Name) > 0 { return fmt.Sprintf("%s-errors.json", copyCollector.Name) diff --git a/pkg/collect/data.go b/pkg/collect/data.go index d2c015ba..3895788f 100644 --- a/pkg/collect/data.go +++ b/pkg/collect/data.go @@ -6,11 +6,9 @@ import ( troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" ) -type DataOutput map[string][]byte - func Data(ctx *Context, dataCollector *troubleshootv1beta1.Data) (map[string][]byte, error) { bundlePath := filepath.Join(dataCollector.Name, dataCollector.CollectorName) - dataOutput := DataOutput{ + dataOutput := map[string][]byte{ bundlePath: []byte(dataCollector.Data), } diff --git a/pkg/collect/exec.go b/pkg/collect/exec.go index 1ce64fc7..43e1b750 100644 --- a/pkg/collect/exec.go +++ b/pkg/collect/exec.go @@ -14,8 +14,6 @@ import ( "k8s.io/client-go/tools/remotecommand" ) -type ExecOutput map[string][]byte - func Exec(ctx *Context, execCollector *troubleshootv1beta1.Exec) (map[string][]byte, error) { if execCollector.Timeout == "" { return execWithoutTimeout(ctx, execCollector) @@ -54,7 +52,7 @@ func execWithoutTimeout(ctx *Context, execCollector *troubleshootv1beta1.Exec) ( return nil, err } - execOutput := ExecOutput{} + execOutput := map[string][]byte{} pods, podsErrors := listPodsInSelectors(client, execCollector.Namespace, execCollector.Selector) if len(podsErrors) > 0 { @@ -86,13 +84,6 @@ func execWithoutTimeout(ctx *Context, execCollector *troubleshootv1beta1.Exec) ( continue } } - - if ctx.Redact { - execOutput, err = execOutput.Redact() - if err != nil { - return nil, err - } - } } return execOutput, nil @@ -142,15 +133,6 @@ func getExecOutputs(ctx *Context, client *kubernetes.Clientset, pod corev1.Pod, return stdout.Bytes(), stderr.Bytes(), nil } -func (r ExecOutput) Redact() (ExecOutput, error) { - results, err := redactMap(r) - if err != nil { - return nil, err - } - - return results, nil -} - func getExecErrosFileName(execCollector *troubleshootv1beta1.Exec) string { if len(execCollector.Name) > 0 { return fmt.Sprintf("%s-errors.json", execCollector.Name) diff --git a/pkg/collect/http.go b/pkg/collect/http.go index a20c72b6..eea244a6 100644 --- a/pkg/collect/http.go +++ b/pkg/collect/http.go @@ -10,11 +10,8 @@ import ( "strings" troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" - "github.com/replicatedhq/troubleshoot/pkg/redact" ) -type HTTPOutput map[string][]byte - type httpResponse struct { Status int `json:"status"` Body string `json:"body"` @@ -48,7 +45,7 @@ func HTTP(ctx *Context, httpCollector *troubleshootv1beta1.HTTP) (map[string][]b if httpCollector.CollectorName != "" { fileName = httpCollector.CollectorName + ".json" } - httpOutput := HTTPOutput{ + httpOutput := map[string][]byte{ filepath.Join(httpCollector.Name, fileName): output, } @@ -135,12 +132,5 @@ func responseToOutput(response *http.Response, err error, doRedact bool) ([]byte return nil, err } - if doRedact { - b, err = redact.Redact(b) - if err != nil { - return nil, err - } - } - return b, nil } diff --git a/pkg/collect/logs.go b/pkg/collect/logs.go index c448b274..2c8ff159 100644 --- a/pkg/collect/logs.go +++ b/pkg/collect/logs.go @@ -15,15 +15,13 @@ import ( "k8s.io/client-go/kubernetes" ) -type LogsOutput map[string][]byte - func Logs(ctx *Context, logsCollector *troubleshootv1beta1.Logs) (map[string][]byte, error) { client, err := kubernetes.NewForConfig(ctx.ClientConfig) if err != nil { return nil, err } - logsOutput := LogsOutput{} + logsOutput := map[string][]byte{} pods, podsErrors := listPodsInSelectors(client, logsCollector.Namespace, logsCollector.Selector) if len(podsErrors) > 0 { @@ -83,13 +81,6 @@ func Logs(ctx *Context, logsCollector *troubleshootv1beta1.Logs) (map[string][]b } } } - - if ctx.Redact { - logsOutput, err = logsOutput.Redact() - if err != nil { - return nil, err - } - } } return logsOutput, nil @@ -176,15 +167,6 @@ func getPodLogs(client *kubernetes.Clientset, pod corev1.Pod, name, container st return result, nil } -func (l LogsOutput) Redact() (LogsOutput, error) { - podLogs, err := redactMap(l) - if err != nil { - return nil, err - } - - return podLogs, nil -} - func getLogsErrorsFileName(logsCollector *troubleshootv1beta1.Logs) string { if len(logsCollector.Name) > 0 { return fmt.Sprintf("%s/errors.json", logsCollector.Name) diff --git a/pkg/collect/mysql.go b/pkg/collect/mysql.go index a6401cd3..ee79e8c2 100644 --- a/pkg/collect/mysql.go +++ b/pkg/collect/mysql.go @@ -10,8 +10,6 @@ import ( troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" ) -type MysqlOutput map[string][]byte - func Mysql(ctx *Context, databaseCollector *troubleshootv1beta1.Database) (map[string][]byte, error) { databaseConnection := DatabaseConnection{} diff --git a/pkg/collect/postgres.go b/pkg/collect/postgres.go index 93f593b0..1967ca36 100644 --- a/pkg/collect/postgres.go +++ b/pkg/collect/postgres.go @@ -10,8 +10,6 @@ import ( troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" ) -type PostgresOutput map[string][]byte - func Postgres(ctx *Context, databaseCollector *troubleshootv1beta1.Database) (map[string][]byte, error) { databaseConnection := DatabaseConnection{} diff --git a/pkg/collect/redact.go b/pkg/collect/redact.go index 0c519704..d1073d10 100644 --- a/pkg/collect/redact.go +++ b/pkg/collect/redact.go @@ -1,14 +1,15 @@ package collect import ( + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" "github.com/replicatedhq/troubleshoot/pkg/redact" ) -func redactMap(input map[string][]byte) (map[string][]byte, error) { +func redactMap(input map[string][]byte, additionalRedactors []*troubleshootv1beta1.Redact) (map[string][]byte, error) { result := make(map[string][]byte) for k, v := range input { if v != nil { - redacted, err := redact.Redact(v) + redacted, err := redact.Redact(v, k, additionalRedactors) if err != nil { return nil, err } diff --git a/pkg/collect/redis.go b/pkg/collect/redis.go index 7776489b..20e4583e 100644 --- a/pkg/collect/redis.go +++ b/pkg/collect/redis.go @@ -10,8 +10,6 @@ import ( troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" ) -type RedisOutput map[string][]byte - func Redis(ctx *Context, databaseCollector *troubleshootv1beta1.Database) (map[string][]byte, error) { databaseConnection := DatabaseConnection{} diff --git a/pkg/collect/run.go b/pkg/collect/run.go index c09668d7..5b4a7c63 100644 --- a/pkg/collect/run.go +++ b/pkg/collect/run.go @@ -11,8 +11,6 @@ import ( "k8s.io/client-go/kubernetes" ) -type RunOutput map[string][]byte - func Run(ctx *Context, runCollector *troubleshootv1beta1.Run) (map[string][]byte, error) { client, err := kubernetes.NewForConfig(ctx.ClientConfig) if err != nil { @@ -79,7 +77,7 @@ func runWithoutTimeout(ctx *Context, pod *corev1.Pod, runCollector *troubleshoot time.Sleep(time.Second * 1) } - runOutput := RunOutput{} + runOutput := map[string][]byte{} limits := troubleshootv1beta1.LogLimits{ MaxLines: 10000, @@ -93,13 +91,6 @@ func runWithoutTimeout(ctx *Context, pod *corev1.Pod, runCollector *troubleshoot runOutput[k] = v } - if ctx.Redact { - runOutput, err = runOutput.Redact() - if err != nil { - return nil, errors.Wrap(err, "failed to redact pod logs") - } - } - return runOutput, nil } @@ -150,12 +141,3 @@ func runPod(client *kubernetes.Clientset, runCollector *troubleshootv1beta1.Run, return created, nil } - -func (r RunOutput) Redact() (RunOutput, error) { - podLogs, err := redactMap(r) - if err != nil { - return nil, err - } - - return podLogs, nil -} diff --git a/pkg/collect/secret.go b/pkg/collect/secret.go index 9c7ab4d0..4a6a60a9 100644 --- a/pkg/collect/secret.go +++ b/pkg/collect/secret.go @@ -20,11 +20,6 @@ type FoundSecret struct { Value string `json:"value,omitempty"` } -type SecretOutput struct { - FoundSecret map[string][]byte `json:"secrets/,omitempty"` - Errors map[string][]byte `json:"secrets-errors/,omitempty"` -} - func Secret(ctx *Context, secretCollector *troubleshootv1beta1.Secret) (map[string][]byte, error) { client, err := kubernetes.NewForConfig(ctx.ClientConfig) if err != nil { @@ -45,13 +40,6 @@ func Secret(ctx *Context, secretCollector *troubleshootv1beta1.Secret) (map[stri secretOutput[path.Join("secrets", filePath)] = encoded } - if ctx.Redact { - secretOutput, err = redactMap(secretOutput) - if err != nil { - return nil, err - } - } - return secretOutput, nil } @@ -104,15 +92,3 @@ func secret(client *kubernetes.Clientset, secretCollector *troubleshootv1beta1.S return path, b, nil } - -func (s *SecretOutput) Redact() (*SecretOutput, error) { - foundSecret, err := redactMap(s.FoundSecret) - if err != nil { - return nil, err - } - - return &SecretOutput{ - FoundSecret: foundSecret, - Errors: s.Errors, - }, nil -} diff --git a/pkg/preflight/collect.go b/pkg/preflight/collect.go index 6318ebad..0b9c557f 100644 --- a/pkg/preflight/collect.go +++ b/pkg/preflight/collect.go @@ -76,7 +76,7 @@ func Collect(opts CollectOpts, p *troubleshootv1beta1.Preflight) (CollectResult, } } - result, err := collector.RunCollectorSync() + result, err := collector.RunCollectorSync(nil) if err != nil { opts.ProgressChan <- errors.Errorf("failed to run collector %s: %v\n", collector.GetDisplayName(), err) continue diff --git a/pkg/redact/literal.go b/pkg/redact/literal.go new file mode 100644 index 00000000..ba270a3d --- /dev/null +++ b/pkg/redact/literal.go @@ -0,0 +1,47 @@ +package redact + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +type literalRedactor struct { + matchString string +} + +func literalString(matchString string) Redactor { + return literalRedactor{matchString: matchString} +} + +func (r literalRedactor) Redact(input io.Reader) io.Reader { + reader, writer := io.Pipe() + + go func() { + var err error + defer func() { + if err == io.EOF { + writer.Close() + } else { + writer.CloseWithError(err) + } + }() + + reader := bufio.NewReader(input) + for { + var line string + line, err = readLine(reader) + if err != nil { + return + } + + // io.WriteString would be nicer, but scanner strips new lines + fmt.Fprintf(writer, "%s\n", strings.ReplaceAll(line, r.matchString, MASK_TEXT)) + if err != nil { + return + } + } + }() + return reader +} diff --git a/pkg/redact/redact.go b/pkg/redact/redact.go index a0ab685c..641dd762 100644 --- a/pkg/redact/redact.go +++ b/pkg/redact/redact.go @@ -6,7 +6,11 @@ import ( "fmt" "io" "io/ioutil" + "path/filepath" "regexp" + + "github.com/pkg/errors" + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" ) const ( @@ -17,12 +21,18 @@ type Redactor interface { Redact(input io.Reader) io.Reader } -func Redact(input []byte) ([]byte, error) { - redactors, err := GetRedactors() +func Redact(input []byte, path string, additionalRedactors []*troubleshootv1beta1.Redact) ([]byte, error) { + redactors, err := getRedactors() if err != nil { return nil, err } + builtRedactors, err := buildAdditionalRedactors(path, additionalRedactors) + if err != nil { + return nil, errors.Wrap(err, "build custom redactors") + } + redactors = append(redactors, builtRedactors...) + nextReader := io.Reader(bytes.NewReader(input)) for _, r := range redactors { nextReader = r.Redact(nextReader) @@ -36,7 +46,66 @@ func Redact(input []byte) ([]byte, error) { return redacted, nil } -func GetRedactors() ([]Redactor, error) { +func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact) ([]Redactor, error) { + additionalRedactors := []Redactor{} + for _, redact := range redacts { + if redact == nil { + continue + } + + // check if redact matches path + matches, err := redactMatchesPath(path, redact) + if err != nil { + return nil, err + } + if !matches { + continue + } + + for _, re := range redact.Regex { + r, err := NewSingleLineRedactor(re, MASK_TEXT) + if err != nil { + return nil, err // maybe skip broken ones? + } + additionalRedactors = append(additionalRedactors, r) + } + + for _, literal := range redact.Values { + additionalRedactors = append(additionalRedactors, literalString(literal)) + } + } + return additionalRedactors, nil +} + +func redactMatchesPath(path string, redact *troubleshootv1beta1.Redact) (bool, error) { + if redact.File == "" && len(redact.Files) == 0 { + return true, nil + } + + if redact.File != "" { + matches, err := filepath.Match(redact.File, path) + if err != nil { + return false, errors.Wrapf(err, "invalid file match string %q", redact.File) + } + if matches { + return true, nil + } + } + + for i, fileGlobString := range redact.Files { + matches, err := filepath.Match(fileGlobString, path) + if err != nil { + return false, errors.Wrapf(err, "invalid file match string %d %q", i, fileGlobString) + } + if matches { + return true, nil + } + } + + return false, nil +} + +func getRedactors() ([]Redactor, error) { // TODO: Make this configurable // (?i) makes it case insensitive diff --git a/pkg/redact/redact_test.go b/pkg/redact/redact_test.go index 8e28899e..62b1a31b 100644 --- a/pkg/redact/redact_test.go +++ b/pkg/redact/redact_test.go @@ -6,7 +6,8 @@ import ( "strings" "testing" - "github.com/stretchr/testify/assert" + troubleshootv1beta1 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta1" + "github.com/stretchr/testify/require" "go.undefinedlabs.com/scopeagent" ) @@ -1622,8 +1623,9 @@ func Test_Redactors(t *testing.T) { t.Run("test default redactors", func(t *testing.T) { scopetest := scopeagent.StartTest(t) defer scopetest.End() - redactors, err := GetRedactors() - assert.NoError(t, err) + req := require.New(t) + redactors, err := getRedactors() + req.NoError(err) nextReader := io.Reader(strings.NewReader(original)) for _, r := range redactors { @@ -1631,8 +1633,101 @@ func Test_Redactors(t *testing.T) { } redacted, err := ioutil.ReadAll(nextReader) - assert.NoError(t, err) + req.NoError(err) - assert.JSONEq(t, expected, string(redacted)) + req.JSONEq(expected, string(redacted)) }) } + +func Test_redactMatchesPath(t *testing.T) { + type args struct { + path string + redact *troubleshootv1beta1.Redact + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "literal path", + args: args{ + path: "/my/test/path", + redact: &troubleshootv1beta1.Redact{ + File: "/my/test/path", + Files: nil, + }, + }, + want: true, + }, + { + name: "no path", + args: args{ + path: "/my/test/path", + redact: &troubleshootv1beta1.Redact{ + File: "", + Files: nil, + }, + }, + want: true, + }, + { + name: "wrong literal path", + args: args{ + path: "/my/test/path", + redact: &troubleshootv1beta1.Redact{ + File: "/my/test/path/two", + Files: nil, + }, + }, + want: false, + }, + { + name: "path with glob", + args: args{ + path: "/my/test/path/two", + redact: &troubleshootv1beta1.Redact{ + File: "/my/test/path/*", + Files: nil, + }, + }, + want: true, + }, + { + name: "path with glob in middle", + args: args{ + path: "/my/test/path/two", + redact: &troubleshootv1beta1.Redact{ + File: "/my/test/*/*", + Files: nil, + }, + }, + want: true, + }, + { + name: "multiple paths", + args: args{ + path: "/my/test/path/two", + redact: &troubleshootv1beta1.Redact{ + File: "", + Files: []string{ + "/not/the/path", + "/my/test/*/*", + }, + }, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scopetest := scopeagent.StartTest(t) + defer scopetest.End() + req := require.New(t) + + got, err := redactMatchesPath(tt.args.path, tt.args.redact) + req.NoError(err) + req.Equal(tt.want, got) + }) + } +} diff --git a/pkg/redact/single_line_test.go b/pkg/redact/single_line_test.go new file mode 100644 index 00000000..f67c0f43 --- /dev/null +++ b/pkg/redact/single_line_test.go @@ -0,0 +1,53 @@ +package redact + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/require" + "go.undefinedlabs.com/scopeagent" +) + +func TestNewSingleLineRedactor(t *testing.T) { + tests := []struct { + name string + re string + inputString string + wantString string + }{ + { + name: "copied from default redactors", + re: `(?i)(Pwd *= *)(?P[^\;]+)(;)`, + inputString: `pwd = abcdef;`, + wantString: "pwd = ***HIDDEN***;\n", + }, + { + name: "no leading matching group", // this is not the ideal behavior - why are we dropping ungrouped match components? + re: `(?i)Pwd *= *(?P[^\;]+)(;)`, + inputString: `pwd = abcdef;`, + wantString: "***HIDDEN***;\n", + }, + { + name: "multiple matching literals", + re: `(?i)(Pwd *= *)(?P[^\;]+)(;)`, + inputString: `pwd = abcdef;abcdef`, + wantString: "pwd = ***HIDDEN***;abcdef\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scopetest := scopeagent.StartTest(t) + defer scopetest.End() + + req := require.New(t) + reRunner, err := NewSingleLineRedactor(tt.re, MASK_TEXT) + req.NoError(err) + + outReader := reRunner.Redact(bytes.NewReader([]byte(tt.inputString))) + gotBytes, err := ioutil.ReadAll(outReader) + req.NoError(err) + req.Equal(tt.wantString, string(gotBytes)) + }) + } +} diff --git a/sample-redactors.yaml b/sample-redactors.yaml new file mode 100644 index 00000000..018b23de --- /dev/null +++ b/sample-redactors.yaml @@ -0,0 +1,13 @@ +apiVersion: troubleshoot.replicated.com/v1beta1 +kind: Redactor +metadata: + name: my-application-name +spec: + redacts: + - name: replace password # names are not used internally, but are useful for recordkeeping + file: data/my-password-dump # this targets a single file + values: + - abc123 # this is a very good password, and I don't want it to be exposed + - name: all files # as no file is specified, this redactor will run against all files + regex: + - (another)(?P.*)(here) # this will replace anything between the strings `another` and `here` with `***HIDDEN***` diff --git a/sample-troubleshoot.yaml b/sample-troubleshoot.yaml index a96a2ffd..8ac4aef8 100644 --- a/sample-troubleshoot.yaml +++ b/sample-troubleshoot.yaml @@ -12,3 +12,12 @@ spec: name: healthz get: url: http://api:3000/healthz + - data: + collectorName: my-password-dump + name: data + data: | + my super secret password is abc123 + another redaction will go here + redactors: + - values: + - secret # this will replace the string 'secret' with '***HIDDEN***' in the files produced by this collector