Merge pull request #3571 from weaveworks/implement-report-censorship

Conditional report censoring

app/api_report.go

@@ -13,12 +13,13 @@ import (
 // Raw report handler
 func makeRawReportHandler(rep Reporter) CtxHandlerFunc {
 	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-		report, err := rep.Report(ctx, time.Now())
+		rawReport, err := rep.Report(ctx, time.Now())
 		if err != nil {
 			respondWith(w, http.StatusInternalServerError, err)
 			return
 		}
-		respondWith(w, http.StatusOK, report)
+		censorCfg := report.GetCensorConfigFromRequest(r)
+		respondWith(w, http.StatusOK, report.CensorRawReport(rawReport, censorCfg))
 	}
 }
 

app/api_topology.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"context"
+
 	"github.com/gorilla/mux"
 	log "github.com/sirupsen/logrus"
 
@@ -41,14 +42,17 @@ type rendererHandler func(context.Context, render.Renderer, render.Transformer,
 
 // Full topology.
 func handleTopology(ctx context.Context, renderer render.Renderer, transformer render.Transformer, rc detailed.RenderContext, w http.ResponseWriter, r *http.Request) {
+	censorCfg := report.GetCensorConfigFromRequest(r)
+	nodeSummaries := detailed.Summaries(ctx, rc, render.Render(ctx, rc.Report, renderer, transformer).Nodes)
 	respondWith(w, http.StatusOK, APITopology{
-		Nodes: detailed.Summaries(ctx, rc, render.Render(ctx, rc.Report, renderer, transformer).Nodes),
+		Nodes: detailed.CensorNodeSummaries(nodeSummaries, censorCfg),
 	})
 }
 
 // Individual nodes.
 func handleNode(ctx context.Context, renderer render.Renderer, transformer render.Transformer, rc detailed.RenderContext, w http.ResponseWriter, r *http.Request) {
 	var (
+		censorCfg  = report.GetCensorConfigFromRequest(r)
 		vars       = mux.Vars(r)
 		topologyID = vars["topology"]
 		nodeID     = vars["id"]
@@ -71,7 +75,8 @@ func handleNode(ctx context.Context, renderer render.Renderer, transformer rende
 		nodes.Nodes[nodeID] = node
 		nodes.Filtered--
 	}
-	respondWith(w, http.StatusOK, APINode{Node: detailed.MakeNode(topologyID, rc, nodes.Nodes, node)})
+	rawNode := detailed.MakeNode(topologyID, rc, nodes.Nodes, node)
+	respondWith(w, http.StatusOK, APINode{Node: detailed.CensorNode(rawNode, censorCfg)})
 }
 
 // Websocket for the full topology.
@@ -120,6 +125,7 @@ func handleWebsocket(
 		wait             = make(chan struct{}, 1)
 		topologyID       = mux.Vars(r)["topology"]
 		startReportingAt = deserializeTimestamp(r.Form.Get("timestamp"))
+		censorCfg        = report.GetCensorConfigFromRequest(r)
 		channelOpenedAt  = time.Now()
 	)
 
@@ -145,7 +151,15 @@ func handleWebsocket(
 			log.Errorf("Error generating report: %v", err)
 			return
 		}
-		newTopo := detailed.Summaries(ctx, RenderContextForReporter(rep, re), render.Render(ctx, re, renderer, filter).Nodes)
+
+		newTopo := detailed.CensorNodeSummaries(
+			detailed.Summaries(
+				ctx,
+				RenderContextForReporter(rep, re),
+				render.Render(ctx, re, renderer, filter).Nodes,
+			),
+			censorCfg,
+		)
 		diff := detailed.TopoDiff(previousTopo, newTopo)
 		previousTopo = newTopo
 

probe/docker/container.go

@@ -53,7 +53,7 @@ const (
 	CPUSystemCPUUsage    = "docker_cpu_system_cpu_usage"
 
 	LabelPrefix = "docker_label_"
-	EnvPrefix   = "docker_env_"
+	EnvPrefix   = report.DockerEnvPrefix
 )
 
 // These 'constants' are used for node states.

probe/process/reporter.go

@@ -2,7 +2,6 @@ package process
 
 import (
 	"strconv"
-	"strings"
 
 	"github.com/weaveworks/common/mtime"
 	"github.com/weaveworks/scope/report"
@@ -93,7 +92,7 @@ func (r *Reporter) processTopology() (report.Topology, error) {
 
 		if p.Cmdline != "" {
 			if r.noCommandLineArguments {
-				node = node.WithLatest(Cmdline, now, strings.Split(p.Cmdline, " ")[0])
+				node = node.WithLatest(Cmdline, now, report.StripCommandArgs(p.Cmdline))
 			} else {
 				node = node.WithLatest(Cmdline, now, p.Cmdline)
 			}

render/detailed/censor.go

@@ -0,0 +1,48 @@
+package detailed
+
+import (
+	"github.com/weaveworks/scope/report"
+)
+
+func censorNodeSummary(s NodeSummary, cfg report.CensorConfig) NodeSummary {
+	if cfg.HideCommandLineArguments && s.Metadata != nil {
+		// Iterate through all the metadata rows and strip the
+		// arguments from all the values containing a command
+		// (while making sure everything is done in a non-mutable way).
+		metadata := []report.MetadataRow{}
+		for _, row := range s.Metadata {
+			if report.IsCommandEntry(row.ID) {
+				row.Value = report.StripCommandArgs(row.Value)
+			}
+			metadata = append(metadata, row)
+		}
+		s.Metadata = metadata
+	}
+	if cfg.HideEnvironmentVariables && s.Tables != nil {
+		// Copy across all the tables except the environment
+		// variable ones (ensuring the operation is non-mutable).
+		tables := []report.Table{}
+		for _, table := range s.Tables {
+			if !report.IsEnvironmentVarsEntry(table.ID) {
+				tables = append(tables, table)
+			}
+		}
+		s.Tables = tables
+	}
+	return s
+}
+
+// CensorNode removes any sensitive data from a node.
+func CensorNode(node Node, cfg report.CensorConfig) Node {
+	node.NodeSummary = censorNodeSummary(node.NodeSummary, cfg)
+	return node
+}
+
+// CensorNodeSummaries removes any sensitive data from a list of node summaries.
+func CensorNodeSummaries(summaries NodeSummaries, cfg report.CensorConfig) NodeSummaries {
+	censored := NodeSummaries{}
+	for key := range summaries {
+		censored[key] = censorNodeSummary(summaries[key], cfg)
+	}
+	return censored
+}

render/detailed/censor_test.go

@@ -0,0 +1,228 @@
+package detailed_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/weaveworks/common/test"
+	"github.com/weaveworks/scope/render/detailed"
+	"github.com/weaveworks/scope/report"
+)
+
+func TestCensorNode(t *testing.T) {
+	node := detailed.Node{
+		NodeSummary: detailed.NodeSummary{
+			Metadata: []report.MetadataRow{
+				{ID: "cmdline", Label: "Command", Value: "prog -a --b=c"},
+			},
+			Tables: []report.Table{
+				{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+				{ID: "docker_env_", Rows: []report.Row{{ID: "env_var"}}},
+			},
+		},
+	}
+
+	for _, c := range []struct {
+		label      string
+		have, want detailed.Node
+	}{
+		{
+			label: "no censoring",
+			have: detailed.CensorNode(node, report.CensorConfig{
+				HideCommandLineArguments: false,
+				HideEnvironmentVariables: false,
+			}),
+			want: detailed.Node{
+				NodeSummary: detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog -a --b=c"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+						{ID: "docker_env_", Rows: []report.Row{{ID: "env_var"}}},
+					},
+				},
+			},
+		},
+		{
+			label: "censor only command line args",
+			have: detailed.CensorNode(node, report.CensorConfig{
+				HideCommandLineArguments: true,
+				HideEnvironmentVariables: false,
+			}),
+			want: detailed.Node{
+				NodeSummary: detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+						{ID: "docker_env_", Rows: []report.Row{{ID: "env_var"}}},
+					},
+				},
+			},
+		},
+		{
+			label: "censor only env variables",
+			have: detailed.CensorNode(node, report.CensorConfig{
+				HideCommandLineArguments: false,
+				HideEnvironmentVariables: true,
+			}),
+			want: detailed.Node{
+				NodeSummary: detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog -a --b=c"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+					},
+				},
+			},
+		},
+		{
+			label: "censor both command line args and env vars",
+			have: detailed.CensorNode(node, report.CensorConfig{
+				HideCommandLineArguments: true,
+				HideEnvironmentVariables: true,
+			}),
+			want: detailed.Node{
+				NodeSummary: detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+					},
+				},
+			},
+		},
+	} {
+		if !reflect.DeepEqual(c.want, c.have) {
+			t.Errorf("%s - %s", c.label, test.Diff(c.want, c.have))
+		}
+	}
+}
+
+func TestCensorNodeSummaries(t *testing.T) {
+	summaries := detailed.NodeSummaries{
+		"a": detailed.NodeSummary{
+			Metadata: []report.MetadataRow{
+				{ID: "blublu", Label: "blabla", Value: "blu blu"},
+				{ID: "docker_container_command", Label: "Command", Value: "scope --token=blibli"},
+			},
+		},
+		"b": detailed.NodeSummary{
+			Metadata: []report.MetadataRow{
+				{ID: "cmdline", Label: "Command", Value: "prog -a --b=c"},
+			},
+			Tables: []report.Table{
+				{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+				{ID: "docker_env_", Rows: []report.Row{{ID: "env_var"}}},
+			},
+		},
+	}
+
+	for _, c := range []struct {
+		label      string
+		have, want detailed.NodeSummaries
+	}{
+		{
+			label: "no censoring",
+			have: detailed.CensorNodeSummaries(summaries, report.CensorConfig{
+				HideCommandLineArguments: false,
+				HideEnvironmentVariables: false,
+			}),
+			want: detailed.NodeSummaries{
+				"a": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "blublu", Label: "blabla", Value: "blu blu"},
+						{ID: "docker_container_command", Label: "Command", Value: "scope --token=blibli"},
+					},
+				},
+				"b": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog -a --b=c"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+						{ID: "docker_env_", Rows: []report.Row{{ID: "env_var"}}},
+					},
+				},
+			},
+		},
+		{
+			label: "censor only command line args",
+			have: detailed.CensorNodeSummaries(summaries, report.CensorConfig{
+				HideCommandLineArguments: true,
+				HideEnvironmentVariables: false,
+			}),
+			want: detailed.NodeSummaries{
+				"a": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "blublu", Label: "blabla", Value: "blu blu"},
+						{ID: "docker_container_command", Label: "Command", Value: "scope"},
+					},
+				},
+				"b": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+						{ID: "docker_env_", Rows: []report.Row{{ID: "env_var"}}},
+					},
+				},
+			},
+		},
+		{
+			label: "censor only env variables",
+			have: detailed.CensorNodeSummaries(summaries, report.CensorConfig{
+				HideCommandLineArguments: false,
+				HideEnvironmentVariables: true,
+			}),
+			want: detailed.NodeSummaries{
+				"a": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "blublu", Label: "blabla", Value: "blu blu"},
+						{ID: "docker_container_command", Label: "Command", Value: "scope --token=blibli"},
+					},
+				},
+				"b": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog -a --b=c"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+					},
+				},
+			},
+		},
+		{
+			label: "censor both command line args and env vars",
+			have: detailed.CensorNodeSummaries(summaries, report.CensorConfig{
+				HideCommandLineArguments: true,
+				HideEnvironmentVariables: true,
+			}),
+			want: detailed.NodeSummaries{
+				"a": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "blublu", Label: "blabla", Value: "blu blu"},
+						{ID: "docker_container_command", Label: "Command", Value: "scope"},
+					},
+				},
+				"b": detailed.NodeSummary{
+					Metadata: []report.MetadataRow{
+						{ID: "cmdline", Label: "Command", Value: "prog"},
+					},
+					Tables: []report.Table{
+						{ID: "blibli", Rows: []report.Row{{ID: "bli"}}},
+					},
+				},
+			},
+		},
+	} {
+		if !reflect.DeepEqual(c.want, c.have) {
+			t.Errorf("%s - %s", c.label, test.Diff(c.want, c.have))
+		}
+	}
+}

report/censor.go

@@ -0,0 +1,68 @@
+package report
+
+import (
+	"net/http"
+	"strings"
+)
+
+// CensorConfig describes how probe reports should
+// be censored when rendered through the API.
+type CensorConfig struct {
+	HideCommandLineArguments bool
+	HideEnvironmentVariables bool
+}
+
+// GetCensorConfigFromRequest extracts censor config from request query params.
+func GetCensorConfigFromRequest(req *http.Request) CensorConfig {
+	return CensorConfig{
+		HideCommandLineArguments: req.URL.Query().Get("hideCommandLineArguments") == "true",
+		HideEnvironmentVariables: req.URL.Query().Get("hideEnvironmentVariables") == "true",
+	}
+}
+
+// IsCommandEntry returns true iff the entry comes from a command line
+// that might need to be conditionally censored.
+func IsCommandEntry(key string) bool {
+	return key == Cmdline || key == DockerContainerCommand
+}
+
+// IsEnvironmentVarsEntry returns true if the entry might expose some
+// environment variables data might need to be conditionally censored.
+func IsEnvironmentVarsEntry(key string) bool {
+	return strings.HasPrefix(key, DockerEnvPrefix)
+}
+
+// StripCommandArgs removes all the arguments from the command
+func StripCommandArgs(command string) string {
+	return strings.Split(command, " ")[0]
+}
+
+// CensorRawReport removes any sensitive data from the raw report based on the request query params.
+func CensorRawReport(rawReport Report, cfg CensorConfig) Report {
+	// Create a copy of the report first to make sure the operation is immutable.
+	censoredReport := rawReport.Copy()
+	censoredReport.ID = rawReport.ID
+
+	censoredReport.WalkTopologies(func(t *Topology) {
+		for nodeID, node := range t.Nodes {
+			if node.Latest != nil {
+				latest := make(StringLatestMap, 0, cap(node.Latest))
+				for _, entry := range node.Latest {
+					// If environment variables are to be hidden, omit passing them to the final report.
+					if cfg.HideEnvironmentVariables && IsEnvironmentVarsEntry(entry.key) {
+						continue
+					}
+					// If command line arguments are to be hidden, strip them away.
+					if cfg.HideCommandLineArguments && IsCommandEntry(entry.key) {
+						entry.Value = StripCommandArgs(entry.Value)
+					}
+					// Pass the latest entry to the final report.
+					latest = append(latest, entry)
+				}
+				node.Latest = latest
+				t.Nodes[nodeID] = node
+			}
+		}
+	})
+	return censoredReport
+}

report/censor_test.go

@@ -0,0 +1,159 @@
+package report_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/weaveworks/common/mtime"
+	"github.com/weaveworks/common/test"
+	"github.com/weaveworks/scope/report"
+	"github.com/weaveworks/scope/test/reflect"
+)
+
+func TestCensorRawReport(t *testing.T) {
+	mtime.NowForce(time.Now())
+	defer mtime.NowReset()
+
+	r := report.Report{
+		Container: report.Topology{
+			Nodes: report.Nodes{
+				"a": report.MakeNodeWith("a", map[string]string{
+					"docker_container_command": "prog -a --b=c",
+					"blublu":                   "blu blu",
+					"docker_env_":              "env_var",
+				}),
+			},
+		},
+		Process: report.Topology{
+			Nodes: report.Nodes{
+				"b": report.MakeNodeWith("b", map[string]string{
+					"cmdline": "scope --token=blibli",
+					"blibli":  "bli bli",
+				}),
+				"c": report.MakeNodeWith("c", map[string]string{
+					"docker_env_": "var",
+				}),
+			},
+		},
+	}
+
+	for _, c := range []struct {
+		label      string
+		have, want report.Report
+	}{
+		{
+			label: "no censoring",
+			have: report.CensorRawReport(r, report.CensorConfig{
+				HideCommandLineArguments: false,
+				HideEnvironmentVariables: false,
+			}),
+			want: report.Report{
+				Container: report.Topology{
+					Nodes: report.Nodes{
+						"a": report.MakeNodeWith("a", map[string]string{
+							"docker_container_command": "prog -a --b=c",
+							"blublu":                   "blu blu",
+							"docker_env_":              "env_var",
+						}),
+					},
+				},
+				Process: report.Topology{
+					Nodes: report.Nodes{
+						"b": report.MakeNodeWith("b", map[string]string{
+							"cmdline": "scope --token=blibli",
+							"blibli":  "bli bli",
+						}),
+						"c": report.MakeNodeWith("c", map[string]string{
+							"docker_env_": "var",
+						}),
+					},
+				},
+			},
+		},
+		{
+			label: "censor only command line args",
+			have: report.CensorRawReport(r, report.CensorConfig{
+				HideCommandLineArguments: true,
+				HideEnvironmentVariables: false,
+			}),
+			want: report.Report{
+				Container: report.Topology{
+					Nodes: report.Nodes{
+						"a": report.MakeNodeWith("a", map[string]string{
+							"docker_container_command": "prog",
+							"blublu":                   "blu blu",
+							"docker_env_":              "env_var",
+						}),
+					},
+				},
+				Process: report.Topology{
+					Nodes: report.Nodes{
+						"b": report.MakeNodeWith("b", map[string]string{
+							"cmdline": "scope",
+							"blibli":  "bli bli",
+						}),
+						"c": report.MakeNodeWith("c", map[string]string{
+							"docker_env_": "var",
+						}),
+					},
+				},
+			},
+		},
+		{
+			label: "censor only env variables",
+			have: report.CensorRawReport(r, report.CensorConfig{
+				HideCommandLineArguments: false,
+				HideEnvironmentVariables: true,
+			}),
+			want: report.Report{
+				Container: report.Topology{
+					Nodes: report.Nodes{
+						"a": report.MakeNodeWith("a", map[string]string{
+							"docker_container_command": "prog -a --b=c",
+							"blublu":                   "blu blu",
+						}),
+					},
+				},
+				Process: report.Topology{
+					Nodes: report.Nodes{
+						"b": report.MakeNodeWith("b", map[string]string{
+							"cmdline": "scope --token=blibli",
+							"blibli":  "bli bli",
+						}),
+						"c": report.MakeNodeWith("c", map[string]string{}),
+					},
+				},
+			},
+		},
+		{
+			label: "censor both command line args and env vars",
+			have: report.CensorRawReport(r, report.CensorConfig{
+				HideCommandLineArguments: true,
+				HideEnvironmentVariables: true,
+			}),
+			want: report.Report{
+				Container: report.Topology{
+					Nodes: report.Nodes{
+						"a": report.MakeNodeWith("a", map[string]string{
+							"docker_container_command": "prog",
+							"blublu":                   "blu blu",
+						}),
+					},
+				},
+				Process: report.Topology{
+					Nodes: report.Nodes{
+						"b": report.MakeNodeWith("b", map[string]string{
+							"cmdline": "scope",
+							"blibli":  "bli bli",
+						}),
+						"c": report.MakeNodeWith("c", map[string]string{}),
+					},
+				},
+			},
+		},
+	} {
+		if !reflect.DeepEqual(c.want, c.have) {
+			t.Errorf("%s - %s", c.label, test.Diff(c.want, c.have))
+		}
+	}
+}

report/controls.go

@@ -35,6 +35,9 @@ func (cs Controls) Merge(other Controls) Controls {
 
 // Copy produces a copy of cs.
 func (cs Controls) Copy() Controls {
+	if cs == nil {
+		return nil
+	}
 	result := Controls{}
 	for k, v := range cs {
 		result[k] = v

report/dns.go

@@ -11,6 +11,9 @@ type DNSRecords map[string]DNSRecord
 
 // Copy makes a copy of the DNSRecords
 func (r DNSRecords) Copy() DNSRecords {
+	if r == nil {
+		return nil
+	}
 	cp := make(DNSRecords, len(r))
 	for k, v := range r {
 		cp[k] = v

report/map_keys.go

@@ -43,6 +43,7 @@ const (
 	DockerContainerUptime        = "docker_container_uptime"
 	DockerContainerRestartCount  = "docker_container_restart_count"
 	DockerContainerNetworkMode   = "docker_container_network_mode"
+	DockerEnvPrefix              = "docker_env_"
 	// probe/kubernetes
 	KubernetesName                 = "kubernetes_name"
 	KubernetesNamespace            = "kubernetes_namespace"

report/table.go

@@ -136,7 +136,7 @@ func (node Node) ExtractTable(template TableTemplate) (rows []Row, truncationCou
 	truncationCount = 0
 	if str, ok := node.Latest.Lookup(truncationCountPrefix + template.Prefix); ok {
 		if n, err := fmt.Sscanf(str, "%d", &truncationCount); n != 1 || err != nil {
-			log.Warn("Unexpected truncation count format %q", str)
+			log.Warnf("Unexpected truncation count format %q", str)
 		}
 	}
 

report/topology.go

@@ -214,6 +214,9 @@ type Nodes map[string]Node
 
 // Copy returns a value copy of the Nodes.
 func (n Nodes) Copy() Nodes {
+	if n == nil {
+		return nil
+	}
 	cp := make(Nodes, len(n))
 	for k, v := range n {
 		cp[k] = v