First cut: expression engine to RenderableNodes

render/dsl/expression.go

@@ -0,0 +1,405 @@
+package dsl
+
+import (
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+
+	"github.com/weaveworks/scope/probe/endpoint"
+	"github.com/weaveworks/scope/probe/host"
+	"github.com/weaveworks/scope/render"
+	"github.com/weaveworks/scope/report"
+)
+
+// Evaluator describes a monadic transformer of a RenderableNodes.
+type Evaluator interface {
+	Eval(render.RenderableNodes) render.RenderableNodes
+}
+
+// Expression is a single evaluator.
+type Expression struct {
+	selector
+	transformer
+}
+
+// Eval implements Evaluator.
+func (e Expression) Eval(rns render.RenderableNodes) render.RenderableNodes {
+	return e.transformer(rns, e.selector(rns))
+}
+
+// Expressions is an ordered collection of expressions.
+type Expressions []Expression
+
+// Eval implements Evaluator.
+func (e Expressions) Eval(rns render.RenderableNodes) render.RenderableNodes {
+	for _, expr := range e {
+		rns = expr.Eval(rns)
+	}
+	return rns
+}
+
+type selector func(render.RenderableNodes) []string
+
+type transformer func(render.RenderableNodes, []string) render.RenderableNodes
+
+func selectAll(rns render.RenderableNodes) []string {
+	out := make([]string, 0, len(rns))
+	for id := range rns {
+		out = append(out, id)
+	}
+	//log.Printf("select ALL: %d", len(out))
+	return out
+}
+
+func selectConnected(rns render.RenderableNodes) []string {
+	degree := map[string]int{}
+	for src, rn := range rns {
+		degree[src] += len(rn.Adjacency)
+		for _, dst := range rn.Adjacency {
+			degree[dst]++
+		}
+	}
+	out := []string{}
+	for id := range rns {
+		if degree[id] > 0 {
+			out = append(out, id)
+		}
+	}
+	return out
+}
+
+func selectNonlocal(rns render.RenderableNodes) []string {
+	local := report.Networks{}
+	for _, rn := range rns {
+		for k, v := range rn.Metadata {
+			if k == host.LocalNetworks {
+				local = append(local, report.ParseNetworks(v)...)
+			}
+		}
+	}
+	out := []string{}
+	for id, rn := range rns {
+		if addr, ok := rn.Metadata[endpoint.Addr]; ok {
+			if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
+				out = append(out, id) // valid addr metadata key, nonlocal
+				continue
+			}
+		}
+		if _, addr, ok := report.ParseAddressNodeID(id); ok {
+			if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
+				out = append(out, id) // valid address node ID, nonlocal
+				continue
+			}
+		}
+		if _, addr, _, ok := report.ParseEndpointNodeID(id); ok {
+			if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
+				out = append(out, id) // valid endpoint node ID, nonlocal
+				continue
+			}
+		}
+	}
+	//log.Printf("select NONLOCAL: %d", len(out))
+	return out
+}
+
+func selectLike(regex string) selector {
+	re, err := regexp.Compile(regex)
+	if err != nil {
+		//log.Printf("select LIKE %q: %v", s, err)
+		re = regexp.MustCompile("")
+	}
+	return func(rns render.RenderableNodes) []string {
+		out := []string{}
+		for id := range rns {
+			if re.MatchString(id) {
+				out = append(out, id)
+			}
+		}
+		//log.Printf("select LIKE %q: %d", s, len(out))
+		return out
+	}
+}
+
+func selectWith(s string) selector {
+	var k, v string
+	if fields := strings.SplitN(s, "=", 2); len(fields) == 1 {
+		k = strings.TrimSpace(fields[0])
+	} else if len(fields) == 2 {
+		k, v = strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1])
+	}
+	return func(rns render.RenderableNodes) []string {
+		out := []string{}
+		for id, md := range rns {
+			if vv, ok := md.Metadata[k]; ok {
+				if v == "" || (v != "" && v == vv) {
+					out = append(out, id)
+				}
+			}
+		}
+		//log.Printf("select WITH %q: %d", s, len(out))
+		return out
+	}
+}
+
+func selectNot(s selector) selector {
+	return func(rns render.RenderableNodes) []string {
+		set := map[string]struct{}{}
+		for _, id := range s(rns) {
+			set[id] = struct{}{}
+		}
+		out := []string{}
+		for id := range rns {
+			if _, ok := set[id]; ok {
+				continue // selected by that one -> not by this one
+			}
+			out = append(out, id)
+		}
+		//log.Printf("select NOT: %d", len(out))
+		return out
+	}
+}
+
+const highlightKey = "_highlight"
+
+func transformHighlight(rns render.RenderableNodes, ids []string) render.RenderableNodes {
+	for _, id := range ids {
+		rn := rns[id]
+		rn.Node.Metadata[highlightKey] = "true"
+		rns[id] = rn
+	}
+	//log.Printf("transform HIGHLIGHT %d: OK", len(ids))
+	return rns
+}
+
+func transformRemove(rns render.RenderableNodes, ids []string) render.RenderableNodes {
+	toRemove := map[string]struct{}{}
+	for _, id := range ids {
+		toRemove[id] = struct{}{}
+	}
+	out := render.RenderableNodes{}
+	for id := range rns {
+		if _, ok := toRemove[id]; ok {
+			continue
+		}
+		cp(out, rns, id)
+	}
+	clean(out, toRemove)
+	//log.Printf("transform REMOVE %d: in %d, out %d", len(ids), len(rns), len(out.NodeMetadatas))
+	return out
+}
+
+func transformShowOnly(rns render.RenderableNodes, ids []string) render.RenderableNodes {
+	out := render.RenderableNodes{}
+	for _, id := range ids {
+		cp(out, rns, id)
+	}
+	toRemove := map[string]struct{}{}
+	for id := range rns {
+		if _, ok := out[id]; !ok {
+			toRemove[id] = struct{}{}
+		}
+	}
+	clean(out, toRemove)
+	//log.Printf("transform SHOWONLY %d: in %d, out %d", len(ids), len(rns), len(out.NodeMetadatas))
+	return out
+}
+
+func transformMerge(newname string) transformer {
+	return func(rns render.RenderableNodes, ids []string) render.RenderableNodes {
+		mapped := map[string]string{}
+		toRemove := map[string]struct{}{}
+		for _, id := range ids {
+			mapped[id] = newname
+			toRemove[id] = struct{}{}
+		}
+		out := render.RenderableNodes{}
+		for id := range rns {
+			if dstID, ok := mapped[id]; ok {
+				merge(out, dstID, rns, id)
+			} else {
+				cp(out, rns, id)
+			}
+		}
+		shift(out, mapped)
+		clean(out, toRemove)
+		//log.Printf("transform MERGE %d: in %d, out %d", len(ids), len(rns), len(out.NodeMetadatas))
+		return out
+	}
+}
+
+// transformGroupBy takes a key, and merges all nodes who share the same value
+// for that key. It ignores nodes that don't have that key.
+func transformGroupBy(s string) transformer {
+	keys := []string{}
+	for _, key := range strings.Split(s, ",") {
+		keys = append(keys, strings.TrimSpace(key))
+	}
+
+	return func(rns render.RenderableNodes, ids []string) render.RenderableNodes {
+		set := map[string]struct{}{}
+		for _, id := range ids {
+			set[id] = struct{}{}
+		}
+
+		// Identify all nodes that should be grouped.
+		mapped := map[string]string{} // src ID: dst ID
+		toRemove := map[string]struct{}{}
+		for id, md := range rns {
+			if _, ok := set[id]; !ok {
+				continue // not selected
+			}
+
+			parts := []string{}
+			for _, key := range keys {
+				if val, ok := md.Metadata[key]; ok {
+					parts = append(parts, fmt.Sprintf("%s-%s", key, val))
+				}
+			}
+			if len(parts) < len(keys) {
+				continue // didn't match all required keys
+			}
+
+			dstID := strings.Join(parts, "-")
+			mapped[id] = dstID
+			toRemove[id] = struct{}{}
+		}
+
+		// Walk nodes again, merging those that should be grouped.
+		out := render.RenderableNodes{}
+		for id := range rns {
+			if dstID, ok := mapped[id]; ok {
+				merge(out, dstID, rns, id)
+			} else {
+				cp(out, rns, id)
+			}
+		}
+		shift(out, mapped)
+		clean(out, toRemove)
+
+		//log.Printf("transform GROUPBY %v %d: in %d, out %d", keys, len(ids), len(rns), len(out.NodeMetadatas))
+		return out
+	}
+}
+
+// transformJoin takes a key whose value is (expected to be) a node ID.
+// Basically, a foreign key. For every unique value (i.e. node) it finds, it
+// copies the foreign node's metadata into every node that had the
+// corresponding foreign key, and then deletes the foreign node.
+//
+// It's kind of like flattening the foreign nodes into all the nodes that
+// point to them, one-to-many.
+func transformJoin(key string) transformer {
+	return func(rns render.RenderableNodes, ids []string) render.RenderableNodes {
+		// key is e.g. host_node_id, value is a valid node ID.
+		// Collect the set of represented values (node IDs).
+		values := map[string]report.Node{}
+		toRemove := map[string]struct{}{}
+		for _, rn := range rns {
+			for k, v := range rn.Node.Metadata {
+				if k == key {
+					values[v] = report.MakeNode() // gather later
+					toRemove[v] = struct{}{}
+				}
+			}
+		}
+
+		// Next, gather the metadata from nodes in the set.
+		for id, rn := range rns {
+			if found, ok := values[id]; ok {
+				values[id] = found.Merge(rn.Node) // gather
+			}
+		}
+
+		// Finally, join that metadata to referential nodes.
+		// And delete the referenced nodes.
+		out := render.RenderableNodes{}
+		for id, rn := range rns {
+			if _, ok := values[id]; ok {
+				continue // delete the foreign nodes
+			}
+			cp(out, rns, id) // copy node
+			for k, v := range rn.Node.Metadata {
+				if k == key {
+					rn.Node = rn.Node.Merge(values[v]) // join metadata
+				}
+			}
+			out[id] = rn // write
+		}
+		clean(out, toRemove)
+
+		//log.Printf("transform JOIN %v %d: in %d, out %d", key, len(ids), len(rns), len(out.NodeMetadatas))
+		return out
+	}
+}
+
+func cp(dst render.RenderableNodes, src render.RenderableNodes, id string) {
+	dst[id] = src[id].Copy()
+
+	// Every transform that calls cp must call clean at the end, to remove
+	// dangling (uncopied) nodes from adjacency lists and edge metadatas.
+}
+
+func merge(dst render.RenderableNodes, dstID string, src render.RenderableNodes, srcID string) {
+	dst[dstID] = dst[dstID].Merge(src[srcID])
+
+	// Every transform that calls merge must call shift at the end, to update
+	// adjacency lists and edge metadatas.
+}
+
+func clean(dst render.RenderableNodes, toRemove map[string]struct{}) {
+	for id, rn := range dst {
+		// Clean out all the orphans from the adjacency list.
+		newAdjacency := report.IDList{}
+		for _, otherID := range rn.Node.Adjacency {
+			if _, ok := toRemove[otherID]; ok {
+				continue // can't be a dst anymore
+			}
+			newAdjacency = newAdjacency.Add(otherID)
+		}
+		rn.Node.Adjacency = newAdjacency
+
+		// Clean out all the orphans from the edges.
+		newEdges := report.EdgeMetadatas{}
+		for otherID, edge := range rn.Node.Edges {
+			if _, ok := toRemove[otherID]; ok {
+				continue // can't be an edge anymore
+			}
+			newEdges[otherID] = edge
+		}
+		rn.Node.Edges = newEdges
+
+		dst[id] = rn
+	}
+
+	// Just to be safe.
+	for id := range toRemove {
+		delete(dst, id)
+	}
+}
+
+func shift(dst render.RenderableNodes, mapping map[string]string) {
+	// We've got a mapping of old IDs to new IDs. Any adjacency targeting an
+	// old ID should be updated to the new ID.
+	for id, rn := range dst {
+		newAdjacency := report.IDList{}
+		for _, otherID := range rn.Node.Adjacency {
+			if mappedID, ok := mapping[otherID]; ok {
+				otherID = mappedID // just shift it on over
+			}
+			newAdjacency = newAdjacency.Add(otherID) // this will dedupe
+		}
+		rn.Node.Adjacency = newAdjacency
+
+		newEdges := report.EdgeMetadatas{}
+		for otherID, edge := range rn.Node.Edges {
+			if mappedID, ok := mapping[otherID]; ok {
+				otherID = mappedID
+			}
+			newEdges[otherID] = newEdges[otherID].Merge(edge) // important to merge here
+		}
+		rn.Node.Edges = newEdges
+
+		dst[id] = rn
+	}
+}

render/dsl/expression_internal_test.go

@@ -0,0 +1,245 @@
+package dsl
+
+import (
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sort"
+	"testing"
+
+	"github.com/weaveworks/scope/probe/endpoint"
+
+	"github.com/weaveworks/scope/probe/host"
+	"github.com/weaveworks/scope/render"
+	"github.com/weaveworks/scope/report"
+)
+
+func TestSelectAll(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a"),
+		"c": render.NewRenderableNode("c"),
+		"b": render.NewRenderableNode("b"),
+	}
+	testStringSlice(t, []string{"a", "b", "c"}, selectAll(rns))
+}
+
+func TestSelectConnected(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("m", "b", "c"))),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("m", "c"))),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("m"))),
+		"m": render.NewRenderableNode("m").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("n"))),
+		"n": render.NewRenderableNode("n"),
+		"x": render.NewRenderableNode("x"),
+		"y": render.NewRenderableNode("y"),
+		"z": render.NewRenderableNode("z"),
+	}
+	testStringSlice(t, []string{"a", "b", "c", "m", "n"}, selectConnected(rns))
+}
+
+func TestSelectNonlocal(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{host.LocalNetworks: "10.10.1.0/24"})),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{host.LocalNetworks: "10.10.2.0/24"})),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{host.LocalNetworks: "10.20.0.0/16"})),
+
+		// Test selecting nonlocal via endpoint.Addr metadata key
+		"x1": render.NewRenderableNode("x1").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "10.10.1.34"})),  // local
+		"x2": render.NewRenderableNode("x2").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "10.20.99.1"})),  // local
+		"x3": render.NewRenderableNode("x3").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "10.10.3.33"})),  // nonlocal
+		"x4": render.NewRenderableNode("x4").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "192.168.1.1"})), // nonlocal
+	}
+
+	// Test selecting nonlocal via parsing the node ID
+	var (
+		local1    = report.MakeAddressNodeID("some-host", "10.10.2.2")
+		nonlocal1 = report.MakeAddressNodeID("some-host", "10.11.12.13")
+		local2    = report.MakeEndpointNodeID("some-host", "10.20.0.1", "4040")
+		nonlocal2 = report.MakeEndpointNodeID("some-host", "10.21.32.43", "8080")
+	)
+	rns[local1] = render.NewRenderableNode(local1).WithNode(report.MakeNode())
+	rns[nonlocal1] = render.NewRenderableNode(nonlocal1).WithNode(report.MakeNode())
+	rns[local2] = render.NewRenderableNode(local2).WithNode(report.MakeNode())
+	rns[nonlocal2] = render.NewRenderableNode(nonlocal2).WithNode(report.MakeNode())
+
+	testStringSlice(t, []string{"x3", "x4", nonlocal1, nonlocal2}, selectNonlocal(rns))
+}
+
+func TestSelectLike(t *testing.T) {
+	rns := render.RenderableNodes{
+		"abcfooxyz": render.NewRenderableNode("abcfooxyz"),
+		"abcfoxyz":  render.NewRenderableNode("abcfoxyz"),
+		"fo":        render.NewRenderableNode("fo"),
+		"foo":       render.NewRenderableNode("foo"),
+		"fooo":      render.NewRenderableNode("fooo"),
+		"x_foo_y":   render.NewRenderableNode("x_foo_y"),
+	}
+	testStringSlice(t, []string{"abcfooxyz", "foo", "fooo", "x_foo_y"}, selectLike(`.*foo.*`)(rns))
+}
+
+func TestSelectWith(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "bar"})),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo": "bar"})),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo": "baz"})),
+		"x": render.NewRenderableNode("x").WithNode(report.MakeNodeWith(map[string]string{"qux": "qix"})),
+		"y": render.NewRenderableNode("y").WithNode(report.MakeNode()),
+	}
+	testStringSlice(t, []string{"a", "b", "c"}, selectWith("foo")(rns))
+	testStringSlice(t, []string{"a", "b"}, selectWith("foo=bar")(rns))
+	testStringSlice(t, []string{"x"}, selectWith("qux")(rns))
+	testStringSlice(t, []string{"x"}, selectWith("qux=qix")(rns))
+	testStringSlice(t, []string{}, selectWith("qux=XXX")(rns))
+}
+
+func TestSelectNot(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"})),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo": "2"})),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo": "3"})),
+		"x": render.NewRenderableNode("x").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"})),
+		"y": render.NewRenderableNode("y").WithNode(report.MakeNodeWith(map[string]string{"bar": "2"})),
+		"z": render.NewRenderableNode("z").WithNode(report.MakeNodeWith(map[string]string{"bar": "3"})),
+	}
+	testStringSlice(t, []string{"x", "y", "z"}, selectNot(selectWith("foo"))(rns))
+	testStringSlice(t, []string{"a", "b", "c"}, selectNot(selectWith("bar"))(rns))
+	testStringSlice(t, []string{}, selectNot(selectAll)(rns))
+}
+
+func TestTransformHighlight(t *testing.T) {
+	rns := render.RenderableNodes{"a": render.NewRenderableNode("a")}
+	get := func(rns render.RenderableNodes) string { return rns["a"].Metadata[highlightKey] }
+	out := transformHighlight(rns, selectAll(rns))
+	if want, have := "true", get(out); want != have {
+		t.Errorf("want %q, have %q", want, have)
+	}
+}
+
+func TestTransformRemove(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b", "c"))),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"}).WithAdjacency(report.MakeIDList("c"))),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"baz": "1"})),
+	}
+
+	// Remove all should totally kill the RenderableNodes
+	if want, have := 0, len(transformRemove(rns, selectAll(rns))); want != have {
+		t.Errorf("remove all: want %d, have %d", want, have)
+	}
+
+	// Removing c should kill the adjacency links to it
+	out := transformRemove(rns, selectLike("c")(rns))
+	if want, have := 2, len(out); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+	if want, have := 1, len(out["a"].Node.Adjacency); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+	if want, have := 0, len(out["b"].Node.Adjacency); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+}
+
+func TestTransformShowOnly(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b", "c"))),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"}).WithAdjacency(report.MakeIDList("c"))),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"baz": "1"})),
+	}
+
+	// Show only b should eliminate a, c, and the link to c
+	out := transformShowOnly(rns, selectLike("b")(rns))
+	if want, have := 1, len(out); want != have {
+		t.Errorf("want %d, have %d (%#+v)", want, have, out)
+	}
+	if want, have := 0, len(out["b"].Node.Adjacency); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+}
+
+func TestTransformMerge(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b", "c"))),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"}).WithAdjacency(report.MakeIDList("c"))),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"baz": "1"})),
+	}
+
+	name := "supernode"
+	out := transformMerge(name)(rns, selectAll(rns))
+	if want, have := 1, len(out); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+	if want, have := 3, len(out[name].Node.Metadata); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+	if want, have := 1, len(out[name].Node.Adjacency); want != have {
+		t.Fatalf("want %d, have %d", want, have)
+	}
+	if want, have := name, out[name].Node.Adjacency[0]; want != have {
+		t.Errorf("want %q, have %q", want, have)
+	}
+}
+
+func TestTransformGroupBy(t *testing.T) {
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b")).WithCounters(map[string]int{"c": 1})),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("c")).WithCounters(map[string]int{"c": 2})),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("d")).WithCounters(map[string]int{"c": 4})),
+		"d": render.NewRenderableNode("d").WithNode(report.MakeNodeWith(map[string]string{"foo": "2"}).WithCounters(map[string]int{"c": 8})),
+	}
+
+	out := transformGroupBy("foo")(rns, selectAll(rns))
+	if want, have := 2, len(out); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+	if _, ok := out["foo-1"]; !ok {
+		t.Fatalf("missing merged node")
+	}
+	if _, ok := out["foo-2"]; !ok {
+		t.Fatalf("missing unmerged node")
+	}
+	if want, have := 2, len(out["foo-1"].Node.Adjacency); want != have { // self-edge, and link to the other
+		t.Errorf("want %d, have %d", want, have)
+	}
+	if want, have := 1+2+4, out["foo-1"].Node.Counters["c"]; want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+	if want, have := 8, out["foo-2"].Node.Counters["c"]; want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+}
+
+func TestTransformJoin(t *testing.T) {
+	const key, value = "😇", "😈"
+	rns := render.RenderableNodes{
+		"a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo_node": "d", "1": "1"}).WithCounters(map[string]int{"c": 1})),
+		"b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo_node": "d", "2": "2"}).WithCounters(map[string]int{"c": 2})),
+		"c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo_node": "x", "4": "4"}).WithCounters(map[string]int{"c": 4})),
+		"d": render.NewRenderableNode("d").WithNode(report.MakeNodeWith(map[string]string{key: value}).WithCounters(map[string]int{"c": 8})),
+	}
+
+	out := transformJoin("foo_node")(rns, selectAll(rns))
+	if want, have := 3, len(out); want != have {
+		t.Errorf("want %d, have %d", want, have)
+	}
+	for _, id := range []string{"a", "b"} {
+		if want, have := value, out[id].Node.Metadata[key]; want != have {
+			t.Errorf("%s[%q]: want %q, have %q", id, key, want, have)
+		}
+	}
+	if want, have := 1+8, out["a"].Node.Counters["c"]; want != have {
+		t.Errorf("a: want %d, have %d", want, have)
+	}
+	if want, have := 2+8, out["b"].Node.Counters["c"]; want != have {
+		t.Errorf("b: want %d, have %d", want, have)
+	}
+}
+
+func testStringSlice(t *testing.T, want, have []string) {
+	sort.Strings(want)
+	sort.Strings(have)
+	if !reflect.DeepEqual(want, have) {
+		_, file, line, _ := runtime.Caller(1)
+		t.Errorf("%s:%d: want %v, have %v", filepath.Base(file), line, want, have)
+	}
+}

render/dsl/lexer.go

@@ -0,0 +1,326 @@
+package dsl
+
+import (
+	"fmt"
+	"strings"
+	"unicode/utf8"
+)
+
+// Expression  = [NOT] Selector [Transformer]
+// Selector    = ALL / CONNECTED / NONLOCAL / LIKE {{ <regex> }} / WITH {{ <key> [= <value>] }}
+// Transformer = HIGHLIGHT / REMOVE / SHOWONLY / MERGE / GROUPBY {{ <key>, ... }} // JOIN {{ key }}
+
+type lexer struct {
+	input string // string being scanned
+	start int    // start position of this item
+	pos   int    // current position within the input
+	width int    // width of last rune read
+	items chan item
+}
+
+func lex(input string) (*lexer, <-chan item) {
+	l := &lexer{
+		input: input,
+		items: make(chan item),
+	}
+	go l.run()
+	return l, l.items
+}
+
+const (
+	keywordNot       = "NOT"
+	keywordAll       = "ALL"
+	keywordConnected = "CONNECTED"
+	keywordNonlocal  = "NONLOCAL"
+	keywordLike      = "LIKE"
+	keywordWith      = "WITH"
+	keywordHighlight = "HIGHLIGHT"
+	keywordRemove    = "REMOVE"
+	keywordShowOnly  = "SHOWONLY"
+	keywordMerge     = "MERGE"
+	keywordGroupBy   = "GROUPBY"
+	keywordJoin      = "JOIN"
+)
+
+type itemType int
+
+const (
+	itemError itemType = iota
+	itemNot
+	itemAll
+	itemConnected
+	itemNonlocal
+	itemLike
+	itemWith
+	itemHighlight
+	itemRemove
+	itemShowOnly
+	itemMerge
+	itemGroupBy
+	itemJoin
+	itemRegex
+	itemKeyValue
+	itemKeyList
+	itemKey
+)
+
+func (t itemType) String() string {
+	switch t {
+	case itemError:
+		return "ERROR"
+	case itemNot:
+		return keywordNot
+	case itemAll:
+		return keywordAll
+	case itemConnected:
+		return keywordConnected
+	case itemNonlocal:
+		return keywordNonlocal
+	case itemLike:
+		return keywordLike
+	case itemWith:
+		return keywordWith
+	case itemHighlight:
+		return keywordHighlight
+	case itemRemove:
+		return keywordRemove
+	case itemShowOnly:
+		return keywordShowOnly
+	case itemMerge:
+		return keywordMerge
+	case itemGroupBy:
+		return keywordGroupBy
+	case itemJoin:
+		return keywordJoin
+	case itemRegex:
+		return "<regex>"
+	case itemKeyValue:
+		return "<key=value>"
+	case itemKeyList:
+		return "<key list>"
+	case itemKey:
+		return "<key>"
+	default:
+		return "unknown"
+	}
+}
+
+type stateFn func(*lexer) stateFn
+
+func (l *lexer) run() {
+	for state := lexExpression; state != nil; {
+		state = state(l)
+	}
+	close(l.items)
+}
+
+func (l *lexer) emit(t itemType) {
+	l.items <- item{t, l.input[l.start:l.pos]}
+	l.start = l.pos
+}
+
+const eof rune = -1
+
+func (l *lexer) next() (r rune) {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+func (l *lexer) backup() { l.pos -= l.width }
+
+// acceptRun consumes a run of runes from the valid set.
+func (l *lexer) acceptRun(validSet string) {
+	for strings.IndexRune(validSet, l.next()) >= 0 {
+		// consume
+	}
+	l.backup()
+}
+
+func (l *lexer) eatWhitespace() {
+	l.acceptRun(" \t\r\n")
+}
+
+// errorf terminates lexing with an error.
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+	l.items <- item{itemError, fmt.Sprintf(format, args...)}
+	return nil
+}
+
+type item struct {
+	itemType itemType
+	literal  string
+}
+
+func (i item) String() string {
+	return fmt.Sprintf("%s %q", i.itemType, i.literal)
+}
+
+func lexExpression(l *lexer) stateFn {
+	l.eatWhitespace()
+	if strings.HasPrefix(l.input[l.pos:], keywordNot) {
+		return lexNot
+	}
+	return lexSelector
+}
+
+func lexNot(l *lexer) stateFn {
+	l.pos += len(keywordNot)
+	l.emit(itemNot)
+	return lexSelector
+}
+
+func lexSelector(l *lexer) stateFn {
+	l.eatWhitespace()
+	switch {
+	case strings.HasPrefix(l.input[l.pos:], keywordAll):
+		return lexAll
+	case strings.HasPrefix(l.input[l.pos:], keywordConnected):
+		return lexConnected
+	case strings.HasPrefix(l.input[l.pos:], keywordNonlocal):
+		return lexNonlocal
+	case strings.HasPrefix(l.input[l.pos:], keywordLike):
+		return lexLike
+	case strings.HasPrefix(l.input[l.pos:], keywordWith):
+		return lexWith
+	default:
+		return l.errorf("bad selector")
+	}
+}
+
+func lexAll(l *lexer) stateFn {
+	l.pos += len(keywordAll)
+	l.emit(itemAll)
+	return lexTransformer
+}
+
+func lexConnected(l *lexer) stateFn {
+	l.pos += len(keywordConnected)
+	l.emit(itemConnected)
+	return lexTransformer
+}
+
+func lexNonlocal(l *lexer) stateFn {
+	l.pos += len(keywordNonlocal)
+	l.emit(itemNonlocal)
+	return lexTransformer
+}
+
+func lexLike(l *lexer) stateFn {
+	l.pos += len(keywordLike)
+	l.emit(itemLike)
+	return lexRegex
+}
+
+func lexWith(l *lexer) stateFn {
+	l.pos += len(keywordWith)
+	l.emit(itemWith)
+	return lexKeyValue
+}
+
+func lexRegex(l *lexer) stateFn {
+	return lexMeta("regex", itemRegex, lexTransformer)
+}
+
+func lexKeyValue(l *lexer) stateFn {
+	return lexMeta("key=value", itemKeyValue, lexTransformer)
+}
+
+func lexTransformer(l *lexer) stateFn {
+	l.eatWhitespace()
+	switch {
+	case l.pos == len(l.input):
+		return nil // done
+	case strings.HasPrefix(l.input[l.pos:], keywordHighlight):
+		return lexHighlight
+	case strings.HasPrefix(l.input[l.pos:], keywordRemove):
+		return lexRemove
+	case strings.HasPrefix(l.input[l.pos:], keywordShowOnly):
+		return lexShowOnly
+	case strings.HasPrefix(l.input[l.pos:], keywordMerge):
+		return lexMerge
+	case strings.HasPrefix(l.input[l.pos:], keywordGroupBy):
+		return lexGroupBy
+	case strings.HasPrefix(l.input[l.pos:], keywordJoin):
+		return lexJoin
+	default:
+		return l.errorf("bad transformer at position %d: %s", l.pos, l.input[l.pos:])
+	}
+}
+
+func lexHighlight(l *lexer) stateFn {
+	l.pos += len(keywordHighlight)
+	l.emit(itemHighlight)
+	return nil
+}
+
+func lexRemove(l *lexer) stateFn {
+	l.pos += len(keywordRemove)
+	l.emit(itemRemove)
+	return nil
+}
+
+func lexShowOnly(l *lexer) stateFn {
+	l.pos += len(keywordShowOnly)
+	l.emit(itemShowOnly)
+	return nil
+}
+
+func lexMerge(l *lexer) stateFn {
+	l.pos += len(keywordMerge)
+	l.emit(itemMerge)
+	return nil
+}
+
+func lexGroupBy(l *lexer) stateFn {
+	l.pos += len(keywordGroupBy)
+	l.emit(itemGroupBy)
+	return lexKeyList
+}
+
+func lexJoin(l *lexer) stateFn {
+	l.pos += len(keywordJoin)
+	l.emit(itemJoin)
+	return lexKey
+}
+
+func lexKeyList(l *lexer) stateFn {
+	return lexMeta("key list", itemKeyList, nil)
+}
+
+func lexKey(l *lexer) stateFn {
+	return lexMeta("key", itemKey, nil)
+}
+
+const (
+	leftMeta  = "{{"
+	rightMeta = "}}"
+)
+
+func lexMeta(what string, t itemType, next stateFn) stateFn {
+	return func(l *lexer) stateFn {
+		l.eatWhitespace()
+		if !strings.HasPrefix(l.input[l.pos:], leftMeta) {
+			return l.errorf("%s must begin with %s", what, leftMeta)
+		}
+		l.pos += len(leftMeta)
+		l.start = l.pos
+		for {
+			if l.pos > len(l.input) {
+				return l.errorf("%s must end with %s", what, rightMeta)
+			}
+			if strings.HasPrefix(l.input[l.pos:], rightMeta) {
+				break
+			}
+			l.pos++
+		}
+		l.emit(t)
+		l.pos += len(rightMeta)
+		l.start = l.pos
+		return next
+	}
+}

render/dsl/lexer_internal_test.go

@@ -0,0 +1,76 @@
+package dsl
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestLexer(t *testing.T) {
+	for _, test := range []struct {
+		input string
+		want  []item
+		err   error
+	}{
+		{
+			"",
+			[]item{},
+			errors.New("bad selector"),
+		},
+		{
+			"foo",
+			[]item{},
+			errors.New("bad selector"),
+		},
+		{
+			"ALL",
+			[]item{{itemAll, keywordAll}},
+			errors.New("bad transformer"),
+		},
+		{
+			"NOT ALL",
+			[]item{{itemNot, keywordNot}, {itemAll, keywordAll}},
+			errors.New("bad transformer"),
+		},
+		{
+			"ALL HIGHLIGHT",
+			[]item{{itemAll, keywordAll}, {itemHighlight, keywordHighlight}},
+			nil,
+		},
+		{
+			"WITH {{pid}} REMOVE",
+			[]item{{itemWith, keywordWith}, {itemKeyValue, "pid"}, {itemRemove, keywordRemove}},
+			nil,
+		},
+	} {
+		_, c := lex(test.input)
+		for item := range c {
+			if item.itemType == itemError {
+				if test.err == nil {
+					t.Errorf("%q: unexpected error: %v", test.input, item.literal)
+					break
+				}
+				if want, have := test.err.Error(), item.literal; want != have {
+					t.Errorf("%q: want error %q, have %q", test.input, want, have)
+					break
+				}
+				t.Logf("%q: got expected error %v", test.input, test.err)
+				break
+			}
+
+			if len(test.want) <= 0 {
+				t.Errorf("%q: got too many items", test.input)
+				break
+			}
+
+			want := test.want[0]
+			test.want = test.want[1:]
+
+			if want, have := want.itemType, item.itemType; want != have {
+				t.Errorf("%q: unexpected item: want %v, have %v", test.input, want, have)
+				break
+			}
+
+			t.Logf("%s: lex %s (%q) OK", test.input, item.itemType, item.literal)
+		}
+	}
+}

render/dsl/parser.go

@@ -0,0 +1,104 @@
+package dsl
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"math/rand"
+)
+
+// ParseExpression parses a single expression string.
+func ParseExpression(str string) (Expression, error) {
+	var (
+		expr Expression
+		not  bool
+	)
+	_, c := lex(str)
+	for item := range c {
+		switch item.itemType {
+		case itemNot:
+			not = !not
+
+		case itemAll:
+			expr.selector = selectAll
+
+		case itemConnected:
+			expr.selector = selectConnected
+
+		case itemNonlocal:
+			expr.selector = selectNonlocal
+
+		case itemLike:
+			item = <-c
+			switch item.itemType {
+			case itemRegex:
+				expr.selector = selectLike(item.literal)
+			default:
+				return Expression{}, fmt.Errorf("bad LIKE: want %s, got %s", itemRegex, item.itemType)
+			}
+
+		case itemWith:
+			item = <-c
+			switch item.itemType {
+			case itemKeyValue:
+				expr.selector = selectWith(item.literal)
+			default:
+				return Expression{}, fmt.Errorf("bad WITH: want %s, got %s", itemKeyValue, item.itemType)
+			}
+
+		case itemHighlight:
+			expr.transformer = transformHighlight
+
+		case itemRemove:
+			expr.transformer = transformRemove
+
+		case itemShowOnly:
+			expr.transformer = transformShowOnly
+
+		case itemMerge:
+			expr.transformer = transformMerge(fmt.Sprintf("%x", rand.Int31())) // TODO(pb): parameterize the name
+
+		case itemGroupBy:
+			item = <-c
+			switch item.itemType {
+			case itemKeyList:
+				expr.transformer = transformGroupBy(item.literal)
+			default:
+				return Expression{}, fmt.Errorf("bad GROUPBY: want %s, got %s", itemKeyList, item.itemType)
+			}
+
+		case itemJoin:
+			item = <-c
+			switch item.itemType {
+			case itemKey:
+				expr.transformer = transformJoin(item.literal)
+			default:
+				return Expression{}, fmt.Errorf("bad JOIN: want %s, got %s", itemKey, item.itemType)
+			}
+
+		default:
+			return Expression{}, errors.New(item.literal)
+		}
+	}
+	if not {
+		expr.selector = selectNot(expr.selector)
+	}
+	if expr.transformer == nil {
+		expr.transformer = transformHighlight
+	}
+	return expr, nil
+}
+
+// ParseExpressions parses multiple expression strings.
+func ParseExpressions(strs ...string) Expressions {
+	var exprs Expressions
+	for _, str := range strs {
+		expr, err := ParseExpression(str)
+		if err != nil {
+			log.Printf("%s: %v", str, err)
+			continue
+		}
+		exprs = append(exprs, expr)
+	}
+	return exprs
+}

render/dsl/parser_internal_test.go

@@ -0,0 +1,68 @@
+package dsl
+
+import (
+	"errors"
+	"reflect"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestParseExpression(t *testing.T) {
+	for _, test := range []struct {
+		input string
+		want  Expression
+		err   error
+	}{
+		{
+			"",
+			Expression{},
+			errors.New("bad selector"),
+		},
+		{
+			"ALL",
+			Expression{selectAll, transformHighlight},
+			nil,
+		},
+		{
+			"HIGHLIGHT",
+			Expression{},
+			errors.New("bad selector"),
+		},
+		{
+			"CONNECTED REMOVE",
+			Expression{selectConnected, transformRemove},
+			nil,
+		},
+		{
+			"NOT CONNECTED MERGE",
+			Expression{selectNot(selectConnected), transformMerge("foobar")},
+			nil,
+		},
+	} {
+		have, err := ParseExpression(test.input)
+		if err == nil && test.err != nil {
+			t.Errorf("%q: want error %q, have no error", test.input, test.err.Error())
+			continue
+		} else if err != nil && test.err == nil {
+			t.Errorf("%q: want no error, have error %q", test.input, err.Error())
+			continue
+		} else if err != nil && test.err != nil && test.err.Error() != err.Error() {
+			t.Errorf("%q: want error %q, have %q", test.input, test.err.Error(), err.Error())
+			continue
+		}
+		if want, have := nameof(test.want.selector), nameof(have.selector); want != have {
+			t.Errorf("%q: selector: want %v, have %v", test.input, want, have)
+		}
+		if want, have := nameof(test.want.transformer), nameof(have.transformer); want != have {
+			t.Errorf("%q: transformer: want %v, have %v", test.input, want, have)
+		}
+	}
+}
+
+func nameof(i interface{}) string {
+	full := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() // github.com/weaveworks/scope/render/dsl.selectAll
+	fields := strings.Split(full, ".")                             // [github com/weaveworks/scope/render/dsl selectAll]
+	last := fields[len(fields)-1]                                  // selectAll
+	return last
+}