experimental/dsl: a graph processing DSL

.gitignore

@@ -49,6 +49,7 @@ experimental/genreport/genreport
 experimental/graphviz/graphviz
 experimental/oneshot/oneshot
 experimental/_integration/_integration
+experimental/dsl/dsl
 *sublime-project
 *sublime-workspace
 *npm-debug.log

experimental/dsl/applications-by-name.txt

@@ -0,0 +1,5 @@
+ALL GROUPBY {{pid}}
+ALL GROUPBY {{comm}}
+NOT WITH {{comm}} REMOVE
+NOT CONNECTED REMOVE
+

experimental/dsl/applications.txt

@@ -0,0 +1,3 @@
+ALL GROUPBY {{pid}}
+NOT WITH {{pid}} REMOVE
+NOT CONNECTED REMOVE

experimental/dsl/containers-by-image.txt

@@ -0,0 +1,4 @@
+ALL GROUPBY {{pid}}
+ALL GROUPBY {{docker_container_id}}
+ALL GROUPBY {{docker_image_id}}
+NOT WITH {{docker_image_name}} REMOVE

experimental/dsl/containers.txt

@@ -0,0 +1,3 @@
+ALL GROUPBY {{pid}}
+ALL GROUPBY {{docker_container_id}}
+NOT WITH {{docker_container_id}} REMOVE

experimental/dsl/expression.go

@@ -0,0 +1,356 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"regexp"
+	"strings"
+
+	"github.com/weaveworks/scope/probe/endpoint"
+	"github.com/weaveworks/scope/probe/host"
+	"github.com/weaveworks/scope/render"
+	"github.com/weaveworks/scope/report"
+)
+
+type view []expression
+
+func (v view) eval(tpy report.Topology) report.Topology {
+	for _, expr := range v {
+		tpy = expr.eval(tpy)
+	}
+	return tpy
+}
+
+type expression struct {
+	selector
+	transformer
+}
+
+func (e expression) eval(tpy report.Topology) report.Topology {
+	if e.transformer == nil {
+		e.transformer = transformHighlight
+	}
+	return e.transformer(tpy, e.selector(tpy))
+}
+
+type selector func(report.Topology) []string
+
+type transformer func(report.Topology, []string) report.Topology
+
+func selectAll(tpy report.Topology) []string {
+	out := make([]string, 0, len(tpy.NodeMetadatas))
+	for id := range tpy.NodeMetadatas {
+		out = append(out, id)
+	}
+	log.Printf("select ALL: %d", len(out))
+	return out
+}
+
+func selectConnected(tpy report.Topology) []string {
+	degree := map[string]int{}
+	for src, dsts := range tpy.Adjacency {
+		a, ok := report.ParseAdjacencyID(src)
+		if !ok {
+			panic(src)
+		}
+		degree[a] += len(dsts)
+		for _, dst := range dsts {
+			degree[dst]++
+		}
+	}
+	out := []string{}
+	for id := range tpy.NodeMetadatas {
+		if degree[id] > 0 {
+			out = append(out, id)
+		}
+	}
+	log.Printf("select CONNECTED: %d", len(out))
+	return out
+}
+
+func selectNonlocal(tpy report.Topology) []string {
+	local := report.Networks{}
+	for _, md := range tpy.NodeMetadatas {
+		for k, v := range md.Metadata {
+			if k == host.LocalNetworks {
+				local = append(local, render.ParseNetworks(v)...)
+			}
+		}
+	}
+	out := []string{}
+	for id, md := range tpy.NodeMetadatas {
+		if addr, ok := md.Metadata[endpoint.Addr]; ok {
+			if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
+				out = append(out, id) // valid addr metadata key, nonlocal
+				continue
+			}
+		}
+		if _, addr, ok := report.ParseAddressNodeID(id); ok {
+			if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
+				out = append(out, id) // valid address node ID, nonlocal
+				continue
+			}
+		}
+		if _, addr, _, ok := report.ParseEndpointNodeID(id); ok {
+			if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) {
+				out = append(out, id) // valid endpoint node ID, nonlocal
+				continue
+			}
+		}
+	}
+	log.Printf("select NONLOCAL: %d", len(out))
+	return out
+}
+
+func selectLike(s string) selector {
+	re, err := regexp.Compile(s)
+	if err != nil {
+		log.Printf("select LIKE %q: %v", s, err)
+		re = regexp.MustCompile("")
+	}
+	return func(tpy report.Topology) []string {
+		out := []string{}
+		for id := range tpy.NodeMetadatas {
+			if re.MatchString(id) {
+				out = append(out, id)
+			}
+		}
+		log.Printf("select LIKE %q: %d", s, len(out))
+		return out
+	}
+}
+
+func selectWith(s string) selector {
+	var k, v string
+	if fields := strings.SplitN(s, "=", 2); len(fields) == 1 {
+		k = strings.TrimSpace(fields[0])
+	} else if len(fields) == 2 {
+		k, v = strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1])
+	}
+
+	return func(tpy report.Topology) []string {
+		out := []string{}
+		for id, md := range tpy.NodeMetadatas {
+			if vv, ok := md.Metadata[k]; ok {
+				if v == "" || (v != "" && v == vv) {
+					out = append(out, id)
+				}
+			}
+		}
+		log.Printf("select WITH %q: %d", s, len(out))
+		return out
+	}
+}
+
+func selectNot(s selector) selector {
+	return func(tpy report.Topology) []string {
+		set := map[string]struct{}{}
+		for _, id := range s(tpy) {
+			set[id] = struct{}{}
+		}
+		out := []string{}
+		for id := range tpy.NodeMetadatas {
+			if _, ok := set[id]; ok {
+				continue // selected by that one -> not by this one
+			}
+			out = append(out, id)
+		}
+		log.Printf("select NOT: %d", len(out))
+		return out
+	}
+}
+
+const highlightKey = "_highlight"
+
+func transformHighlight(tpy report.Topology, ids []string) report.Topology {
+	for _, id := range ids {
+		tpy.NodeMetadatas[id] = tpy.NodeMetadatas[id].Merge(report.MakeNodeMetadataWith(map[string]string{highlightKey: "true"}))
+	}
+	log.Printf("transform HIGHLIGHT %d: OK", len(ids))
+	return tpy
+}
+
+func transformRemove(tpy report.Topology, ids []string) report.Topology {
+	toRemove := map[string]struct{}{}
+	for _, id := range ids {
+		toRemove[id] = struct{}{}
+	}
+	out := report.MakeTopology()
+	for id := range tpy.NodeMetadatas {
+		if _, ok := toRemove[id]; ok {
+			continue
+		}
+		cp(out, tpy, id)
+	}
+	log.Printf("transform REMOVE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas))
+	return out
+}
+
+func transformShowOnly(tpy report.Topology, ids []string) report.Topology {
+	out := report.MakeTopology()
+	for _, id := range ids {
+		if _, ok := tpy.NodeMetadatas[id]; !ok {
+			continue
+		}
+		cp(out, tpy, id)
+	}
+	log.Printf("transform SHOWONLY %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas))
+	return out
+}
+
+func transformMerge(tpy report.Topology, ids []string) report.Topology {
+	name := fmt.Sprintf("%x", rand.Int31())
+	mapped := map[string]string{}
+	for _, id := range ids {
+		mapped[id] = name
+	}
+	out := report.MakeTopology()
+	for id := range tpy.NodeMetadatas {
+		if dstID, ok := mapped[id]; ok {
+			merge(out, dstID, tpy, id, mapped)
+		} else {
+			cp(out, tpy, id)
+		}
+	}
+	log.Printf("transform MERGE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas))
+	return out
+}
+
+func transformGroupBy(s string) transformer {
+	keys := []string{}
+	for _, key := range strings.Split(s, ",") {
+		keys = append(keys, strings.TrimSpace(key))
+	}
+
+	return func(tpy report.Topology, ids []string) report.Topology {
+		set := map[string]struct{}{}
+		for _, id := range ids {
+			set[id] = struct{}{}
+		}
+
+		// Identify all nodes that should be grouped.
+		mapped := map[string]string{} // src ID: dst ID
+		for id, md := range tpy.NodeMetadatas {
+			if _, ok := set[id]; !ok {
+				continue // not selected
+			}
+
+			parts := []string{}
+			for _, key := range keys {
+				if val, ok := md.Metadata[key]; ok {
+					parts = append(parts, fmt.Sprintf("%s-%s", key, val))
+				}
+			}
+			if len(parts) < len(keys) {
+				continue // didn't match all required keys
+			}
+
+			dstID := strings.Join(parts, "-")
+			mapped[id] = dstID
+		}
+
+		// Walk nodes again, merging those that should be grouped.
+		out := report.MakeTopology()
+		for id := range tpy.NodeMetadatas {
+			if dstID, ok := mapped[id]; ok {
+				merge(out, dstID, tpy, id, mapped)
+			} else {
+				cp(out, tpy, id)
+			}
+		}
+
+		log.Printf("transform GROUPBY %v %d: in %d, out %d", keys, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas))
+		return out
+	}
+}
+
+func transformJoin(key string) transformer {
+	return func(tpy report.Topology, ids []string) report.Topology {
+		// key is e.g. host_node_id.
+		// Collect the set of represented values.
+		values := map[string]report.NodeMetadata{}
+		for _, md := range tpy.NodeMetadatas {
+			for k, v := range md.Metadata {
+				if k == key {
+					values[v] = report.MakeNodeMetadata() // gather later
+				}
+			}
+		}
+
+		// Next, gather the metadata from nodes in the set.
+		for id, md := range tpy.NodeMetadatas {
+			if found, ok := values[id]; ok {
+				values[id] = found.Merge(md) // gather
+			}
+		}
+
+		// Finally, join that metadata to referential nodes.
+		// And delete the referenced nodes.
+		out := report.MakeTopology()
+		for id, md := range tpy.NodeMetadatas {
+			if _, ok := values[id]; ok {
+				continue // delete
+			}
+			cp(out, tpy, id) // copy node
+			for k, v := range md.Metadata {
+				if k == key {
+					md = md.Merge(values[v]) // join metadata
+				}
+			}
+			out.NodeMetadatas[id] = md // write
+		}
+
+		log.Printf("transform JOIN %v %d: in %d, out %d", key, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas))
+		return out
+	}
+}
+
+func cp(dst report.Topology, src report.Topology, id string) {
+	adjacencyID := report.MakeAdjacencyID(id)
+	dst.Adjacency[adjacencyID] = src.Adjacency[adjacencyID]
+
+	for _, otherID := range dst.Adjacency[id] {
+		edgeID := report.MakeEdgeID(id, otherID)
+		dst.EdgeMetadatas[edgeID] = src.EdgeMetadatas[edgeID]
+	}
+
+	dst.NodeMetadatas[id] = src.NodeMetadatas[id]
+}
+
+func merge(dst report.Topology, dstID string, src report.Topology, srcID string, mapped map[string]string) {
+	// We gonna take srcID from src and merge it into dstID in dst. That's
+	// like renaming the node, so we gotta update the adjacency lists. Both
+	// outgoing *and* incoming links!
+	dstAdjacencyID := report.MakeAdjacencyID(dstID)
+	srcAdjacencyID := report.MakeAdjacencyID(srcID)
+
+	// Merge the src's adjacency list into the dst topology.
+	dst.Adjacency[dstAdjacencyID] = dst.Adjacency[dstAdjacencyID].Merge(src.Adjacency[srcAdjacencyID])
+
+	// Update any dst adjacencies from the old ID to the new ID.
+	for existingSrcAdjacencyID, existingDstIDs := range dst.Adjacency {
+		for i, existingDstID := range existingDstIDs {
+			if newDstID, ok := mapped[existingDstID]; ok {
+				existingDstIDs[i] = newDstID
+			}
+		}
+		dst.Adjacency[existingSrcAdjacencyID] = existingDstIDs
+	}
+
+	// Update the EdgeMetadatas to have the new IDs.
+	for _, otherID := range src.Adjacency[srcAdjacencyID] {
+		oldEdgeID := report.MakeEdgeID(srcID, otherID)
+		newEdgeID := report.MakeEdgeID(dstID, otherID)
+		dst.EdgeMetadatas[newEdgeID] = dst.EdgeMetadatas[newEdgeID].Merge(src.EdgeMetadatas[oldEdgeID])
+	}
+
+	// Merge the src node metadata into the dst node metadata.
+	md, ok := dst.NodeMetadatas[dstID]
+	if !ok {
+		md = report.MakeNodeMetadata()
+	}
+	md = md.Merge(src.NodeMetadatas[srcID])
+	dst.NodeMetadatas[dstID] = md
+}

experimental/dsl/exprs.txt

@@ -0,0 +1,3 @@
+ALL GROUPBY {{ pid }}
+ALL GROUPBY {{ docker_container_id }}
+NOT WITH {{ docker_container_id }} REMOVE

experimental/dsl/graphviz.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/weaveworks/scope/probe/host"
+
+	"github.com/weaveworks/scope/report"
+)
+
+func dot(w io.Writer, tpy report.Topology) {
+	var nodes []string
+	for id, md := range tpy.NodeMetadatas {
+		label := id + "\n"
+
+		var lines []string
+		var highlight bool
+		for k, v := range md.Metadata {
+			if k == host.LocalNetworks {
+				continue
+			}
+			if k == highlightKey {
+				highlight = true
+				continue
+			}
+			lines = append(lines, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(lines)
+		for _, line := range lines {
+			label += "\n" + line
+		}
+
+		fillcolor := "white"
+		if highlight {
+			fillcolor = "yellow"
+		}
+
+		nodes = append(nodes, fmt.Sprintf("%q [label=%q fillcolor=%s];", id, label, fillcolor))
+	}
+
+	var edges []string
+	for src, dsts := range tpy.Adjacency {
+		a, ok := report.ParseAdjacencyID(src)
+		if !ok {
+			panic(src)
+		}
+		for _, dst := range dsts {
+			edges = append(edges, fmt.Sprintf("%q -> %q;", a, dst))
+		}
+	}
+
+	sort.Strings(nodes)
+	sort.Strings(edges)
+
+	fmt.Fprintf(w, "digraph G {\n")
+	fmt.Fprintf(w, "\tgraph [ overlap=false, mode=hier ];\n")
+	fmt.Fprintf(w, "\tnode [ shape=rect, style=filled ];\n")
+	fmt.Fprintf(w, "\toutputorder=edgesfirst;\n")
+	fmt.Fprintf(w, "\n")
+	for _, line := range nodes {
+		fmt.Fprintf(w, "\t%s\n", line)
+	}
+	fmt.Fprintf(w, "\n")
+	for _, line := range edges {
+		fmt.Fprintf(w, "\t%s\n", line)
+	}
+	fmt.Fprintf(w, "}\n")
+}

experimental/dsl/handle.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os/exec"
+
+	"github.com/weaveworks/scope/report"
+)
+
+func handleJSON(tpy report.Topology) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(getView(r).eval(tpy.Copy()).NodeMetadatas); err != nil {
+			log.Print(err)
+			return
+		}
+	}
+}
+
+func handleDOT(tpy report.Topology) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		dot(w, getView(r).eval(tpy.Copy()))
+	}
+}
+
+func handleSVG(tpy report.Topology) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		cmd := exec.Command(engine(r), "-Tsvg")
+
+		wc, err := cmd.StdinPipe()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		cmd.Stdout = w
+
+		dot(wc, getView(r).eval(tpy.Copy()))
+		wc.Close()
+
+		w.Header().Set("Content-Type", "image/svg+xml")
+		if err := cmd.Run(); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func handleHTML(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/html")
+	fmt.Fprintf(w, "<html><head>\n")
+	//fmt.Fprintf(w, `<meta http-equiv="refresh" content="10">`+"\n")
+	fmt.Fprintf(w, "</head><body>\n")
+	fmt.Fprintf(w, `<center><img src="/svg?%s" width="100%%" height="95%%"></center>`+"\n", r.URL.Query().Encode())
+	fmt.Fprintf(w, "</body></html>\n")
+}
+
+func getView(r *http.Request) view {
+	var expressions []expression
+	expressions = append(expressions, getExpressionsFromBody(r)...)
+	expressions = append(expressions, getExpressionsFromQuery(r)...)
+	log.Printf("Serving view with %d expression(s)", len(expressions))
+	return view(expressions)
+}
+
+func getExpressionsFromQuery(r *http.Request) []expression {
+	strs := []string{}
+	if err := r.ParseForm(); err != nil {
+		log.Printf("Get expressions from query: %v", err)
+		return []expression{}
+	}
+	for _, str := range r.Form["expr"] {
+		log.Printf("Query expression: %s", str)
+		strs = append(strs, str)
+	}
+	return parseExpressions(strs)
+}
+
+func getExpressionsFromBody(r *http.Request) []expression {
+	strs := []string{}
+	scanner := bufio.NewScanner(r.Body)
+	for scanner.Scan() {
+		log.Printf("Body expression: %s", scanner.Text())
+		strs = append(strs, scanner.Text())
+	}
+	return parseExpressions(strs)
+}
+
+func engine(r *http.Request) string {
+	engine := r.FormValue("engine")
+	if engine == "" {
+		engine = "dot"
+	}
+	return engine
+}

experimental/dsl/lexer.go

@@ -0,0 +1,326 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"unicode/utf8"
+)
+
+// Expression  = [NOT] Selector [Transformer]
+// Selector    = ALL / CONNECTED / NONLOCAL / LIKE {{ <regex> }} / WITH {{ <key> [= <value>] }}
+// Transformer = HIGHLIGHT / REMOVE / SHOWONLY / MERGE / GROUPBY {{ <key>, ... }} // JOIN {{ key }}
+
+type lexer struct {
+	input string // string being scanned
+	start int    // start position of this item
+	pos   int    // current position within the input
+	width int    // width of last rune read
+	items chan item
+}
+
+func lex(input string) (*lexer, <-chan item) {
+	l := &lexer{
+		input: input,
+		items: make(chan item),
+	}
+	go l.run()
+	return l, l.items
+}
+
+const (
+	keywordNot       = "NOT"
+	keywordAll       = "ALL"
+	keywordConnected = "CONNECTED"
+	keywordNonlocal  = "NONLOCAL"
+	keywordLike      = "LIKE"
+	keywordWith      = "WITH"
+	keywordHighlight = "HIGHLIGHT"
+	keywordRemove    = "REMOVE"
+	keywordShowOnly  = "SHOWONLY"
+	keywordMerge     = "MERGE"
+	keywordGroupBy   = "GROUPBY"
+	keywordJoin      = "JOIN"
+)
+
+type itemType int
+
+const (
+	itemError itemType = iota
+	itemNot
+	itemAll
+	itemConnected
+	itemNonlocal
+	itemLike
+	itemWith
+	itemHighlight
+	itemRemove
+	itemShowOnly
+	itemMerge
+	itemGroupBy
+	itemJoin
+	itemRegex
+	itemKeyValue
+	itemKeyList
+	itemKey
+)
+
+func (t itemType) String() string {
+	switch t {
+	case itemError:
+		return "ERROR"
+	case itemNot:
+		return keywordNot
+	case itemAll:
+		return keywordAll
+	case itemConnected:
+		return keywordConnected
+	case itemNonlocal:
+		return keywordNonlocal
+	case itemLike:
+		return keywordLike
+	case itemWith:
+		return keywordWith
+	case itemHighlight:
+		return keywordHighlight
+	case itemRemove:
+		return keywordRemove
+	case itemShowOnly:
+		return keywordShowOnly
+	case itemMerge:
+		return keywordMerge
+	case itemGroupBy:
+		return keywordGroupBy
+	case itemJoin:
+		return keywordJoin
+	case itemRegex:
+		return "<regex>"
+	case itemKeyValue:
+		return "<key=value>"
+	case itemKeyList:
+		return "<key list>"
+	case itemKey:
+		return "<key>"
+	default:
+		return "unknown"
+	}
+}
+
+type stateFn func(*lexer) stateFn
+
+func (l *lexer) run() {
+	for state := lexExpression; state != nil; {
+		state = state(l)
+	}
+	close(l.items)
+}
+
+func (l *lexer) emit(t itemType) {
+	l.items <- item{t, l.input[l.start:l.pos]}
+	l.start = l.pos
+}
+
+const eof rune = -1
+
+func (l *lexer) next() (r rune) {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+func (l *lexer) backup() { l.pos -= l.width }
+
+// acceptRun consumes a run of runes from the valid set.
+func (l *lexer) acceptRun(validSet string) {
+	for strings.IndexRune(validSet, l.next()) >= 0 {
+		// consume
+	}
+	l.backup()
+}
+
+func (l *lexer) eatWhitespace() {
+	l.acceptRun(" \t\r\n")
+}
+
+// errorf terminates lexing with an error.
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+	l.items <- item{itemError, fmt.Sprintf(format, args...)}
+	return nil
+}
+
+type item struct {
+	itemType itemType
+	literal  string
+}
+
+func (i item) String() string {
+	return fmt.Sprintf("%s %q", i.itemType, i.literal)
+}
+
+func lexExpression(l *lexer) stateFn {
+	l.eatWhitespace()
+	if strings.HasPrefix(l.input[l.pos:], keywordNot) {
+		return lexNot
+	}
+	return lexSelector
+}
+
+func lexNot(l *lexer) stateFn {
+	l.pos += len(keywordNot)
+	l.emit(itemNot)
+	return lexSelector
+}
+
+func lexSelector(l *lexer) stateFn {
+	l.eatWhitespace()
+	switch {
+	case strings.HasPrefix(l.input[l.pos:], keywordAll):
+		return lexAll
+	case strings.HasPrefix(l.input[l.pos:], keywordConnected):
+		return lexConnected
+	case strings.HasPrefix(l.input[l.pos:], keywordNonlocal):
+		return lexNonlocal
+	case strings.HasPrefix(l.input[l.pos:], keywordLike):
+		return lexLike
+	case strings.HasPrefix(l.input[l.pos:], keywordWith):
+		return lexWith
+	default:
+		return l.errorf("bad selector")
+	}
+}
+
+func lexAll(l *lexer) stateFn {
+	l.pos += len(keywordAll)
+	l.emit(itemAll)
+	return lexTransformer
+}
+
+func lexConnected(l *lexer) stateFn {
+	l.pos += len(keywordConnected)
+	l.emit(itemConnected)
+	return lexTransformer
+}
+
+func lexNonlocal(l *lexer) stateFn {
+	l.pos += len(keywordNonlocal)
+	l.emit(itemNonlocal)
+	return lexTransformer
+}
+
+func lexLike(l *lexer) stateFn {
+	l.pos += len(keywordLike)
+	l.emit(itemLike)
+	return lexRegex
+}
+
+func lexWith(l *lexer) stateFn {
+	l.pos += len(keywordWith)
+	l.emit(itemWith)
+	return lexKeyValue
+}
+
+func lexRegex(l *lexer) stateFn {
+	return lexMeta("regex", itemRegex, lexTransformer)
+}
+
+func lexKeyValue(l *lexer) stateFn {
+	return lexMeta("key=value", itemKeyValue, lexTransformer)
+}
+
+func lexTransformer(l *lexer) stateFn {
+	l.eatWhitespace()
+	switch {
+	case l.pos == len(l.input):
+		return nil // done
+	case strings.HasPrefix(l.input[l.pos:], keywordHighlight):
+		return lexHighlight
+	case strings.HasPrefix(l.input[l.pos:], keywordRemove):
+		return lexRemove
+	case strings.HasPrefix(l.input[l.pos:], keywordShowOnly):
+		return lexShowOnly
+	case strings.HasPrefix(l.input[l.pos:], keywordMerge):
+		return lexMerge
+	case strings.HasPrefix(l.input[l.pos:], keywordGroupBy):
+		return lexGroupBy
+	case strings.HasPrefix(l.input[l.pos:], keywordJoin):
+		return lexJoin
+	default:
+		return l.errorf("bad transformer at position %d: %s", l.pos, l.input[l.pos:])
+	}
+}
+
+func lexHighlight(l *lexer) stateFn {
+	l.pos += len(keywordHighlight)
+	l.emit(itemHighlight)
+	return nil
+}
+
+func lexRemove(l *lexer) stateFn {
+	l.pos += len(keywordRemove)
+	l.emit(itemRemove)
+	return nil
+}
+
+func lexShowOnly(l *lexer) stateFn {
+	l.pos += len(keywordShowOnly)
+	l.emit(itemShowOnly)
+	return nil
+}
+
+func lexMerge(l *lexer) stateFn {
+	l.pos += len(keywordMerge)
+	l.emit(itemMerge)
+	return nil
+}
+
+func lexGroupBy(l *lexer) stateFn {
+	l.pos += len(keywordGroupBy)
+	l.emit(itemGroupBy)
+	return lexKeyList
+}
+
+func lexJoin(l *lexer) stateFn {
+	l.pos += len(keywordJoin)
+	l.emit(itemJoin)
+	return lexKey
+}
+
+func lexKeyList(l *lexer) stateFn {
+	return lexMeta("key list", itemKeyList, nil)
+}
+
+func lexKey(l *lexer) stateFn {
+	return lexMeta("key", itemKey, nil)
+}
+
+const (
+	leftMeta  = "{{"
+	rightMeta = "}}"
+)
+
+func lexMeta(what string, t itemType, next stateFn) stateFn {
+	return func(l *lexer) stateFn {
+		l.eatWhitespace()
+		if !strings.HasPrefix(l.input[l.pos:], leftMeta) {
+			return l.errorf("%s must begin with %s", what, leftMeta)
+		}
+		l.pos += len(leftMeta)
+		l.start = l.pos
+		for {
+			if l.pos > len(l.input) {
+				return l.errorf("%s must end with %s", what, rightMeta)
+			}
+			if strings.HasPrefix(l.input[l.pos:], rightMeta) {
+				break
+			}
+			l.pos++
+		}
+		l.emit(t)
+		l.pos += len(rightMeta)
+		l.start = l.pos
+		return next
+	}
+}

experimental/dsl/main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/weaveworks/scope/report"
+)
+
+func main() {
+	var (
+		listen = flag.String("listen", ":8080", "HTTP listen address")
+	)
+	flag.Parse()
+
+	// Decode report
+	var rpt report.Report
+	if err := json.NewDecoder(os.Stdin).Decode(&rpt); err != nil {
+		log.Fatal(err)
+	}
+
+	// Flatten all the topologies
+	tpy := report.MakeTopology()
+	for _, t := range rpt.Topologies() {
+		if err := conflict(tpy, t); err != nil {
+			log.Fatal(err)
+		}
+		tpy = tpy.Merge(t)
+	}
+	log.Printf("topology with %d node(s)", len(tpy.NodeMetadatas))
+
+	// Serve HTTP
+	http.HandleFunc("/json", handleJSON(tpy))
+	http.HandleFunc("/dot", handleDOT(tpy))
+	http.HandleFunc("/svg", handleSVG(tpy))
+	http.HandleFunc("/", handleHTML)
+	log.Printf("listening on %s", *listen)
+	log.Fatal(http.ListenAndServe(*listen, nil))
+}
+
+// Just a safety check.
+func conflict(a, b report.Topology) error {
+	var errs []string
+	for id := range a.NodeMetadatas {
+		if _, ok := b.NodeMetadatas[id]; ok {
+			errs = append(errs, id)
+		}
+	}
+	for id := range b.NodeMetadatas {
+		if _, ok := a.NodeMetadatas[id]; ok {
+			errs = append(errs, id)
+		}
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf(strings.Join(errs, ", "))
+	}
+	return nil
+}

experimental/dsl/medium.json

@@ -0,0 +1,82 @@
+{
+  "Process": {
+    "NodeMetadatas": {
+      "nydev.bourgon.org;1295": {
+        "Metadata": {
+          "cmdline": "/app db1 db2 db3 ",
+          "comm": "app",
+          "docker_container_id": "0952127d5294bf20591d90fc9dc8f15d684c1a140e50ac92e8807293faeb6c5b",
+          "host_node_id": "nydev.bourgon.org;<host>",
+          "pid": "1295",
+          "ppid": "5082",
+          "threads": "7",
+          "topology": "process"
+        }
+      }
+    }
+  },
+  "Endpoint": {
+    "Adjacency": {
+      ">nydev.bourgon.org;172.17.0.56;45890": [
+        "nydev.bourgon.org;172.17.0.54;9000"
+      ],
+      ">nydev.bourgon.org;172.17.0.56;49569": [
+        "nydev.bourgon.org;172.17.0.50;9000"
+      ],
+      ">nydev.bourgon.org;172.17.0.56;60705": [
+        "nydev.bourgon.org;172.17.0.52;9000"
+      ],
+      ">nydev.bourgon.org;172.17.0.56;8080": [
+        "nydev.bourgon.org;172.17.0.60;49914",
+        "nydev.bourgon.org;172.17.0.62;42329"
+      ]
+    },
+    "NodeMetadatas": {
+      "nydev.bourgon.org;172.17.0.56;45890": {
+        "Metadata": {
+          "addr": "172.17.0.56",
+          "host_node_id": "nydev.bourgon.org;<host>",
+          "pid": "1295",
+          "port": "45890",
+          "topology": "endpoint"
+        }
+      },
+      "nydev.bourgon.org;172.17.0.56;49569": {
+        "Metadata": {
+          "addr": "172.17.0.56",
+          "host_node_id": "nydev.bourgon.org;<host>",
+          "pid": "1295",
+          "port": "49569",
+          "topology": "endpoint"
+        }
+      },
+      "nydev.bourgon.org;172.17.0.56;60705": {
+        "Metadata": {
+          "addr": "172.17.0.56",
+          "host_node_id": "nydev.bourgon.org;<host>",
+          "pid": "1295",
+          "port": "60705",
+          "topology": "endpoint"
+        }
+      },
+      "nydev.bourgon.org;172.17.0.56;8080": {
+        "Metadata": {
+          "addr": "172.17.0.56",
+          "host_node_id": "nydev.bourgon.org;<host>",
+          "pid": "1295",
+          "port": "8080",
+          "topology": "endpoint"
+        }
+      },
+      "nydev.bourgon.org;172.17.0.54;9000": {
+        "Metadata": {
+          "addr": "172.17.0.54",
+          "host_node_id": "nydev.bourgon.org;<host>",
+          "pid": "1212",
+          "port": "9000",
+          "topology": "endpoint"
+        }
+      }
+    }
+  }
+}

experimental/dsl/parser.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"fmt"
+	"log"
+)
+
+func parseExpressions(strs []string) []expression {
+	var exprs []expression
+	for _, str := range strs {
+		expr, err := parseExpression(str)
+		if err != nil {
+			log.Printf("%s: %v", str, err)
+			continue
+		}
+		log.Printf("%s: OK", str)
+		exprs = append(exprs, expr)
+	}
+	return exprs
+}
+
+func parseExpression(str string) (expression, error) {
+	var (
+		expr expression
+		not  bool
+	)
+	_, c := lex(str)
+	for item := range c {
+		switch item.itemType {
+		case itemNot:
+			not = !not
+
+		case itemAll:
+			expr.selector = selectAll
+
+		case itemConnected:
+			expr.selector = selectConnected
+
+		case itemNonlocal:
+			expr.selector = selectNonlocal
+
+		case itemLike:
+			item = <-c
+			switch item.itemType {
+			case itemRegex:
+				expr.selector = selectLike(item.literal)
+			default:
+				return expression{}, fmt.Errorf("bad LIKE: want %s, got %s", itemRegex, item.itemType)
+			}
+
+		case itemWith:
+			item = <-c
+			switch item.itemType {
+			case itemKeyValue:
+				expr.selector = selectWith(item.literal)
+			default:
+				return expression{}, fmt.Errorf("bad WITH: want %s, got %s", itemKeyValue, item.itemType)
+			}
+
+		case itemHighlight:
+			expr.transformer = transformHighlight
+
+		case itemRemove:
+			expr.transformer = transformRemove
+
+		case itemShowOnly:
+			expr.transformer = transformShowOnly
+
+		case itemMerge:
+			expr.transformer = transformMerge
+
+		case itemGroupBy:
+			item = <-c
+			switch item.itemType {
+			case itemKeyList:
+				expr.transformer = transformGroupBy(item.literal)
+			default:
+				return expression{}, fmt.Errorf("bad GROUPBY: want %s, got %s", itemKeyList, item.itemType)
+			}
+
+		case itemJoin:
+			item = <-c
+			switch item.itemType {
+			case itemKey:
+				expr.transformer = transformJoin(item.literal)
+			default:
+				return expression{}, fmt.Errorf("bad JOIN: want %s, got %s", itemKey, item.itemType)
+			}
+
+		default:
+			return expression{}, fmt.Errorf("%s: %s", str, item.literal)
+		}
+	}
+	if not {
+		expr.selector = selectNot(expr.selector)
+	}
+	return expr, nil
+}

experimental/dsl/small.json

@@ -0,0 +1,60 @@
+{
+  "Endpoint": {
+    "Adjacency": {
+      ">endpoint-1": [
+        "endpoint-2",
+        "endpoint-3"
+      ]
+    },
+    "EdgeMetadatas": {},
+    "NodeMetadatas": {
+      "endpoint-1": {
+        "Metadata": {
+          "endpoint_statistic": "one",
+          "pid": "1234",
+          "host_node_id": "nydev.bourgon.org;<host>"
+        }
+      },
+      "endpoint-2": {
+        "Metadata": {
+          "endpoint_statistic": "two",
+          "host_node_id": "nydev.bourgon.org;<host>"
+        }
+      },
+      "endpoint-3": {
+        "Metadata": {
+          "endpoint_statistic": "three",
+          "host_node_id": "nydev.bourgon.org;<host>"
+        }
+      }
+    }
+  },
+  "Process": {
+    "Adjacency": {},
+    "EdgeMetadatas": {},
+    "NodeMetadatas": {
+      "process-1234": {
+        "Metadata": {
+          "pid": "1234",
+          "docker_container_id": "abc",
+          "process_statistic": "555",
+          "host_node_id": "nydev.bourgon.org;<host>"
+        }
+      }
+    }
+  },
+  "Container": {
+    "Adjacency": {},
+    "EdgeMetadatas": {},
+    "NodeMetadatas": {
+      "abc": {
+        "Metadata": {
+          "docker_container_id": "abc",
+          "docker_statistic": "999",
+          "host_node_id": "nydev.bourgon.org;<host>"
+        }
+      }
+    }
+  },
+  "Window": 60000000000
+}

render/theinternet.go

@@ -13,27 +13,33 @@ import (
 // used to determine which nodes in the report are "remote", i.e. outside of
 // our infrastructure.
 func LocalNetworks(r report.Report) report.Networks {
-	var (
-		result   = report.Networks{}
-		networks = map[string]struct{}{}
-	)
-
+	result := report.Networks{}
 	for _, md := range r.Host.NodeMetadatas {
 		val, ok := md.Metadata[host.LocalNetworks]
 		if !ok {
 			continue
 		}
-		for _, s := range strings.Fields(val) {
-			_, ipNet, err := net.ParseCIDR(s)
-			if err != nil {
-				continue
-			}
-			_, ok := networks[ipNet.String()]
-			if !ok {
-				result = append(result, ipNet)
-				networks[ipNet.String()] = struct{}{}
-			}
-		}
+		result = append(result, ParseNetworks(val)...)
 	}
 	return result
 }
+
+// ParseNetworks converts a string of space-separated CIDRs to a
+// report.Networks.
+func ParseNetworks(v string) report.Networks {
+	var (
+		nets = report.Networks{}
+		set  = map[string]struct{}{}
+	)
+	for _, s := range strings.Fields(v) {
+		_, ipNet, err := net.ParseCIDR(s)
+		if err != nil {
+			continue
+		}
+		if _, ok := set[ipNet.String()]; !ok {
+			nets = append(nets, ipNet)
+			set[ipNet.String()] = struct{}{}
+		}
+	}
+	return nets
+}

render/theinternet_test.go

@@ -1,6 +1,7 @@
 package render_test
 
 import (
+	"fmt"
 	"net"
 	"reflect"
 	"testing"
@@ -32,6 +33,30 @@ func TestReportLocalNetworks(t *testing.T) {
 	}
 }
 
+func TestParseNetworks(t *testing.T) {
+	var (
+		hugenetStr  = "1.0.0.0/8"
+		bignetStr   = "10.1.0.1/16"
+		smallnetStr = "5.6.7.8/32"
+		hugenet     = mustParseCIDR(hugenetStr)
+		bignet      = mustParseCIDR(bignetStr)
+		smallnet    = mustParseCIDR(smallnetStr)
+	)
+	for _, tc := range []struct {
+		input string
+		want  report.Networks
+	}{
+		{"", report.Networks{}},
+		{fmt.Sprintf("%s", bignetStr), report.Networks([]*net.IPNet{bignet})},
+		{fmt.Sprintf("%s %s", bignetStr, bignetStr), report.Networks([]*net.IPNet{bignet})},
+		{fmt.Sprintf("%s foo %s oops  %s", hugenetStr, smallnetStr, hugenetStr), report.Networks([]*net.IPNet{hugenet, smallnet})},
+	} {
+		if want, have := tc.want, render.ParseNetworks(tc.input); !reflect.DeepEqual(want, have) {
+			t.Error(test.Diff(want, have))
+		}
+	}
+}
+
 func mustParseCIDR(s string) *net.IPNet {
 	_, ipNet, err := net.ParseCIDR(s)
 	if err != nil {