diff --git a/.gitignore b/.gitignore index 830caffec..99696caaf 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ experimental/genreport/genreport experimental/graphviz/graphviz experimental/oneshot/oneshot experimental/_integration/_integration +experimental/dsl/dsl *sublime-project *sublime-workspace *npm-debug.log diff --git a/experimental/dsl/applications-by-name.txt b/experimental/dsl/applications-by-name.txt new file mode 100644 index 000000000..b3a73a71c --- /dev/null +++ b/experimental/dsl/applications-by-name.txt @@ -0,0 +1,5 @@ +ALL GROUPBY {{pid}} +ALL GROUPBY {{comm}} +NOT WITH {{comm}} REMOVE +NOT CONNECTED REMOVE + diff --git a/experimental/dsl/applications.txt b/experimental/dsl/applications.txt new file mode 100644 index 000000000..efeaff18e --- /dev/null +++ b/experimental/dsl/applications.txt @@ -0,0 +1,3 @@ +ALL GROUPBY {{pid}} +NOT WITH {{pid}} REMOVE +NOT CONNECTED REMOVE diff --git a/experimental/dsl/containers-by-image.txt b/experimental/dsl/containers-by-image.txt new file mode 100644 index 000000000..f8972dc16 --- /dev/null +++ b/experimental/dsl/containers-by-image.txt @@ -0,0 +1,4 @@ +ALL GROUPBY {{pid}} +ALL GROUPBY {{docker_container_id}} +ALL GROUPBY {{docker_image_id}} +NOT WITH {{docker_image_name}} REMOVE diff --git a/experimental/dsl/containers.txt b/experimental/dsl/containers.txt new file mode 100644 index 000000000..654b5430d --- /dev/null +++ b/experimental/dsl/containers.txt @@ -0,0 +1,3 @@ +ALL GROUPBY {{pid}} +ALL GROUPBY {{docker_container_id}} +NOT WITH {{docker_container_id}} REMOVE diff --git a/experimental/dsl/expression.go b/experimental/dsl/expression.go new file mode 100644 index 000000000..323bbd68a --- /dev/null +++ b/experimental/dsl/expression.go @@ -0,0 +1,356 @@ +package main + +import ( + "fmt" + "log" + "math/rand" + "net" + "regexp" + "strings" + + "github.com/weaveworks/scope/probe/endpoint" + "github.com/weaveworks/scope/probe/host" + "github.com/weaveworks/scope/render" + "github.com/weaveworks/scope/report" +) + +type view []expression + +func (v view) eval(tpy report.Topology) report.Topology { + for _, expr := range v { + tpy = expr.eval(tpy) + } + return tpy +} + +type expression struct { + selector + transformer +} + +func (e expression) eval(tpy report.Topology) report.Topology { + if e.transformer == nil { + e.transformer = transformHighlight + } + return e.transformer(tpy, e.selector(tpy)) +} + +type selector func(report.Topology) []string + +type transformer func(report.Topology, []string) report.Topology + +func selectAll(tpy report.Topology) []string { + out := make([]string, 0, len(tpy.NodeMetadatas)) + for id := range tpy.NodeMetadatas { + out = append(out, id) + } + log.Printf("select ALL: %d", len(out)) + return out +} + +func selectConnected(tpy report.Topology) []string { + degree := map[string]int{} + for src, dsts := range tpy.Adjacency { + a, ok := report.ParseAdjacencyID(src) + if !ok { + panic(src) + } + degree[a] += len(dsts) + for _, dst := range dsts { + degree[dst]++ + } + } + out := []string{} + for id := range tpy.NodeMetadatas { + if degree[id] > 0 { + out = append(out, id) + } + } + log.Printf("select CONNECTED: %d", len(out)) + return out +} + +func selectNonlocal(tpy report.Topology) []string { + local := report.Networks{} + for _, md := range tpy.NodeMetadatas { + for k, v := range md.Metadata { + if k == host.LocalNetworks { + local = append(local, render.ParseNetworks(v)...) + } + } + } + out := []string{} + for id, md := range tpy.NodeMetadatas { + if addr, ok := md.Metadata[endpoint.Addr]; ok { + if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) { + out = append(out, id) // valid addr metadata key, nonlocal + continue + } + } + if _, addr, ok := report.ParseAddressNodeID(id); ok { + if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) { + out = append(out, id) // valid address node ID, nonlocal + continue + } + } + if _, addr, _, ok := report.ParseEndpointNodeID(id); ok { + if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) { + out = append(out, id) // valid endpoint node ID, nonlocal + continue + } + } + } + log.Printf("select NONLOCAL: %d", len(out)) + return out +} + +func selectLike(s string) selector { + re, err := regexp.Compile(s) + if err != nil { + log.Printf("select LIKE %q: %v", s, err) + re = regexp.MustCompile("") + } + return func(tpy report.Topology) []string { + out := []string{} + for id := range tpy.NodeMetadatas { + if re.MatchString(id) { + out = append(out, id) + } + } + log.Printf("select LIKE %q: %d", s, len(out)) + return out + } +} + +func selectWith(s string) selector { + var k, v string + if fields := strings.SplitN(s, "=", 2); len(fields) == 1 { + k = strings.TrimSpace(fields[0]) + } else if len(fields) == 2 { + k, v = strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1]) + } + + return func(tpy report.Topology) []string { + out := []string{} + for id, md := range tpy.NodeMetadatas { + if vv, ok := md.Metadata[k]; ok { + if v == "" || (v != "" && v == vv) { + out = append(out, id) + } + } + } + log.Printf("select WITH %q: %d", s, len(out)) + return out + } +} + +func selectNot(s selector) selector { + return func(tpy report.Topology) []string { + set := map[string]struct{}{} + for _, id := range s(tpy) { + set[id] = struct{}{} + } + out := []string{} + for id := range tpy.NodeMetadatas { + if _, ok := set[id]; ok { + continue // selected by that one -> not by this one + } + out = append(out, id) + } + log.Printf("select NOT: %d", len(out)) + return out + } +} + +const highlightKey = "_highlight" + +func transformHighlight(tpy report.Topology, ids []string) report.Topology { + for _, id := range ids { + tpy.NodeMetadatas[id] = tpy.NodeMetadatas[id].Merge(report.MakeNodeMetadataWith(map[string]string{highlightKey: "true"})) + } + log.Printf("transform HIGHLIGHT %d: OK", len(ids)) + return tpy +} + +func transformRemove(tpy report.Topology, ids []string) report.Topology { + toRemove := map[string]struct{}{} + for _, id := range ids { + toRemove[id] = struct{}{} + } + out := report.MakeTopology() + for id := range tpy.NodeMetadatas { + if _, ok := toRemove[id]; ok { + continue + } + cp(out, tpy, id) + } + log.Printf("transform REMOVE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + return out +} + +func transformShowOnly(tpy report.Topology, ids []string) report.Topology { + out := report.MakeTopology() + for _, id := range ids { + if _, ok := tpy.NodeMetadatas[id]; !ok { + continue + } + cp(out, tpy, id) + } + log.Printf("transform SHOWONLY %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + return out +} + +func transformMerge(tpy report.Topology, ids []string) report.Topology { + name := fmt.Sprintf("%x", rand.Int31()) + mapped := map[string]string{} + for _, id := range ids { + mapped[id] = name + } + out := report.MakeTopology() + for id := range tpy.NodeMetadatas { + if dstID, ok := mapped[id]; ok { + merge(out, dstID, tpy, id, mapped) + } else { + cp(out, tpy, id) + } + } + log.Printf("transform MERGE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + return out +} + +func transformGroupBy(s string) transformer { + keys := []string{} + for _, key := range strings.Split(s, ",") { + keys = append(keys, strings.TrimSpace(key)) + } + + return func(tpy report.Topology, ids []string) report.Topology { + set := map[string]struct{}{} + for _, id := range ids { + set[id] = struct{}{} + } + + // Identify all nodes that should be grouped. + mapped := map[string]string{} // src ID: dst ID + for id, md := range tpy.NodeMetadatas { + if _, ok := set[id]; !ok { + continue // not selected + } + + parts := []string{} + for _, key := range keys { + if val, ok := md.Metadata[key]; ok { + parts = append(parts, fmt.Sprintf("%s-%s", key, val)) + } + } + if len(parts) < len(keys) { + continue // didn't match all required keys + } + + dstID := strings.Join(parts, "-") + mapped[id] = dstID + } + + // Walk nodes again, merging those that should be grouped. + out := report.MakeTopology() + for id := range tpy.NodeMetadatas { + if dstID, ok := mapped[id]; ok { + merge(out, dstID, tpy, id, mapped) + } else { + cp(out, tpy, id) + } + } + + log.Printf("transform GROUPBY %v %d: in %d, out %d", keys, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + return out + } +} + +func transformJoin(key string) transformer { + return func(tpy report.Topology, ids []string) report.Topology { + // key is e.g. host_node_id. + // Collect the set of represented values. + values := map[string]report.NodeMetadata{} + for _, md := range tpy.NodeMetadatas { + for k, v := range md.Metadata { + if k == key { + values[v] = report.MakeNodeMetadata() // gather later + } + } + } + + // Next, gather the metadata from nodes in the set. + for id, md := range tpy.NodeMetadatas { + if found, ok := values[id]; ok { + values[id] = found.Merge(md) // gather + } + } + + // Finally, join that metadata to referential nodes. + // And delete the referenced nodes. + out := report.MakeTopology() + for id, md := range tpy.NodeMetadatas { + if _, ok := values[id]; ok { + continue // delete + } + cp(out, tpy, id) // copy node + for k, v := range md.Metadata { + if k == key { + md = md.Merge(values[v]) // join metadata + } + } + out.NodeMetadatas[id] = md // write + } + + log.Printf("transform JOIN %v %d: in %d, out %d", key, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + return out + } +} + +func cp(dst report.Topology, src report.Topology, id string) { + adjacencyID := report.MakeAdjacencyID(id) + dst.Adjacency[adjacencyID] = src.Adjacency[adjacencyID] + + for _, otherID := range dst.Adjacency[id] { + edgeID := report.MakeEdgeID(id, otherID) + dst.EdgeMetadatas[edgeID] = src.EdgeMetadatas[edgeID] + } + + dst.NodeMetadatas[id] = src.NodeMetadatas[id] +} + +func merge(dst report.Topology, dstID string, src report.Topology, srcID string, mapped map[string]string) { + // We gonna take srcID from src and merge it into dstID in dst. That's + // like renaming the node, so we gotta update the adjacency lists. Both + // outgoing *and* incoming links! + dstAdjacencyID := report.MakeAdjacencyID(dstID) + srcAdjacencyID := report.MakeAdjacencyID(srcID) + + // Merge the src's adjacency list into the dst topology. + dst.Adjacency[dstAdjacencyID] = dst.Adjacency[dstAdjacencyID].Merge(src.Adjacency[srcAdjacencyID]) + + // Update any dst adjacencies from the old ID to the new ID. + for existingSrcAdjacencyID, existingDstIDs := range dst.Adjacency { + for i, existingDstID := range existingDstIDs { + if newDstID, ok := mapped[existingDstID]; ok { + existingDstIDs[i] = newDstID + } + } + dst.Adjacency[existingSrcAdjacencyID] = existingDstIDs + } + + // Update the EdgeMetadatas to have the new IDs. + for _, otherID := range src.Adjacency[srcAdjacencyID] { + oldEdgeID := report.MakeEdgeID(srcID, otherID) + newEdgeID := report.MakeEdgeID(dstID, otherID) + dst.EdgeMetadatas[newEdgeID] = dst.EdgeMetadatas[newEdgeID].Merge(src.EdgeMetadatas[oldEdgeID]) + } + + // Merge the src node metadata into the dst node metadata. + md, ok := dst.NodeMetadatas[dstID] + if !ok { + md = report.MakeNodeMetadata() + } + md = md.Merge(src.NodeMetadatas[srcID]) + dst.NodeMetadatas[dstID] = md +} diff --git a/experimental/dsl/exprs.txt b/experimental/dsl/exprs.txt new file mode 100644 index 000000000..3b19fa2ce --- /dev/null +++ b/experimental/dsl/exprs.txt @@ -0,0 +1,3 @@ +ALL GROUPBY {{ pid }} +ALL GROUPBY {{ docker_container_id }} +NOT WITH {{ docker_container_id }} REMOVE \ No newline at end of file diff --git a/experimental/dsl/graphviz.go b/experimental/dsl/graphviz.go new file mode 100644 index 000000000..9f5ea2155 --- /dev/null +++ b/experimental/dsl/graphviz.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "io" + "sort" + + "github.com/weaveworks/scope/probe/host" + + "github.com/weaveworks/scope/report" +) + +func dot(w io.Writer, tpy report.Topology) { + var nodes []string + for id, md := range tpy.NodeMetadatas { + label := id + "\n" + + var lines []string + var highlight bool + for k, v := range md.Metadata { + if k == host.LocalNetworks { + continue + } + if k == highlightKey { + highlight = true + continue + } + lines = append(lines, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(lines) + for _, line := range lines { + label += "\n" + line + } + + fillcolor := "white" + if highlight { + fillcolor = "yellow" + } + + nodes = append(nodes, fmt.Sprintf("%q [label=%q fillcolor=%s];", id, label, fillcolor)) + } + + var edges []string + for src, dsts := range tpy.Adjacency { + a, ok := report.ParseAdjacencyID(src) + if !ok { + panic(src) + } + for _, dst := range dsts { + edges = append(edges, fmt.Sprintf("%q -> %q;", a, dst)) + } + } + + sort.Strings(nodes) + sort.Strings(edges) + + fmt.Fprintf(w, "digraph G {\n") + fmt.Fprintf(w, "\tgraph [ overlap=false, mode=hier ];\n") + fmt.Fprintf(w, "\tnode [ shape=rect, style=filled ];\n") + fmt.Fprintf(w, "\toutputorder=edgesfirst;\n") + fmt.Fprintf(w, "\n") + for _, line := range nodes { + fmt.Fprintf(w, "\t%s\n", line) + } + fmt.Fprintf(w, "\n") + for _, line := range edges { + fmt.Fprintf(w, "\t%s\n", line) + } + fmt.Fprintf(w, "}\n") +} diff --git a/experimental/dsl/handle.go b/experimental/dsl/handle.go new file mode 100644 index 000000000..4efa83a24 --- /dev/null +++ b/experimental/dsl/handle.go @@ -0,0 +1,100 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "log" + "net/http" + "os/exec" + + "github.com/weaveworks/scope/report" +) + +func handleJSON(tpy report.Topology) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(getView(r).eval(tpy.Copy()).NodeMetadatas); err != nil { + log.Print(err) + return + } + } +} + +func handleDOT(tpy report.Topology) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + dot(w, getView(r).eval(tpy.Copy())) + } +} + +func handleSVG(tpy report.Topology) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + cmd := exec.Command(engine(r), "-Tsvg") + + wc, err := cmd.StdinPipe() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + cmd.Stdout = w + + dot(wc, getView(r).eval(tpy.Copy())) + wc.Close() + + w.Header().Set("Content-Type", "image/svg+xml") + if err := cmd.Run(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +func handleHTML(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprintf(w, "\n") + //fmt.Fprintf(w, ``+"\n") + fmt.Fprintf(w, "\n") + fmt.Fprintf(w, `
`+"\n", r.URL.Query().Encode()) + fmt.Fprintf(w, "\n") +} + +func getView(r *http.Request) view { + var expressions []expression + expressions = append(expressions, getExpressionsFromBody(r)...) + expressions = append(expressions, getExpressionsFromQuery(r)...) + log.Printf("Serving view with %d expression(s)", len(expressions)) + return view(expressions) +} + +func getExpressionsFromQuery(r *http.Request) []expression { + strs := []string{} + if err := r.ParseForm(); err != nil { + log.Printf("Get expressions from query: %v", err) + return []expression{} + } + for _, str := range r.Form["expr"] { + log.Printf("Query expression: %s", str) + strs = append(strs, str) + } + return parseExpressions(strs) +} + +func getExpressionsFromBody(r *http.Request) []expression { + strs := []string{} + scanner := bufio.NewScanner(r.Body) + for scanner.Scan() { + log.Printf("Body expression: %s", scanner.Text()) + strs = append(strs, scanner.Text()) + } + return parseExpressions(strs) +} + +func engine(r *http.Request) string { + engine := r.FormValue("engine") + if engine == "" { + engine = "dot" + } + return engine +} diff --git a/experimental/dsl/lexer.go b/experimental/dsl/lexer.go new file mode 100644 index 000000000..cec642031 --- /dev/null +++ b/experimental/dsl/lexer.go @@ -0,0 +1,326 @@ +package main + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +// Expression = [NOT] Selector [Transformer] +// Selector = ALL / CONNECTED / NONLOCAL / LIKE {{ }} / WITH {{ [= ] }} +// Transformer = HIGHLIGHT / REMOVE / SHOWONLY / MERGE / GROUPBY {{ , ... }} // JOIN {{ key }} + +type lexer struct { + input string // string being scanned + start int // start position of this item + pos int // current position within the input + width int // width of last rune read + items chan item +} + +func lex(input string) (*lexer, <-chan item) { + l := &lexer{ + input: input, + items: make(chan item), + } + go l.run() + return l, l.items +} + +const ( + keywordNot = "NOT" + keywordAll = "ALL" + keywordConnected = "CONNECTED" + keywordNonlocal = "NONLOCAL" + keywordLike = "LIKE" + keywordWith = "WITH" + keywordHighlight = "HIGHLIGHT" + keywordRemove = "REMOVE" + keywordShowOnly = "SHOWONLY" + keywordMerge = "MERGE" + keywordGroupBy = "GROUPBY" + keywordJoin = "JOIN" +) + +type itemType int + +const ( + itemError itemType = iota + itemNot + itemAll + itemConnected + itemNonlocal + itemLike + itemWith + itemHighlight + itemRemove + itemShowOnly + itemMerge + itemGroupBy + itemJoin + itemRegex + itemKeyValue + itemKeyList + itemKey +) + +func (t itemType) String() string { + switch t { + case itemError: + return "ERROR" + case itemNot: + return keywordNot + case itemAll: + return keywordAll + case itemConnected: + return keywordConnected + case itemNonlocal: + return keywordNonlocal + case itemLike: + return keywordLike + case itemWith: + return keywordWith + case itemHighlight: + return keywordHighlight + case itemRemove: + return keywordRemove + case itemShowOnly: + return keywordShowOnly + case itemMerge: + return keywordMerge + case itemGroupBy: + return keywordGroupBy + case itemJoin: + return keywordJoin + case itemRegex: + return "" + case itemKeyValue: + return "" + case itemKeyList: + return "" + case itemKey: + return "" + default: + return "unknown" + } +} + +type stateFn func(*lexer) stateFn + +func (l *lexer) run() { + for state := lexExpression; state != nil; { + state = state(l) + } + close(l.items) +} + +func (l *lexer) emit(t itemType) { + l.items <- item{t, l.input[l.start:l.pos]} + l.start = l.pos +} + +const eof rune = -1 + +func (l *lexer) next() (r rune) { + if l.pos >= len(l.input) { + l.width = 0 + return eof + } + r, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) + l.pos += l.width + return r +} + +func (l *lexer) backup() { l.pos -= l.width } + +// acceptRun consumes a run of runes from the valid set. +func (l *lexer) acceptRun(validSet string) { + for strings.IndexRune(validSet, l.next()) >= 0 { + // consume + } + l.backup() +} + +func (l *lexer) eatWhitespace() { + l.acceptRun(" \t\r\n") +} + +// errorf terminates lexing with an error. +func (l *lexer) errorf(format string, args ...interface{}) stateFn { + l.items <- item{itemError, fmt.Sprintf(format, args...)} + return nil +} + +type item struct { + itemType itemType + literal string +} + +func (i item) String() string { + return fmt.Sprintf("%s %q", i.itemType, i.literal) +} + +func lexExpression(l *lexer) stateFn { + l.eatWhitespace() + if strings.HasPrefix(l.input[l.pos:], keywordNot) { + return lexNot + } + return lexSelector +} + +func lexNot(l *lexer) stateFn { + l.pos += len(keywordNot) + l.emit(itemNot) + return lexSelector +} + +func lexSelector(l *lexer) stateFn { + l.eatWhitespace() + switch { + case strings.HasPrefix(l.input[l.pos:], keywordAll): + return lexAll + case strings.HasPrefix(l.input[l.pos:], keywordConnected): + return lexConnected + case strings.HasPrefix(l.input[l.pos:], keywordNonlocal): + return lexNonlocal + case strings.HasPrefix(l.input[l.pos:], keywordLike): + return lexLike + case strings.HasPrefix(l.input[l.pos:], keywordWith): + return lexWith + default: + return l.errorf("bad selector") + } +} + +func lexAll(l *lexer) stateFn { + l.pos += len(keywordAll) + l.emit(itemAll) + return lexTransformer +} + +func lexConnected(l *lexer) stateFn { + l.pos += len(keywordConnected) + l.emit(itemConnected) + return lexTransformer +} + +func lexNonlocal(l *lexer) stateFn { + l.pos += len(keywordNonlocal) + l.emit(itemNonlocal) + return lexTransformer +} + +func lexLike(l *lexer) stateFn { + l.pos += len(keywordLike) + l.emit(itemLike) + return lexRegex +} + +func lexWith(l *lexer) stateFn { + l.pos += len(keywordWith) + l.emit(itemWith) + return lexKeyValue +} + +func lexRegex(l *lexer) stateFn { + return lexMeta("regex", itemRegex, lexTransformer) +} + +func lexKeyValue(l *lexer) stateFn { + return lexMeta("key=value", itemKeyValue, lexTransformer) +} + +func lexTransformer(l *lexer) stateFn { + l.eatWhitespace() + switch { + case l.pos == len(l.input): + return nil // done + case strings.HasPrefix(l.input[l.pos:], keywordHighlight): + return lexHighlight + case strings.HasPrefix(l.input[l.pos:], keywordRemove): + return lexRemove + case strings.HasPrefix(l.input[l.pos:], keywordShowOnly): + return lexShowOnly + case strings.HasPrefix(l.input[l.pos:], keywordMerge): + return lexMerge + case strings.HasPrefix(l.input[l.pos:], keywordGroupBy): + return lexGroupBy + case strings.HasPrefix(l.input[l.pos:], keywordJoin): + return lexJoin + default: + return l.errorf("bad transformer at position %d: %s", l.pos, l.input[l.pos:]) + } +} + +func lexHighlight(l *lexer) stateFn { + l.pos += len(keywordHighlight) + l.emit(itemHighlight) + return nil +} + +func lexRemove(l *lexer) stateFn { + l.pos += len(keywordRemove) + l.emit(itemRemove) + return nil +} + +func lexShowOnly(l *lexer) stateFn { + l.pos += len(keywordShowOnly) + l.emit(itemShowOnly) + return nil +} + +func lexMerge(l *lexer) stateFn { + l.pos += len(keywordMerge) + l.emit(itemMerge) + return nil +} + +func lexGroupBy(l *lexer) stateFn { + l.pos += len(keywordGroupBy) + l.emit(itemGroupBy) + return lexKeyList +} + +func lexJoin(l *lexer) stateFn { + l.pos += len(keywordJoin) + l.emit(itemJoin) + return lexKey +} + +func lexKeyList(l *lexer) stateFn { + return lexMeta("key list", itemKeyList, nil) +} + +func lexKey(l *lexer) stateFn { + return lexMeta("key", itemKey, nil) +} + +const ( + leftMeta = "{{" + rightMeta = "}}" +) + +func lexMeta(what string, t itemType, next stateFn) stateFn { + return func(l *lexer) stateFn { + l.eatWhitespace() + if !strings.HasPrefix(l.input[l.pos:], leftMeta) { + return l.errorf("%s must begin with %s", what, leftMeta) + } + l.pos += len(leftMeta) + l.start = l.pos + for { + if l.pos > len(l.input) { + return l.errorf("%s must end with %s", what, rightMeta) + } + if strings.HasPrefix(l.input[l.pos:], rightMeta) { + break + } + l.pos++ + } + l.emit(t) + l.pos += len(rightMeta) + l.start = l.pos + return next + } +} diff --git a/experimental/dsl/main.go b/experimental/dsl/main.go new file mode 100644 index 000000000..3164e7767 --- /dev/null +++ b/experimental/dsl/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "os" + "strings" + + "github.com/weaveworks/scope/report" +) + +func main() { + var ( + listen = flag.String("listen", ":8080", "HTTP listen address") + ) + flag.Parse() + + // Decode report + var rpt report.Report + if err := json.NewDecoder(os.Stdin).Decode(&rpt); err != nil { + log.Fatal(err) + } + + // Flatten all the topologies + tpy := report.MakeTopology() + for _, t := range rpt.Topologies() { + if err := conflict(tpy, t); err != nil { + log.Fatal(err) + } + tpy = tpy.Merge(t) + } + log.Printf("topology with %d node(s)", len(tpy.NodeMetadatas)) + + // Serve HTTP + http.HandleFunc("/json", handleJSON(tpy)) + http.HandleFunc("/dot", handleDOT(tpy)) + http.HandleFunc("/svg", handleSVG(tpy)) + http.HandleFunc("/", handleHTML) + log.Printf("listening on %s", *listen) + log.Fatal(http.ListenAndServe(*listen, nil)) +} + +// Just a safety check. +func conflict(a, b report.Topology) error { + var errs []string + for id := range a.NodeMetadatas { + if _, ok := b.NodeMetadatas[id]; ok { + errs = append(errs, id) + } + } + for id := range b.NodeMetadatas { + if _, ok := a.NodeMetadatas[id]; ok { + errs = append(errs, id) + } + } + if len(errs) > 0 { + return fmt.Errorf(strings.Join(errs, ", ")) + } + return nil +} diff --git a/experimental/dsl/medium.json b/experimental/dsl/medium.json new file mode 100644 index 000000000..8d64bf4d3 --- /dev/null +++ b/experimental/dsl/medium.json @@ -0,0 +1,82 @@ +{ + "Process": { + "NodeMetadatas": { + "nydev.bourgon.org;1295": { + "Metadata": { + "cmdline": "/app db1 db2 db3 ", + "comm": "app", + "docker_container_id": "0952127d5294bf20591d90fc9dc8f15d684c1a140e50ac92e8807293faeb6c5b", + "host_node_id": "nydev.bourgon.org;", + "pid": "1295", + "ppid": "5082", + "threads": "7", + "topology": "process" + } + } + } + }, + "Endpoint": { + "Adjacency": { + ">nydev.bourgon.org;172.17.0.56;45890": [ + "nydev.bourgon.org;172.17.0.54;9000" + ], + ">nydev.bourgon.org;172.17.0.56;49569": [ + "nydev.bourgon.org;172.17.0.50;9000" + ], + ">nydev.bourgon.org;172.17.0.56;60705": [ + "nydev.bourgon.org;172.17.0.52;9000" + ], + ">nydev.bourgon.org;172.17.0.56;8080": [ + "nydev.bourgon.org;172.17.0.60;49914", + "nydev.bourgon.org;172.17.0.62;42329" + ] + }, + "NodeMetadatas": { + "nydev.bourgon.org;172.17.0.56;45890": { + "Metadata": { + "addr": "172.17.0.56", + "host_node_id": "nydev.bourgon.org;", + "pid": "1295", + "port": "45890", + "topology": "endpoint" + } + }, + "nydev.bourgon.org;172.17.0.56;49569": { + "Metadata": { + "addr": "172.17.0.56", + "host_node_id": "nydev.bourgon.org;", + "pid": "1295", + "port": "49569", + "topology": "endpoint" + } + }, + "nydev.bourgon.org;172.17.0.56;60705": { + "Metadata": { + "addr": "172.17.0.56", + "host_node_id": "nydev.bourgon.org;", + "pid": "1295", + "port": "60705", + "topology": "endpoint" + } + }, + "nydev.bourgon.org;172.17.0.56;8080": { + "Metadata": { + "addr": "172.17.0.56", + "host_node_id": "nydev.bourgon.org;", + "pid": "1295", + "port": "8080", + "topology": "endpoint" + } + }, + "nydev.bourgon.org;172.17.0.54;9000": { + "Metadata": { + "addr": "172.17.0.54", + "host_node_id": "nydev.bourgon.org;", + "pid": "1212", + "port": "9000", + "topology": "endpoint" + } + } + } + } +} diff --git a/experimental/dsl/parser.go b/experimental/dsl/parser.go new file mode 100644 index 000000000..29db899e6 --- /dev/null +++ b/experimental/dsl/parser.go @@ -0,0 +1,98 @@ +package main + +import ( + "fmt" + "log" +) + +func parseExpressions(strs []string) []expression { + var exprs []expression + for _, str := range strs { + expr, err := parseExpression(str) + if err != nil { + log.Printf("%s: %v", str, err) + continue + } + log.Printf("%s: OK", str) + exprs = append(exprs, expr) + } + return exprs +} + +func parseExpression(str string) (expression, error) { + var ( + expr expression + not bool + ) + _, c := lex(str) + for item := range c { + switch item.itemType { + case itemNot: + not = !not + + case itemAll: + expr.selector = selectAll + + case itemConnected: + expr.selector = selectConnected + + case itemNonlocal: + expr.selector = selectNonlocal + + case itemLike: + item = <-c + switch item.itemType { + case itemRegex: + expr.selector = selectLike(item.literal) + default: + return expression{}, fmt.Errorf("bad LIKE: want %s, got %s", itemRegex, item.itemType) + } + + case itemWith: + item = <-c + switch item.itemType { + case itemKeyValue: + expr.selector = selectWith(item.literal) + default: + return expression{}, fmt.Errorf("bad WITH: want %s, got %s", itemKeyValue, item.itemType) + } + + case itemHighlight: + expr.transformer = transformHighlight + + case itemRemove: + expr.transformer = transformRemove + + case itemShowOnly: + expr.transformer = transformShowOnly + + case itemMerge: + expr.transformer = transformMerge + + case itemGroupBy: + item = <-c + switch item.itemType { + case itemKeyList: + expr.transformer = transformGroupBy(item.literal) + default: + return expression{}, fmt.Errorf("bad GROUPBY: want %s, got %s", itemKeyList, item.itemType) + } + + case itemJoin: + item = <-c + switch item.itemType { + case itemKey: + expr.transformer = transformJoin(item.literal) + default: + return expression{}, fmt.Errorf("bad JOIN: want %s, got %s", itemKey, item.itemType) + } + + default: + return expression{}, fmt.Errorf("%s: %s", str, item.literal) + } + } + if not { + expr.selector = selectNot(expr.selector) + } + return expr, nil +} diff --git a/experimental/dsl/small.json b/experimental/dsl/small.json new file mode 100644 index 000000000..de619aa86 --- /dev/null +++ b/experimental/dsl/small.json @@ -0,0 +1,60 @@ +{ + "Endpoint": { + "Adjacency": { + ">endpoint-1": [ + "endpoint-2", + "endpoint-3" + ] + }, + "EdgeMetadatas": {}, + "NodeMetadatas": { + "endpoint-1": { + "Metadata": { + "endpoint_statistic": "one", + "pid": "1234", + "host_node_id": "nydev.bourgon.org;" + } + }, + "endpoint-2": { + "Metadata": { + "endpoint_statistic": "two", + "host_node_id": "nydev.bourgon.org;" + } + }, + "endpoint-3": { + "Metadata": { + "endpoint_statistic": "three", + "host_node_id": "nydev.bourgon.org;" + } + } + } + }, + "Process": { + "Adjacency": {}, + "EdgeMetadatas": {}, + "NodeMetadatas": { + "process-1234": { + "Metadata": { + "pid": "1234", + "docker_container_id": "abc", + "process_statistic": "555", + "host_node_id": "nydev.bourgon.org;" + } + } + } + }, + "Container": { + "Adjacency": {}, + "EdgeMetadatas": {}, + "NodeMetadatas": { + "abc": { + "Metadata": { + "docker_container_id": "abc", + "docker_statistic": "999", + "host_node_id": "nydev.bourgon.org;" + } + } + } + }, + "Window": 60000000000 +} diff --git a/render/theinternet.go b/render/theinternet.go index 07d8a5545..81e0dd867 100644 --- a/render/theinternet.go +++ b/render/theinternet.go @@ -13,27 +13,33 @@ import ( // used to determine which nodes in the report are "remote", i.e. outside of // our infrastructure. func LocalNetworks(r report.Report) report.Networks { - var ( - result = report.Networks{} - networks = map[string]struct{}{} - ) - + result := report.Networks{} for _, md := range r.Host.NodeMetadatas { val, ok := md.Metadata[host.LocalNetworks] if !ok { continue } - for _, s := range strings.Fields(val) { - _, ipNet, err := net.ParseCIDR(s) - if err != nil { - continue - } - _, ok := networks[ipNet.String()] - if !ok { - result = append(result, ipNet) - networks[ipNet.String()] = struct{}{} - } - } + result = append(result, ParseNetworks(val)...) } return result } + +// ParseNetworks converts a string of space-separated CIDRs to a +// report.Networks. +func ParseNetworks(v string) report.Networks { + var ( + nets = report.Networks{} + set = map[string]struct{}{} + ) + for _, s := range strings.Fields(v) { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + continue + } + if _, ok := set[ipNet.String()]; !ok { + nets = append(nets, ipNet) + set[ipNet.String()] = struct{}{} + } + } + return nets +} diff --git a/render/theinternet_test.go b/render/theinternet_test.go index fad04b5c3..9c90314fb 100644 --- a/render/theinternet_test.go +++ b/render/theinternet_test.go @@ -1,6 +1,7 @@ package render_test import ( + "fmt" "net" "reflect" "testing" @@ -32,6 +33,30 @@ func TestReportLocalNetworks(t *testing.T) { } } +func TestParseNetworks(t *testing.T) { + var ( + hugenetStr = "1.0.0.0/8" + bignetStr = "10.1.0.1/16" + smallnetStr = "5.6.7.8/32" + hugenet = mustParseCIDR(hugenetStr) + bignet = mustParseCIDR(bignetStr) + smallnet = mustParseCIDR(smallnetStr) + ) + for _, tc := range []struct { + input string + want report.Networks + }{ + {"", report.Networks{}}, + {fmt.Sprintf("%s", bignetStr), report.Networks([]*net.IPNet{bignet})}, + {fmt.Sprintf("%s %s", bignetStr, bignetStr), report.Networks([]*net.IPNet{bignet})}, + {fmt.Sprintf("%s foo %s oops %s", hugenetStr, smallnetStr, hugenetStr), report.Networks([]*net.IPNet{hugenet, smallnet})}, + } { + if want, have := tc.want, render.ParseNetworks(tc.input); !reflect.DeepEqual(want, have) { + t.Error(test.Diff(want, have)) + } + } +} + func mustParseCIDR(s string) *net.IPNet { _, ipNet, err := net.ParseCIDR(s) if err != nil {