From a94731c59bce2529d4c03af5a0f825de4ed4913e Mon Sep 17 00:00:00 2001 From: Peter Bourgon Date: Wed, 9 Sep 2015 12:59:40 +0200 Subject: [PATCH] First cut: expression engine to RenderableNodes --- render/dsl/expression.go | 405 +++++++++++++++++++++++++ render/dsl/expression_internal_test.go | 245 +++++++++++++++ render/dsl/lexer.go | 326 ++++++++++++++++++++ render/dsl/lexer_internal_test.go | 76 +++++ render/dsl/parser.go | 104 +++++++ render/dsl/parser_internal_test.go | 68 +++++ 6 files changed, 1224 insertions(+) create mode 100644 render/dsl/expression.go create mode 100644 render/dsl/expression_internal_test.go create mode 100644 render/dsl/lexer.go create mode 100644 render/dsl/lexer_internal_test.go create mode 100644 render/dsl/parser.go create mode 100644 render/dsl/parser_internal_test.go diff --git a/render/dsl/expression.go b/render/dsl/expression.go new file mode 100644 index 000000000..f41b46d79 --- /dev/null +++ b/render/dsl/expression.go @@ -0,0 +1,405 @@ +package dsl + +import ( + "fmt" + "net" + "regexp" + "strings" + + "github.com/weaveworks/scope/probe/endpoint" + "github.com/weaveworks/scope/probe/host" + "github.com/weaveworks/scope/render" + "github.com/weaveworks/scope/report" +) + +// Evaluator describes a monadic transformer of a RenderableNodes. +type Evaluator interface { + Eval(render.RenderableNodes) render.RenderableNodes +} + +// Expression is a single evaluator. +type Expression struct { + selector + transformer +} + +// Eval implements Evaluator. +func (e Expression) Eval(rns render.RenderableNodes) render.RenderableNodes { + return e.transformer(rns, e.selector(rns)) +} + +// Expressions is an ordered collection of expressions. +type Expressions []Expression + +// Eval implements Evaluator. +func (e Expressions) Eval(rns render.RenderableNodes) render.RenderableNodes { + for _, expr := range e { + rns = expr.Eval(rns) + } + return rns +} + +type selector func(render.RenderableNodes) []string + +type transformer func(render.RenderableNodes, []string) render.RenderableNodes + +func selectAll(rns render.RenderableNodes) []string { + out := make([]string, 0, len(rns)) + for id := range rns { + out = append(out, id) + } + //log.Printf("select ALL: %d", len(out)) + return out +} + +func selectConnected(rns render.RenderableNodes) []string { + degree := map[string]int{} + for src, rn := range rns { + degree[src] += len(rn.Adjacency) + for _, dst := range rn.Adjacency { + degree[dst]++ + } + } + out := []string{} + for id := range rns { + if degree[id] > 0 { + out = append(out, id) + } + } + return out +} + +func selectNonlocal(rns render.RenderableNodes) []string { + local := report.Networks{} + for _, rn := range rns { + for k, v := range rn.Metadata { + if k == host.LocalNetworks { + local = append(local, report.ParseNetworks(v)...) + } + } + } + out := []string{} + for id, rn := range rns { + if addr, ok := rn.Metadata[endpoint.Addr]; ok { + if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) { + out = append(out, id) // valid addr metadata key, nonlocal + continue + } + } + if _, addr, ok := report.ParseAddressNodeID(id); ok { + if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) { + out = append(out, id) // valid address node ID, nonlocal + continue + } + } + if _, addr, _, ok := report.ParseEndpointNodeID(id); ok { + if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) { + out = append(out, id) // valid endpoint node ID, nonlocal + continue + } + } + } + //log.Printf("select NONLOCAL: %d", len(out)) + return out +} + +func selectLike(regex string) selector { + re, err := regexp.Compile(regex) + if err != nil { + //log.Printf("select LIKE %q: %v", s, err) + re = regexp.MustCompile("") + } + return func(rns render.RenderableNodes) []string { + out := []string{} + for id := range rns { + if re.MatchString(id) { + out = append(out, id) + } + } + //log.Printf("select LIKE %q: %d", s, len(out)) + return out + } +} + +func selectWith(s string) selector { + var k, v string + if fields := strings.SplitN(s, "=", 2); len(fields) == 1 { + k = strings.TrimSpace(fields[0]) + } else if len(fields) == 2 { + k, v = strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1]) + } + return func(rns render.RenderableNodes) []string { + out := []string{} + for id, md := range rns { + if vv, ok := md.Metadata[k]; ok { + if v == "" || (v != "" && v == vv) { + out = append(out, id) + } + } + } + //log.Printf("select WITH %q: %d", s, len(out)) + return out + } +} + +func selectNot(s selector) selector { + return func(rns render.RenderableNodes) []string { + set := map[string]struct{}{} + for _, id := range s(rns) { + set[id] = struct{}{} + } + out := []string{} + for id := range rns { + if _, ok := set[id]; ok { + continue // selected by that one -> not by this one + } + out = append(out, id) + } + //log.Printf("select NOT: %d", len(out)) + return out + } +} + +const highlightKey = "_highlight" + +func transformHighlight(rns render.RenderableNodes, ids []string) render.RenderableNodes { + for _, id := range ids { + rn := rns[id] + rn.Node.Metadata[highlightKey] = "true" + rns[id] = rn + } + //log.Printf("transform HIGHLIGHT %d: OK", len(ids)) + return rns +} + +func transformRemove(rns render.RenderableNodes, ids []string) render.RenderableNodes { + toRemove := map[string]struct{}{} + for _, id := range ids { + toRemove[id] = struct{}{} + } + out := render.RenderableNodes{} + for id := range rns { + if _, ok := toRemove[id]; ok { + continue + } + cp(out, rns, id) + } + clean(out, toRemove) + //log.Printf("transform REMOVE %d: in %d, out %d", len(ids), len(rns), len(out.NodeMetadatas)) + return out +} + +func transformShowOnly(rns render.RenderableNodes, ids []string) render.RenderableNodes { + out := render.RenderableNodes{} + for _, id := range ids { + cp(out, rns, id) + } + toRemove := map[string]struct{}{} + for id := range rns { + if _, ok := out[id]; !ok { + toRemove[id] = struct{}{} + } + } + clean(out, toRemove) + //log.Printf("transform SHOWONLY %d: in %d, out %d", len(ids), len(rns), len(out.NodeMetadatas)) + return out +} + +func transformMerge(newname string) transformer { + return func(rns render.RenderableNodes, ids []string) render.RenderableNodes { + mapped := map[string]string{} + toRemove := map[string]struct{}{} + for _, id := range ids { + mapped[id] = newname + toRemove[id] = struct{}{} + } + out := render.RenderableNodes{} + for id := range rns { + if dstID, ok := mapped[id]; ok { + merge(out, dstID, rns, id) + } else { + cp(out, rns, id) + } + } + shift(out, mapped) + clean(out, toRemove) + //log.Printf("transform MERGE %d: in %d, out %d", len(ids), len(rns), len(out.NodeMetadatas)) + return out + } +} + +// transformGroupBy takes a key, and merges all nodes who share the same value +// for that key. It ignores nodes that don't have that key. +func transformGroupBy(s string) transformer { + keys := []string{} + for _, key := range strings.Split(s, ",") { + keys = append(keys, strings.TrimSpace(key)) + } + + return func(rns render.RenderableNodes, ids []string) render.RenderableNodes { + set := map[string]struct{}{} + for _, id := range ids { + set[id] = struct{}{} + } + + // Identify all nodes that should be grouped. + mapped := map[string]string{} // src ID: dst ID + toRemove := map[string]struct{}{} + for id, md := range rns { + if _, ok := set[id]; !ok { + continue // not selected + } + + parts := []string{} + for _, key := range keys { + if val, ok := md.Metadata[key]; ok { + parts = append(parts, fmt.Sprintf("%s-%s", key, val)) + } + } + if len(parts) < len(keys) { + continue // didn't match all required keys + } + + dstID := strings.Join(parts, "-") + mapped[id] = dstID + toRemove[id] = struct{}{} + } + + // Walk nodes again, merging those that should be grouped. + out := render.RenderableNodes{} + for id := range rns { + if dstID, ok := mapped[id]; ok { + merge(out, dstID, rns, id) + } else { + cp(out, rns, id) + } + } + shift(out, mapped) + clean(out, toRemove) + + //log.Printf("transform GROUPBY %v %d: in %d, out %d", keys, len(ids), len(rns), len(out.NodeMetadatas)) + return out + } +} + +// transformJoin takes a key whose value is (expected to be) a node ID. +// Basically, a foreign key. For every unique value (i.e. node) it finds, it +// copies the foreign node's metadata into every node that had the +// corresponding foreign key, and then deletes the foreign node. +// +// It's kind of like flattening the foreign nodes into all the nodes that +// point to them, one-to-many. +func transformJoin(key string) transformer { + return func(rns render.RenderableNodes, ids []string) render.RenderableNodes { + // key is e.g. host_node_id, value is a valid node ID. + // Collect the set of represented values (node IDs). + values := map[string]report.Node{} + toRemove := map[string]struct{}{} + for _, rn := range rns { + for k, v := range rn.Node.Metadata { + if k == key { + values[v] = report.MakeNode() // gather later + toRemove[v] = struct{}{} + } + } + } + + // Next, gather the metadata from nodes in the set. + for id, rn := range rns { + if found, ok := values[id]; ok { + values[id] = found.Merge(rn.Node) // gather + } + } + + // Finally, join that metadata to referential nodes. + // And delete the referenced nodes. + out := render.RenderableNodes{} + for id, rn := range rns { + if _, ok := values[id]; ok { + continue // delete the foreign nodes + } + cp(out, rns, id) // copy node + for k, v := range rn.Node.Metadata { + if k == key { + rn.Node = rn.Node.Merge(values[v]) // join metadata + } + } + out[id] = rn // write + } + clean(out, toRemove) + + //log.Printf("transform JOIN %v %d: in %d, out %d", key, len(ids), len(rns), len(out.NodeMetadatas)) + return out + } +} + +func cp(dst render.RenderableNodes, src render.RenderableNodes, id string) { + dst[id] = src[id].Copy() + + // Every transform that calls cp must call clean at the end, to remove + // dangling (uncopied) nodes from adjacency lists and edge metadatas. +} + +func merge(dst render.RenderableNodes, dstID string, src render.RenderableNodes, srcID string) { + dst[dstID] = dst[dstID].Merge(src[srcID]) + + // Every transform that calls merge must call shift at the end, to update + // adjacency lists and edge metadatas. +} + +func clean(dst render.RenderableNodes, toRemove map[string]struct{}) { + for id, rn := range dst { + // Clean out all the orphans from the adjacency list. + newAdjacency := report.IDList{} + for _, otherID := range rn.Node.Adjacency { + if _, ok := toRemove[otherID]; ok { + continue // can't be a dst anymore + } + newAdjacency = newAdjacency.Add(otherID) + } + rn.Node.Adjacency = newAdjacency + + // Clean out all the orphans from the edges. + newEdges := report.EdgeMetadatas{} + for otherID, edge := range rn.Node.Edges { + if _, ok := toRemove[otherID]; ok { + continue // can't be an edge anymore + } + newEdges[otherID] = edge + } + rn.Node.Edges = newEdges + + dst[id] = rn + } + + // Just to be safe. + for id := range toRemove { + delete(dst, id) + } +} + +func shift(dst render.RenderableNodes, mapping map[string]string) { + // We've got a mapping of old IDs to new IDs. Any adjacency targeting an + // old ID should be updated to the new ID. + for id, rn := range dst { + newAdjacency := report.IDList{} + for _, otherID := range rn.Node.Adjacency { + if mappedID, ok := mapping[otherID]; ok { + otherID = mappedID // just shift it on over + } + newAdjacency = newAdjacency.Add(otherID) // this will dedupe + } + rn.Node.Adjacency = newAdjacency + + newEdges := report.EdgeMetadatas{} + for otherID, edge := range rn.Node.Edges { + if mappedID, ok := mapping[otherID]; ok { + otherID = mappedID + } + newEdges[otherID] = newEdges[otherID].Merge(edge) // important to merge here + } + rn.Node.Edges = newEdges + + dst[id] = rn + } +} diff --git a/render/dsl/expression_internal_test.go b/render/dsl/expression_internal_test.go new file mode 100644 index 000000000..847939fc3 --- /dev/null +++ b/render/dsl/expression_internal_test.go @@ -0,0 +1,245 @@ +package dsl + +import ( + "path/filepath" + "reflect" + "runtime" + "sort" + "testing" + + "github.com/weaveworks/scope/probe/endpoint" + + "github.com/weaveworks/scope/probe/host" + "github.com/weaveworks/scope/render" + "github.com/weaveworks/scope/report" +) + +func TestSelectAll(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a"), + "c": render.NewRenderableNode("c"), + "b": render.NewRenderableNode("b"), + } + testStringSlice(t, []string{"a", "b", "c"}, selectAll(rns)) +} + +func TestSelectConnected(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("m", "b", "c"))), + "b": render.NewRenderableNode("b").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("m", "c"))), + "c": render.NewRenderableNode("c").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("m"))), + "m": render.NewRenderableNode("m").WithNode(report.MakeNode().WithAdjacency(report.MakeIDList("n"))), + "n": render.NewRenderableNode("n"), + "x": render.NewRenderableNode("x"), + "y": render.NewRenderableNode("y"), + "z": render.NewRenderableNode("z"), + } + testStringSlice(t, []string{"a", "b", "c", "m", "n"}, selectConnected(rns)) +} + +func TestSelectNonlocal(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{host.LocalNetworks: "10.10.1.0/24"})), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{host.LocalNetworks: "10.10.2.0/24"})), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{host.LocalNetworks: "10.20.0.0/16"})), + + // Test selecting nonlocal via endpoint.Addr metadata key + "x1": render.NewRenderableNode("x1").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "10.10.1.34"})), // local + "x2": render.NewRenderableNode("x2").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "10.20.99.1"})), // local + "x3": render.NewRenderableNode("x3").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "10.10.3.33"})), // nonlocal + "x4": render.NewRenderableNode("x4").WithNode(report.MakeNodeWith(map[string]string{endpoint.Addr: "192.168.1.1"})), // nonlocal + } + + // Test selecting nonlocal via parsing the node ID + var ( + local1 = report.MakeAddressNodeID("some-host", "10.10.2.2") + nonlocal1 = report.MakeAddressNodeID("some-host", "10.11.12.13") + local2 = report.MakeEndpointNodeID("some-host", "10.20.0.1", "4040") + nonlocal2 = report.MakeEndpointNodeID("some-host", "10.21.32.43", "8080") + ) + rns[local1] = render.NewRenderableNode(local1).WithNode(report.MakeNode()) + rns[nonlocal1] = render.NewRenderableNode(nonlocal1).WithNode(report.MakeNode()) + rns[local2] = render.NewRenderableNode(local2).WithNode(report.MakeNode()) + rns[nonlocal2] = render.NewRenderableNode(nonlocal2).WithNode(report.MakeNode()) + + testStringSlice(t, []string{"x3", "x4", nonlocal1, nonlocal2}, selectNonlocal(rns)) +} + +func TestSelectLike(t *testing.T) { + rns := render.RenderableNodes{ + "abcfooxyz": render.NewRenderableNode("abcfooxyz"), + "abcfoxyz": render.NewRenderableNode("abcfoxyz"), + "fo": render.NewRenderableNode("fo"), + "foo": render.NewRenderableNode("foo"), + "fooo": render.NewRenderableNode("fooo"), + "x_foo_y": render.NewRenderableNode("x_foo_y"), + } + testStringSlice(t, []string{"abcfooxyz", "foo", "fooo", "x_foo_y"}, selectLike(`.*foo.*`)(rns)) +} + +func TestSelectWith(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "bar"})), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo": "bar"})), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo": "baz"})), + "x": render.NewRenderableNode("x").WithNode(report.MakeNodeWith(map[string]string{"qux": "qix"})), + "y": render.NewRenderableNode("y").WithNode(report.MakeNode()), + } + testStringSlice(t, []string{"a", "b", "c"}, selectWith("foo")(rns)) + testStringSlice(t, []string{"a", "b"}, selectWith("foo=bar")(rns)) + testStringSlice(t, []string{"x"}, selectWith("qux")(rns)) + testStringSlice(t, []string{"x"}, selectWith("qux=qix")(rns)) + testStringSlice(t, []string{}, selectWith("qux=XXX")(rns)) +} + +func TestSelectNot(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"})), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo": "2"})), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo": "3"})), + "x": render.NewRenderableNode("x").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"})), + "y": render.NewRenderableNode("y").WithNode(report.MakeNodeWith(map[string]string{"bar": "2"})), + "z": render.NewRenderableNode("z").WithNode(report.MakeNodeWith(map[string]string{"bar": "3"})), + } + testStringSlice(t, []string{"x", "y", "z"}, selectNot(selectWith("foo"))(rns)) + testStringSlice(t, []string{"a", "b", "c"}, selectNot(selectWith("bar"))(rns)) + testStringSlice(t, []string{}, selectNot(selectAll)(rns)) +} + +func TestTransformHighlight(t *testing.T) { + rns := render.RenderableNodes{"a": render.NewRenderableNode("a")} + get := func(rns render.RenderableNodes) string { return rns["a"].Metadata[highlightKey] } + out := transformHighlight(rns, selectAll(rns)) + if want, have := "true", get(out); want != have { + t.Errorf("want %q, have %q", want, have) + } +} + +func TestTransformRemove(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b", "c"))), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"}).WithAdjacency(report.MakeIDList("c"))), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"baz": "1"})), + } + + // Remove all should totally kill the RenderableNodes + if want, have := 0, len(transformRemove(rns, selectAll(rns))); want != have { + t.Errorf("remove all: want %d, have %d", want, have) + } + + // Removing c should kill the adjacency links to it + out := transformRemove(rns, selectLike("c")(rns)) + if want, have := 2, len(out); want != have { + t.Errorf("want %d, have %d", want, have) + } + if want, have := 1, len(out["a"].Node.Adjacency); want != have { + t.Errorf("want %d, have %d", want, have) + } + if want, have := 0, len(out["b"].Node.Adjacency); want != have { + t.Errorf("want %d, have %d", want, have) + } +} + +func TestTransformShowOnly(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b", "c"))), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"}).WithAdjacency(report.MakeIDList("c"))), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"baz": "1"})), + } + + // Show only b should eliminate a, c, and the link to c + out := transformShowOnly(rns, selectLike("b")(rns)) + if want, have := 1, len(out); want != have { + t.Errorf("want %d, have %d (%#+v)", want, have, out) + } + if want, have := 0, len(out["b"].Node.Adjacency); want != have { + t.Errorf("want %d, have %d", want, have) + } +} + +func TestTransformMerge(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b", "c"))), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"bar": "1"}).WithAdjacency(report.MakeIDList("c"))), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"baz": "1"})), + } + + name := "supernode" + out := transformMerge(name)(rns, selectAll(rns)) + if want, have := 1, len(out); want != have { + t.Errorf("want %d, have %d", want, have) + } + if want, have := 3, len(out[name].Node.Metadata); want != have { + t.Errorf("want %d, have %d", want, have) + } + if want, have := 1, len(out[name].Node.Adjacency); want != have { + t.Fatalf("want %d, have %d", want, have) + } + if want, have := name, out[name].Node.Adjacency[0]; want != have { + t.Errorf("want %q, have %q", want, have) + } +} + +func TestTransformGroupBy(t *testing.T) { + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("b")).WithCounters(map[string]int{"c": 1})), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("c")).WithCounters(map[string]int{"c": 2})), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo": "1"}).WithAdjacency(report.MakeIDList("d")).WithCounters(map[string]int{"c": 4})), + "d": render.NewRenderableNode("d").WithNode(report.MakeNodeWith(map[string]string{"foo": "2"}).WithCounters(map[string]int{"c": 8})), + } + + out := transformGroupBy("foo")(rns, selectAll(rns)) + if want, have := 2, len(out); want != have { + t.Errorf("want %d, have %d", want, have) + } + if _, ok := out["foo-1"]; !ok { + t.Fatalf("missing merged node") + } + if _, ok := out["foo-2"]; !ok { + t.Fatalf("missing unmerged node") + } + if want, have := 2, len(out["foo-1"].Node.Adjacency); want != have { // self-edge, and link to the other + t.Errorf("want %d, have %d", want, have) + } + if want, have := 1+2+4, out["foo-1"].Node.Counters["c"]; want != have { + t.Errorf("want %d, have %d", want, have) + } + if want, have := 8, out["foo-2"].Node.Counters["c"]; want != have { + t.Errorf("want %d, have %d", want, have) + } +} + +func TestTransformJoin(t *testing.T) { + const key, value = "😇", "😈" + rns := render.RenderableNodes{ + "a": render.NewRenderableNode("a").WithNode(report.MakeNodeWith(map[string]string{"foo_node": "d", "1": "1"}).WithCounters(map[string]int{"c": 1})), + "b": render.NewRenderableNode("b").WithNode(report.MakeNodeWith(map[string]string{"foo_node": "d", "2": "2"}).WithCounters(map[string]int{"c": 2})), + "c": render.NewRenderableNode("c").WithNode(report.MakeNodeWith(map[string]string{"foo_node": "x", "4": "4"}).WithCounters(map[string]int{"c": 4})), + "d": render.NewRenderableNode("d").WithNode(report.MakeNodeWith(map[string]string{key: value}).WithCounters(map[string]int{"c": 8})), + } + + out := transformJoin("foo_node")(rns, selectAll(rns)) + if want, have := 3, len(out); want != have { + t.Errorf("want %d, have %d", want, have) + } + for _, id := range []string{"a", "b"} { + if want, have := value, out[id].Node.Metadata[key]; want != have { + t.Errorf("%s[%q]: want %q, have %q", id, key, want, have) + } + } + if want, have := 1+8, out["a"].Node.Counters["c"]; want != have { + t.Errorf("a: want %d, have %d", want, have) + } + if want, have := 2+8, out["b"].Node.Counters["c"]; want != have { + t.Errorf("b: want %d, have %d", want, have) + } +} + +func testStringSlice(t *testing.T, want, have []string) { + sort.Strings(want) + sort.Strings(have) + if !reflect.DeepEqual(want, have) { + _, file, line, _ := runtime.Caller(1) + t.Errorf("%s:%d: want %v, have %v", filepath.Base(file), line, want, have) + } +} diff --git a/render/dsl/lexer.go b/render/dsl/lexer.go new file mode 100644 index 000000000..5153a4dfe --- /dev/null +++ b/render/dsl/lexer.go @@ -0,0 +1,326 @@ +package dsl + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +// Expression = [NOT] Selector [Transformer] +// Selector = ALL / CONNECTED / NONLOCAL / LIKE {{ }} / WITH {{ [= ] }} +// Transformer = HIGHLIGHT / REMOVE / SHOWONLY / MERGE / GROUPBY {{ , ... }} // JOIN {{ key }} + +type lexer struct { + input string // string being scanned + start int // start position of this item + pos int // current position within the input + width int // width of last rune read + items chan item +} + +func lex(input string) (*lexer, <-chan item) { + l := &lexer{ + input: input, + items: make(chan item), + } + go l.run() + return l, l.items +} + +const ( + keywordNot = "NOT" + keywordAll = "ALL" + keywordConnected = "CONNECTED" + keywordNonlocal = "NONLOCAL" + keywordLike = "LIKE" + keywordWith = "WITH" + keywordHighlight = "HIGHLIGHT" + keywordRemove = "REMOVE" + keywordShowOnly = "SHOWONLY" + keywordMerge = "MERGE" + keywordGroupBy = "GROUPBY" + keywordJoin = "JOIN" +) + +type itemType int + +const ( + itemError itemType = iota + itemNot + itemAll + itemConnected + itemNonlocal + itemLike + itemWith + itemHighlight + itemRemove + itemShowOnly + itemMerge + itemGroupBy + itemJoin + itemRegex + itemKeyValue + itemKeyList + itemKey +) + +func (t itemType) String() string { + switch t { + case itemError: + return "ERROR" + case itemNot: + return keywordNot + case itemAll: + return keywordAll + case itemConnected: + return keywordConnected + case itemNonlocal: + return keywordNonlocal + case itemLike: + return keywordLike + case itemWith: + return keywordWith + case itemHighlight: + return keywordHighlight + case itemRemove: + return keywordRemove + case itemShowOnly: + return keywordShowOnly + case itemMerge: + return keywordMerge + case itemGroupBy: + return keywordGroupBy + case itemJoin: + return keywordJoin + case itemRegex: + return "" + case itemKeyValue: + return "" + case itemKeyList: + return "" + case itemKey: + return "" + default: + return "unknown" + } +} + +type stateFn func(*lexer) stateFn + +func (l *lexer) run() { + for state := lexExpression; state != nil; { + state = state(l) + } + close(l.items) +} + +func (l *lexer) emit(t itemType) { + l.items <- item{t, l.input[l.start:l.pos]} + l.start = l.pos +} + +const eof rune = -1 + +func (l *lexer) next() (r rune) { + if l.pos >= len(l.input) { + l.width = 0 + return eof + } + r, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) + l.pos += l.width + return r +} + +func (l *lexer) backup() { l.pos -= l.width } + +// acceptRun consumes a run of runes from the valid set. +func (l *lexer) acceptRun(validSet string) { + for strings.IndexRune(validSet, l.next()) >= 0 { + // consume + } + l.backup() +} + +func (l *lexer) eatWhitespace() { + l.acceptRun(" \t\r\n") +} + +// errorf terminates lexing with an error. +func (l *lexer) errorf(format string, args ...interface{}) stateFn { + l.items <- item{itemError, fmt.Sprintf(format, args...)} + return nil +} + +type item struct { + itemType itemType + literal string +} + +func (i item) String() string { + return fmt.Sprintf("%s %q", i.itemType, i.literal) +} + +func lexExpression(l *lexer) stateFn { + l.eatWhitespace() + if strings.HasPrefix(l.input[l.pos:], keywordNot) { + return lexNot + } + return lexSelector +} + +func lexNot(l *lexer) stateFn { + l.pos += len(keywordNot) + l.emit(itemNot) + return lexSelector +} + +func lexSelector(l *lexer) stateFn { + l.eatWhitespace() + switch { + case strings.HasPrefix(l.input[l.pos:], keywordAll): + return lexAll + case strings.HasPrefix(l.input[l.pos:], keywordConnected): + return lexConnected + case strings.HasPrefix(l.input[l.pos:], keywordNonlocal): + return lexNonlocal + case strings.HasPrefix(l.input[l.pos:], keywordLike): + return lexLike + case strings.HasPrefix(l.input[l.pos:], keywordWith): + return lexWith + default: + return l.errorf("bad selector") + } +} + +func lexAll(l *lexer) stateFn { + l.pos += len(keywordAll) + l.emit(itemAll) + return lexTransformer +} + +func lexConnected(l *lexer) stateFn { + l.pos += len(keywordConnected) + l.emit(itemConnected) + return lexTransformer +} + +func lexNonlocal(l *lexer) stateFn { + l.pos += len(keywordNonlocal) + l.emit(itemNonlocal) + return lexTransformer +} + +func lexLike(l *lexer) stateFn { + l.pos += len(keywordLike) + l.emit(itemLike) + return lexRegex +} + +func lexWith(l *lexer) stateFn { + l.pos += len(keywordWith) + l.emit(itemWith) + return lexKeyValue +} + +func lexRegex(l *lexer) stateFn { + return lexMeta("regex", itemRegex, lexTransformer) +} + +func lexKeyValue(l *lexer) stateFn { + return lexMeta("key=value", itemKeyValue, lexTransformer) +} + +func lexTransformer(l *lexer) stateFn { + l.eatWhitespace() + switch { + case l.pos == len(l.input): + return nil // done + case strings.HasPrefix(l.input[l.pos:], keywordHighlight): + return lexHighlight + case strings.HasPrefix(l.input[l.pos:], keywordRemove): + return lexRemove + case strings.HasPrefix(l.input[l.pos:], keywordShowOnly): + return lexShowOnly + case strings.HasPrefix(l.input[l.pos:], keywordMerge): + return lexMerge + case strings.HasPrefix(l.input[l.pos:], keywordGroupBy): + return lexGroupBy + case strings.HasPrefix(l.input[l.pos:], keywordJoin): + return lexJoin + default: + return l.errorf("bad transformer at position %d: %s", l.pos, l.input[l.pos:]) + } +} + +func lexHighlight(l *lexer) stateFn { + l.pos += len(keywordHighlight) + l.emit(itemHighlight) + return nil +} + +func lexRemove(l *lexer) stateFn { + l.pos += len(keywordRemove) + l.emit(itemRemove) + return nil +} + +func lexShowOnly(l *lexer) stateFn { + l.pos += len(keywordShowOnly) + l.emit(itemShowOnly) + return nil +} + +func lexMerge(l *lexer) stateFn { + l.pos += len(keywordMerge) + l.emit(itemMerge) + return nil +} + +func lexGroupBy(l *lexer) stateFn { + l.pos += len(keywordGroupBy) + l.emit(itemGroupBy) + return lexKeyList +} + +func lexJoin(l *lexer) stateFn { + l.pos += len(keywordJoin) + l.emit(itemJoin) + return lexKey +} + +func lexKeyList(l *lexer) stateFn { + return lexMeta("key list", itemKeyList, nil) +} + +func lexKey(l *lexer) stateFn { + return lexMeta("key", itemKey, nil) +} + +const ( + leftMeta = "{{" + rightMeta = "}}" +) + +func lexMeta(what string, t itemType, next stateFn) stateFn { + return func(l *lexer) stateFn { + l.eatWhitespace() + if !strings.HasPrefix(l.input[l.pos:], leftMeta) { + return l.errorf("%s must begin with %s", what, leftMeta) + } + l.pos += len(leftMeta) + l.start = l.pos + for { + if l.pos > len(l.input) { + return l.errorf("%s must end with %s", what, rightMeta) + } + if strings.HasPrefix(l.input[l.pos:], rightMeta) { + break + } + l.pos++ + } + l.emit(t) + l.pos += len(rightMeta) + l.start = l.pos + return next + } +} diff --git a/render/dsl/lexer_internal_test.go b/render/dsl/lexer_internal_test.go new file mode 100644 index 000000000..104452cc4 --- /dev/null +++ b/render/dsl/lexer_internal_test.go @@ -0,0 +1,76 @@ +package dsl + +import ( + "errors" + "testing" +) + +func TestLexer(t *testing.T) { + for _, test := range []struct { + input string + want []item + err error + }{ + { + "", + []item{}, + errors.New("bad selector"), + }, + { + "foo", + []item{}, + errors.New("bad selector"), + }, + { + "ALL", + []item{{itemAll, keywordAll}}, + errors.New("bad transformer"), + }, + { + "NOT ALL", + []item{{itemNot, keywordNot}, {itemAll, keywordAll}}, + errors.New("bad transformer"), + }, + { + "ALL HIGHLIGHT", + []item{{itemAll, keywordAll}, {itemHighlight, keywordHighlight}}, + nil, + }, + { + "WITH {{pid}} REMOVE", + []item{{itemWith, keywordWith}, {itemKeyValue, "pid"}, {itemRemove, keywordRemove}}, + nil, + }, + } { + _, c := lex(test.input) + for item := range c { + if item.itemType == itemError { + if test.err == nil { + t.Errorf("%q: unexpected error: %v", test.input, item.literal) + break + } + if want, have := test.err.Error(), item.literal; want != have { + t.Errorf("%q: want error %q, have %q", test.input, want, have) + break + } + t.Logf("%q: got expected error %v", test.input, test.err) + break + } + + if len(test.want) <= 0 { + t.Errorf("%q: got too many items", test.input) + break + } + + want := test.want[0] + test.want = test.want[1:] + + if want, have := want.itemType, item.itemType; want != have { + t.Errorf("%q: unexpected item: want %v, have %v", test.input, want, have) + break + } + + t.Logf("%s: lex %s (%q) OK", test.input, item.itemType, item.literal) + } + } +} diff --git a/render/dsl/parser.go b/render/dsl/parser.go new file mode 100644 index 000000000..5f51c089c --- /dev/null +++ b/render/dsl/parser.go @@ -0,0 +1,104 @@ +package dsl + +import ( + "errors" + "fmt" + "log" + "math/rand" +) + +// ParseExpression parses a single expression string. +func ParseExpression(str string) (Expression, error) { + var ( + expr Expression + not bool + ) + _, c := lex(str) + for item := range c { + switch item.itemType { + case itemNot: + not = !not + + case itemAll: + expr.selector = selectAll + + case itemConnected: + expr.selector = selectConnected + + case itemNonlocal: + expr.selector = selectNonlocal + + case itemLike: + item = <-c + switch item.itemType { + case itemRegex: + expr.selector = selectLike(item.literal) + default: + return Expression{}, fmt.Errorf("bad LIKE: want %s, got %s", itemRegex, item.itemType) + } + + case itemWith: + item = <-c + switch item.itemType { + case itemKeyValue: + expr.selector = selectWith(item.literal) + default: + return Expression{}, fmt.Errorf("bad WITH: want %s, got %s", itemKeyValue, item.itemType) + } + + case itemHighlight: + expr.transformer = transformHighlight + + case itemRemove: + expr.transformer = transformRemove + + case itemShowOnly: + expr.transformer = transformShowOnly + + case itemMerge: + expr.transformer = transformMerge(fmt.Sprintf("%x", rand.Int31())) // TODO(pb): parameterize the name + + case itemGroupBy: + item = <-c + switch item.itemType { + case itemKeyList: + expr.transformer = transformGroupBy(item.literal) + default: + return Expression{}, fmt.Errorf("bad GROUPBY: want %s, got %s", itemKeyList, item.itemType) + } + + case itemJoin: + item = <-c + switch item.itemType { + case itemKey: + expr.transformer = transformJoin(item.literal) + default: + return Expression{}, fmt.Errorf("bad JOIN: want %s, got %s", itemKey, item.itemType) + } + + default: + return Expression{}, errors.New(item.literal) + } + } + if not { + expr.selector = selectNot(expr.selector) + } + if expr.transformer == nil { + expr.transformer = transformHighlight + } + return expr, nil +} + +// ParseExpressions parses multiple expression strings. +func ParseExpressions(strs ...string) Expressions { + var exprs Expressions + for _, str := range strs { + expr, err := ParseExpression(str) + if err != nil { + log.Printf("%s: %v", str, err) + continue + } + exprs = append(exprs, expr) + } + return exprs +} diff --git a/render/dsl/parser_internal_test.go b/render/dsl/parser_internal_test.go new file mode 100644 index 000000000..d2183adb3 --- /dev/null +++ b/render/dsl/parser_internal_test.go @@ -0,0 +1,68 @@ +package dsl + +import ( + "errors" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestParseExpression(t *testing.T) { + for _, test := range []struct { + input string + want Expression + err error + }{ + { + "", + Expression{}, + errors.New("bad selector"), + }, + { + "ALL", + Expression{selectAll, transformHighlight}, + nil, + }, + { + "HIGHLIGHT", + Expression{}, + errors.New("bad selector"), + }, + { + "CONNECTED REMOVE", + Expression{selectConnected, transformRemove}, + nil, + }, + { + "NOT CONNECTED MERGE", + Expression{selectNot(selectConnected), transformMerge("foobar")}, + nil, + }, + } { + have, err := ParseExpression(test.input) + if err == nil && test.err != nil { + t.Errorf("%q: want error %q, have no error", test.input, test.err.Error()) + continue + } else if err != nil && test.err == nil { + t.Errorf("%q: want no error, have error %q", test.input, err.Error()) + continue + } else if err != nil && test.err != nil && test.err.Error() != err.Error() { + t.Errorf("%q: want error %q, have %q", test.input, test.err.Error(), err.Error()) + continue + } + if want, have := nameof(test.want.selector), nameof(have.selector); want != have { + t.Errorf("%q: selector: want %v, have %v", test.input, want, have) + } + if want, have := nameof(test.want.transformer), nameof(have.transformer); want != have { + t.Errorf("%q: transformer: want %v, have %v", test.input, want, have) + } + } +} + +func nameof(i interface{}) string { + full := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() // github.com/weaveworks/scope/render/dsl.selectAll + fields := strings.Split(full, ".") // [github com/weaveworks/scope/render/dsl selectAll] + last := fields[len(fields)-1] // selectAll + return last +}