diff --git a/app/api_topologies_test.go b/app/api_topologies_test.go index 3d8494a1c..ae52f1247 100644 --- a/app/api_topologies_test.go +++ b/app/api_topologies_test.go @@ -16,7 +16,7 @@ func TestAPITopology(t *testing.T) { if err := json.Unmarshal(body, &topologies); err != nil { t.Fatalf("JSON parse error: %s", err) } - equals(t, 3, len(topologies)) + equals(t, 5, len(topologies)) for _, topology := range topologies { is200(t, ts, topology.URL) diff --git a/app/router.go b/app/router.go index a8f7b8702..34d9c5c78 100644 --- a/app/router.go +++ b/app/router.go @@ -124,6 +124,45 @@ func apiHandler(w http.ResponseWriter, r *http.Request) { } var topologyRegistry = map[string]topologyView{ + "applications-expr": { + human: "Applications (expr)", + parent: "", + renderer: render.ExpressionRenderer( + "ALL GROUPBY {{pid}}", + "NOT WITH {{pid}} REMOVE", + "NOT CONNECTED REMOVE", + ), + }, + "applications-expr-by-name": { + human: "by name (expr)", + parent: "applications-expr", + renderer: render.ExpressionRenderer( + "ALL GROUPBY {{pid}}", + "ALL GROUPBY {{comm}}", + "NOT WITH {{comm}} REMOVE", + "NOT CONNECTED REMOVE", + ), + }, + "containers-expr": { + human: "Containers (expr)", + parent: "", + renderer: render.ExpressionRenderer( + "ALL GROUPBY {{pid}}", + "ALL GROUPBY {{docker_container_id}}", + "NOT WITH {{docker_container_id}} REMOVE", + ), + }, + "containers-expr-by-image": { + human: "by image (expr)", + parent: "containers-expr", + renderer: render.ExpressionRenderer( + "ALL GROUPBY {{pid}}", + "ALL GROUPBY {{docker_container_id}}", + "ALL GROUPBY {{docker_image_id}}", + "NOT WITH {{docker_image_name}} REMOVE", + ), + }, + "applications": { human: "Applications", parent: "", @@ -144,6 +183,7 @@ var topologyRegistry = map[string]topologyView{ parent: "containers", renderer: render.ContainerImageRenderer, }, + "hosts": { human: "Hosts", parent: "", diff --git a/experimental/dsl/expression.go b/experimental/dsl/expression.go index 323bbd68a..dacb4cb04 100644 --- a/experimental/dsl/expression.go +++ b/experimental/dsl/expression.go @@ -10,7 +10,6 @@ import ( "github.com/weaveworks/scope/probe/endpoint" "github.com/weaveworks/scope/probe/host" - "github.com/weaveworks/scope/render" "github.com/weaveworks/scope/report" ) @@ -75,7 +74,7 @@ func selectNonlocal(tpy report.Topology) []string { for _, md := range tpy.NodeMetadatas { for k, v := range md.Metadata { if k == host.LocalNetworks { - local = append(local, render.ParseNetworks(v)...) + local = append(local, report.ParseNetworks(v)...) } } } diff --git a/render/dsl/expression.go b/render/dsl/expression.go index 50d4c6ebe..41bf793a0 100644 --- a/render/dsl/expression.go +++ b/render/dsl/expression.go @@ -2,7 +2,6 @@ package dsl import ( "fmt" - "log" "math/rand" "net" "regexp" @@ -10,7 +9,6 @@ import ( "github.com/weaveworks/scope/probe/endpoint" "github.com/weaveworks/scope/probe/host" - "github.com/weaveworks/scope/render" "github.com/weaveworks/scope/report" ) @@ -51,7 +49,7 @@ func selectAll(tpy report.Topology) []string { for id := range tpy.NodeMetadatas { out = append(out, id) } - log.Printf("select ALL: %d", len(out)) + //log.Printf("select ALL: %d", len(out)) return out } @@ -73,7 +71,7 @@ func selectConnected(tpy report.Topology) []string { out = append(out, id) } } - log.Printf("select CONNECTED: %d", len(out)) + //log.Printf("select CONNECTED: %d", len(out)) return out } @@ -82,7 +80,7 @@ func selectNonlocal(tpy report.Topology) []string { for _, md := range tpy.NodeMetadatas { for k, v := range md.Metadata { if k == host.LocalNetworks { - local = append(local, render.ParseNetworks(v)...) + local = append(local, report.ParseNetworks(v)...) } } } @@ -107,14 +105,14 @@ func selectNonlocal(tpy report.Topology) []string { } } } - log.Printf("select NONLOCAL: %d", len(out)) + //log.Printf("select NONLOCAL: %d", len(out)) return out } func selectLike(s string) selector { re, err := regexp.Compile(s) if err != nil { - log.Printf("select LIKE %q: %v", s, err) + //log.Printf("select LIKE %q: %v", s, err) re = regexp.MustCompile("") } return func(tpy report.Topology) []string { @@ -124,7 +122,7 @@ func selectLike(s string) selector { out = append(out, id) } } - log.Printf("select LIKE %q: %d", s, len(out)) + //log.Printf("select LIKE %q: %d", s, len(out)) return out } } @@ -146,7 +144,7 @@ func selectWith(s string) selector { } } } - log.Printf("select WITH %q: %d", s, len(out)) + //log.Printf("select WITH %q: %d", s, len(out)) return out } } @@ -164,7 +162,7 @@ func selectNot(s selector) selector { } out = append(out, id) } - log.Printf("select NOT: %d", len(out)) + //log.Printf("select NOT: %d", len(out)) return out } } @@ -175,7 +173,7 @@ func transformHighlight(tpy report.Topology, ids []string) report.Topology { for _, id := range ids { tpy.NodeMetadatas[id] = tpy.NodeMetadatas[id].Merge(report.MakeNodeMetadataWith(map[string]string{highlightKey: "true"})) } - log.Printf("transform HIGHLIGHT %d: OK", len(ids)) + //log.Printf("transform HIGHLIGHT %d: OK", len(ids)) return tpy } @@ -192,7 +190,7 @@ func transformRemove(tpy report.Topology, ids []string) report.Topology { cp(out, tpy, id) } clean(out, toRemove) - log.Printf("transform REMOVE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + //log.Printf("transform REMOVE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) return out } @@ -204,7 +202,7 @@ func transformShowOnly(tpy report.Topology, ids []string) report.Topology { } cp(out, tpy, id) } - log.Printf("transform SHOWONLY %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + //log.Printf("transform SHOWONLY %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) return out } @@ -222,7 +220,7 @@ func transformMerge(tpy report.Topology, ids []string) report.Topology { cp(out, tpy, id) } } - log.Printf("transform MERGE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + //log.Printf("transform MERGE %d: in %d, out %d", len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) return out } @@ -269,7 +267,7 @@ func transformGroupBy(s string) transformer { } } - log.Printf("transform GROUPBY %v %d: in %d, out %d", keys, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + //log.Printf("transform GROUPBY %v %d: in %d, out %d", keys, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) return out } } @@ -310,7 +308,7 @@ func transformJoin(key string) transformer { out.NodeMetadatas[id] = md // write } - log.Printf("transform JOIN %v %d: in %d, out %d", key, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) + //log.Printf("transform JOIN %v %d: in %d, out %d", key, len(ids), len(tpy.NodeMetadatas), len(out.NodeMetadatas)) return out } } diff --git a/render/dsl/expression_test.go b/render/dsl/expression_test.go index fa2cfc8a7..b92bbf0fd 100644 --- a/render/dsl/expression_test.go +++ b/render/dsl/expression_test.go @@ -21,8 +21,8 @@ var fixture = report.Topology{ "c": report.MakeNodeMetadataWith(map[string]string{"is-a-or-b-or-c": "true"}), }, EdgeMetadatas: map[string]report.EdgeMetadata{ - report.MakeEdgeID("a", "c"): report.EdgeMetadata{EgressPacketCount: newu64(1)}, - report.MakeEdgeID("b", "c"): report.EdgeMetadata{EgressPacketCount: newu64(2)}, + report.MakeEdgeID("a", "c"): {EgressPacketCount: newu64(1)}, + report.MakeEdgeID("b", "c"): {EgressPacketCount: newu64(2)}, }, } diff --git a/render/dsl/parser.go b/render/dsl/parser.go index 10eba02ef..5c4f5a1da 100644 --- a/render/dsl/parser.go +++ b/render/dsl/parser.go @@ -97,7 +97,6 @@ func ParseExpressions(strs ...string) Expressions { log.Printf("%s: %v", str, err) continue } - log.Printf("%s: OK", str) exprs = append(exprs, expr) } return exprs diff --git a/render/render.go b/render/render.go index d1db1ed9c..328aa1b75 100644 --- a/render/render.go +++ b/render/render.go @@ -1,8 +1,11 @@ package render import ( + "fmt" "log" + "strings" + "github.com/weaveworks/scope/render/dsl" "github.com/weaveworks/scope/report" ) @@ -354,3 +357,85 @@ func (f Filter) Render(rpt report.Report) RenderableNodes { } return output } + +// ExpressionRenderer is a Renderer that's based on a set of DSL expressions. +func ExpressionRenderer(exprStrs ...string) Renderer { + return expressionRenderer{dsl.ParseExpressions(exprStrs...)} +} + +type expressionRenderer struct { + exprs dsl.Expressions +} + +func (r expressionRenderer) Render(rpt report.Report) RenderableNodes { + // Flatten all the topologies. + tpy := report.MakeTopology() + for _, t := range rpt.Topologies() { + if err := conflict(tpy, t); err != nil { + log.Printf("conflict in node IDs: %v", err) // oh well... + } + tpy = tpy.Merge(t) + } + + // Evaluate expressions. + tpy = r.exprs.Eval(tpy) + + // Convert to RenderableNodes. + return tpy2rns(tpy) +} + +func (r expressionRenderer) EdgeMetadata(rpt report.Report, localID, remoteID string) report.EdgeMetadata { + return report.EdgeMetadata{} // TODO(pb) +} + +func tpy2rns(tpy report.Topology) RenderableNodes { + out := RenderableNodes{} + + // First, get the nodes. + for id, md := range tpy.NodeMetadatas { + var ( + major = "" // TODO(pb) + minor = "" // TODO(pb) + rank = "" // TODO(pb) + ) + out[id] = NewRenderableNode(id, major, minor, rank, md) + } + + // Then, inject the adjacencies. + for adjacencyID, dstIDs := range tpy.Adjacency { + srcID, ok := report.ParseAdjacencyID(adjacencyID) + if !ok { + log.Printf("invalid adjacency ID %q", adjacencyID) + continue + } + node := out[srcID] + node.Adjacency = dstIDs + out[srcID] = node + } + + // TODO(pb): we have no way to track origins at the moment. We should have + // a separate NodeMetadata.Sets field, and push origin IDs there when we + // MERGE or GROUPBY. + + // TODO(pb): EdgeMetadata + + return out +} + +func conflict(a, b report.Topology) error { + var errs []string + for id := range a.NodeMetadatas { + if _, ok := b.NodeMetadatas[id]; ok { + errs = append(errs, id) + } + } + for id := range b.NodeMetadatas { + if _, ok := a.NodeMetadatas[id]; ok { + errs = append(errs, id) + } + } + if len(errs) > 0 { + return fmt.Errorf(strings.Join(errs, ", ")) + } + return nil +} diff --git a/render/theinternet.go b/render/theinternet.go index 81e0dd867..07d8a5545 100644 --- a/render/theinternet.go +++ b/render/theinternet.go @@ -13,33 +13,27 @@ import ( // used to determine which nodes in the report are "remote", i.e. outside of // our infrastructure. func LocalNetworks(r report.Report) report.Networks { - result := report.Networks{} + var ( + result = report.Networks{} + networks = map[string]struct{}{} + ) + for _, md := range r.Host.NodeMetadatas { val, ok := md.Metadata[host.LocalNetworks] if !ok { continue } - result = append(result, ParseNetworks(val)...) + for _, s := range strings.Fields(val) { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + continue + } + _, ok := networks[ipNet.String()] + if !ok { + result = append(result, ipNet) + networks[ipNet.String()] = struct{}{} + } + } } return result } - -// ParseNetworks converts a string of space-separated CIDRs to a -// report.Networks. -func ParseNetworks(v string) report.Networks { - var ( - nets = report.Networks{} - set = map[string]struct{}{} - ) - for _, s := range strings.Fields(v) { - _, ipNet, err := net.ParseCIDR(s) - if err != nil { - continue - } - if _, ok := set[ipNet.String()]; !ok { - nets = append(nets, ipNet) - set[ipNet.String()] = struct{}{} - } - } - return nets -} diff --git a/render/theinternet_test.go b/render/theinternet_test.go index 9c90314fb..fad04b5c3 100644 --- a/render/theinternet_test.go +++ b/render/theinternet_test.go @@ -1,7 +1,6 @@ package render_test import ( - "fmt" "net" "reflect" "testing" @@ -33,30 +32,6 @@ func TestReportLocalNetworks(t *testing.T) { } } -func TestParseNetworks(t *testing.T) { - var ( - hugenetStr = "1.0.0.0/8" - bignetStr = "10.1.0.1/16" - smallnetStr = "5.6.7.8/32" - hugenet = mustParseCIDR(hugenetStr) - bignet = mustParseCIDR(bignetStr) - smallnet = mustParseCIDR(smallnetStr) - ) - for _, tc := range []struct { - input string - want report.Networks - }{ - {"", report.Networks{}}, - {fmt.Sprintf("%s", bignetStr), report.Networks([]*net.IPNet{bignet})}, - {fmt.Sprintf("%s %s", bignetStr, bignetStr), report.Networks([]*net.IPNet{bignet})}, - {fmt.Sprintf("%s foo %s oops %s", hugenetStr, smallnetStr, hugenetStr), report.Networks([]*net.IPNet{hugenet, smallnet})}, - } { - if want, have := tc.want, render.ParseNetworks(tc.input); !reflect.DeepEqual(want, have) { - t.Error(test.Diff(want, have)) - } - } -} - func mustParseCIDR(s string) *net.IPNet { _, ipNet, err := net.ParseCIDR(s) if err != nil { diff --git a/report/networks.go b/report/networks.go index a1aa9e6e4..fa0707ab4 100644 --- a/report/networks.go +++ b/report/networks.go @@ -2,11 +2,30 @@ package report import ( "net" + "strings" ) // Networks represent a set of subnets type Networks []*net.IPNet +// ParseNetworks converts a string of space-separated CIDRs to a Networks. +func ParseNetworks(v string) Networks { + set := map[string]struct{}{} + for _, s := range strings.Fields(v) { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + continue + } + set[ipNet.String()] = struct{}{} + } + nets := Networks{} + for s := range set { + _, ipNet, _ := net.ParseCIDR(s) + nets = append(nets, ipNet) + } + return nets +} + // Interface is exported for testing. type Interface interface { Addrs() ([]net.Addr, error) diff --git a/report/networks_test.go b/report/networks_test.go index 0c18207f3..2d552bd13 100644 --- a/report/networks_test.go +++ b/report/networks_test.go @@ -1,6 +1,7 @@ package report_test import ( + "fmt" "net" "reflect" "testing" @@ -24,6 +25,28 @@ func TestContains(t *testing.T) { } } +func TestParseNetworks(t *testing.T) { + var ( + bignetStr = "10.1.0.1/16" + smallnetStr = "5.6.7.8/32" + bignet = mustParseCIDR(bignetStr) + smallnet = mustParseCIDR(smallnetStr) + ) + for _, tc := range []struct { + input string + want report.Networks + }{ + {"", report.Networks{}}, + {fmt.Sprintf("%s", bignetStr), report.Networks([]*net.IPNet{bignet})}, + {fmt.Sprintf("%s %s", bignetStr, bignetStr), report.Networks([]*net.IPNet{bignet})}, + {fmt.Sprintf("%s foo %s oops %s", smallnetStr, smallnetStr, smallnetStr), report.Networks([]*net.IPNet{smallnet})}, + } { + if want, have := tc.want, report.ParseNetworks(tc.input); !reflect.DeepEqual(want, have) { + t.Error(test.Diff(want, have)) + } + } +} + func mustParseCIDR(s string) *net.IPNet { _, ipNet, err := net.ParseCIDR(s) if err != nil {