From a759db89313f92eeb5338b2ba9a3ecc9409eac17 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 16 Jun 2015 13:34:52 +0000 Subject: [PATCH] Rename Map -> LeafMap, introduce Map (with tests) --- app/router.go | 12 +-- experimental/graphviz/handle.go | 6 +- render/mapping.go | 23 +++--- render/mapping_test.go | 2 +- render/render.go | 130 +++++++++++++++++++++++++------- render/render_test.go | 121 ++++++++++++++++++++++++++++- 6 files changed, 246 insertions(+), 48 deletions(-) diff --git a/app/router.go b/app/router.go index 4fa40f517..baeb97103 100644 --- a/app/router.go +++ b/app/router.go @@ -50,30 +50,30 @@ var topologyRegistry = map[string]topologyView{ "applications": { human: "Applications", parent: "", - renderer: render.Map{Selector: report.SelectEndpoint, Mapper: render.ProcessPID, Pseudo: render.GenericPseudoNode}, + renderer: render.LeafMap{Selector: report.SelectEndpoint, Mapper: render.ProcessPID, Pseudo: render.GenericPseudoNode}, }, "applications-by-name": { human: "by name", parent: "applications", - renderer: render.Map{Selector: report.SelectEndpoint, Mapper: render.ProcessName, Pseudo: render.GenericGroupedPseudoNode}, + renderer: render.LeafMap{Selector: report.SelectEndpoint, Mapper: render.ProcessName, Pseudo: render.GenericGroupedPseudoNode}, }, "containers": { human: "Containers", parent: "", renderer: render.Reduce([]render.Renderer{ - render.Map{Selector: report.SelectEndpoint, Mapper: render.MapEndpoint2Container, Pseudo: render.InternetOnlyPseudoNode}, - render.Map{Selector: report.SelectContainer, Mapper: render.MapContainerIdentity, Pseudo: render.InternetOnlyPseudoNode}, + render.LeafMap{Selector: report.SelectEndpoint, Mapper: render.MapEndpoint2Container, Pseudo: render.InternetOnlyPseudoNode}, + render.LeafMap{Selector: report.SelectContainer, Mapper: render.MapContainerIdentity, Pseudo: render.InternetOnlyPseudoNode}, }), }, "containers-by-image": { human: "by image", parent: "containers", - renderer: render.Map{Selector: report.SelectEndpoint, Mapper: render.ProcessContainerImage, Pseudo: render.InternetOnlyPseudoNode}, + renderer: render.LeafMap{Selector: report.SelectEndpoint, Mapper: render.ProcessContainerImage, Pseudo: render.InternetOnlyPseudoNode}, }, "hosts": { human: "Hosts", parent: "", - renderer: render.Map{Selector: report.SelectAddress, Mapper: render.NetworkHostname, Pseudo: render.GenericPseudoNode}, + renderer: render.LeafMap{Selector: report.SelectAddress, Mapper: render.NetworkHostname, Pseudo: render.GenericPseudoNode}, }, } diff --git a/experimental/graphviz/handle.go b/experimental/graphviz/handle.go index 5dc611a94..fbd560613 100644 --- a/experimental/graphviz/handle.go +++ b/experimental/graphviz/handle.go @@ -16,7 +16,7 @@ func handleTXT(r Reporter) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "text/plain") - renderer := render.Map{Selector: report.SelectEndpoint, Mapper: mapFunc(req), Pseudo: nil} + renderer := render.LeafMap{Selector: report.SelectEndpoint, Mapper: mapFunc(req), Pseudo: nil} dot(w, renderer.Render(r.Report())) //report.Render(r.Report(), report.SelectEndpoint, mapFunc(req), report.NoPseudoNode)) @@ -35,7 +35,7 @@ func handleSVG(r Reporter) http.HandlerFunc { cmd.Stdout = w - renderer := render.Map{Selector: report.SelectEndpoint, Mapper: mapFunc(req), Pseudo: nil} + renderer := render.LeafMap{Selector: report.SelectEndpoint, Mapper: mapFunc(req), Pseudo: nil} dot(wc, renderer.Render(r.Report())) wc.Close() @@ -102,7 +102,7 @@ func engine(r *http.Request) string { return engine } -func mapFunc(r *http.Request) render.MapFunc { +func mapFunc(r *http.Request) render.LeafMapFunc { switch strings.ToLower(r.FormValue("map_func")) { case "hosts", "networkhost", "networkhostname": return render.NetworkHostname diff --git a/render/mapping.go b/render/mapping.go index e1973a85b..856209c1b 100644 --- a/render/mapping.go +++ b/render/mapping.go @@ -9,7 +9,8 @@ import ( const humanTheInternet = "the Internet" -func newRenderableNode(id, major, minor, rank string) RenderableNode { +// NewRenderableNode makes a new RenderableNode +func NewRenderableNode(id, major, minor, rank string) RenderableNode { return RenderableNode{ ID: id, LabelMajor: major, @@ -31,7 +32,7 @@ func newPseudoNode(id, major, minor string) RenderableNode { } } -// MapFunc is anything which can take an arbitrary NodeMetadata, which is +// LeafMapFunc is anything which can take an arbitrary NodeMetadata, which is // always one-to-one with nodes in a topology, and return a specific // representation of the referenced node, in the form of a node ID and a // human-readable major and minor labels. @@ -41,7 +42,7 @@ func newPseudoNode(id, major, minor string) RenderableNode { // // If the final output parameter is false, the node shall be omitted from the // rendered topology. -type MapFunc func(report.NodeMetadata) (RenderableNode, bool) +type LeafMapFunc func(report.NodeMetadata) (RenderableNode, bool) // PseudoFunc creates RenderableNode representing pseudo nodes given the dstNodeID. // The srcNode renderable node is essentially from MapFunc, representing one of @@ -49,6 +50,10 @@ type MapFunc func(report.NodeMetadata) (RenderableNode, bool) // node IDs prior to mapping. type PseudoFunc func(srcNodeID string, srcNode RenderableNode, dstNodeID string) (RenderableNode, bool) +// MapFunc is anything which can take an arbitrary RenderableNode and +// return another RenderableNode. +type MapFunc func(RenderableNode) (RenderableNode, bool) + // ProcessPID takes a node NodeMetadata from topology, and returns a // representation with the ID based on the process PID and the labels based on // the process name. @@ -59,7 +64,7 @@ func ProcessPID(m report.NodeMetadata) (RenderableNode, bool) { show = m["pid"] != "" && m["name"] != "" ) - return newRenderableNode(identifier, m["name"], minor, m["pid"]), show + return NewRenderableNode(identifier, m["name"], minor, m["pid"]), show } // ProcessName takes a node NodeMetadata from a topology, and returns a @@ -67,7 +72,7 @@ func ProcessPID(m report.NodeMetadata) (RenderableNode, bool) { // processes with the same name together). func ProcessName(m report.NodeMetadata) (RenderableNode, bool) { show := m["pid"] != "" && m["name"] != "" - return newRenderableNode(m["name"], m["name"], "", m["name"]), show + return NewRenderableNode(m["name"], m["name"], "", m["name"]), show } // MapEndpoint2Container maps endpoint topology nodes to the containers they run @@ -82,7 +87,7 @@ func MapEndpoint2Container(m report.NodeMetadata) (RenderableNode, bool) { id, major, minor, rank = m["docker_container_id"], "", m["domain"], "" } - return newRenderableNode(id, major, minor, rank), true + return NewRenderableNode(id, major, minor, rank), true } // MapContainerIdentity maps container topology node to container mapped nodes. @@ -94,7 +99,7 @@ func MapContainerIdentity(m report.NodeMetadata) (RenderableNode, bool) { id, major, minor, rank = m["docker_container_id"], m["docker_container_name"], m["domain"], m["docker_image_id"] } - return newRenderableNode(id, major, minor, rank), true + return NewRenderableNode(id, major, minor, rank), true } // ProcessContainerImage maps topology nodes to the container images they run @@ -108,7 +113,7 @@ func ProcessContainerImage(m report.NodeMetadata) (RenderableNode, bool) { id, major, minor, rank = m["docker_image_id"], m["docker_image_name"], "", m["docker_image_id"] } - return newRenderableNode(id, major, minor, rank), true + return NewRenderableNode(id, major, minor, rank), true } // NetworkHostname takes a node NodeMetadata and returns a representation @@ -125,7 +130,7 @@ func NetworkHostname(m report.NodeMetadata) (RenderableNode, bool) { domain = parts[1] } - return newRenderableNode(fmt.Sprintf("host:%s", name), parts[0], domain, parts[0]), name != "" + return NewRenderableNode(fmt.Sprintf("host:%s", name), parts[0], domain, parts[0]), name != "" } // GenericPseudoNode contains heuristics for building sensible pseudo nodes. diff --git a/render/mapping_test.go b/render/mapping_test.go index 7579e6074..5f1475fe8 100644 --- a/render/mapping_test.go +++ b/render/mapping_test.go @@ -10,7 +10,7 @@ import ( func TestUngroupedMapping(t *testing.T) { for i, c := range []struct { - f render.MapFunc + f render.LeafMapFunc id string meta report.NodeMetadata wantOK bool diff --git a/render/render.go b/render/render.go index f89843683..1fbc0637e 100644 --- a/render/render.go +++ b/render/render.go @@ -16,6 +16,21 @@ type Renderer interface { // other renderers type Reduce []Renderer +// Map is a Renderer which produces a set of RendererNodes from the set of +// RendererNodes produces by another Renderer +type Map struct { + MapFunc + Renderer +} + +// LeafMap is a Renderer which produces a set of RendererNodes from a report.Topology +// by using a map functions and topology selector. +type LeafMap struct { + Selector report.TopologySelector + Mapper LeafMapFunc + Pseudo PseudoFunc +} + // Render produces a set of RenderableNodes given a Report func (r Reduce) Render(rpt report.Report) RenderableNodes { result := RenderableNodes{} @@ -34,26 +49,93 @@ func (r Reduce) AggregateMetadata(rpt report.Report, localID, remoteID string) r return metadata } -// Map is a Renderer which produces a set of RendererNodes by using a -// Mapper functions and topology selector. -type Map struct { - Selector report.TopologySelector - Mapper MapFunc - Pseudo PseudoFunc -} - -// Render produces a set of RenderableNodes given a Report +// Render transforms a set of RendererNodes produces by another Renderer +// using a map function func (m Map) Render(rpt report.Report) RenderableNodes { - return Topology(m.Selector(rpt), m.Mapper, m.Pseudo) + output, _ := m.render(rpt) + return output } -// Topology transforms a given Topology into a set of RenderableNodes, which +func (m Map) render(rpt report.Report) (RenderableNodes, map[string]string) { + input := m.Renderer.Render(rpt) + output := RenderableNodes{} + mapped := map[string]string{} // input node ID -> output node ID + adjacencies := map[string]report.IDList{} // output node ID -> input node Adjacencies + + for _, inRenderable := range input { + outRenderable, ok := m.MapFunc(inRenderable) + if !ok { + continue + } + + existing, ok := output[outRenderable.ID] + if ok { + outRenderable.Merge(existing) + } + + output[outRenderable.ID] = outRenderable + mapped[inRenderable.ID] = outRenderable.ID + adjacencies[outRenderable.ID] = adjacencies[outRenderable.ID].Add(inRenderable.Adjacency...) + } + + // Rewrite Adjacency for new node IDs. + // NB we don't do pseudo nodes here; we assumer the input graph + // we properly-connected, and if the map func dropped a node, + // we drop links to it. + for outNodeID, inAdjacency := range adjacencies { + outAdjacency := report.MakeIDList() + for _, inAdjacent := range inAdjacency { + outAdjacent, ok := mapped[inAdjacent] + if ok { + outAdjacency = outAdjacency.Add(outAdjacent) + } + } + outNode := output[outNodeID] + outNode.Adjacency = outAdjacency + output[outNodeID] = outNode + } + + return output, mapped +} + +// AggregateMetadata gives the metadata of an edge from the perspective of the +// srcRenderableID. Since an edgeID can have multiple edges on the address +// level, it uses the supplied mapping function to translate address IDs to +// renderable node (mapped) IDs. +func (m Map) AggregateMetadata(rpt report.Report, srcRenderableID, dstRenderableID string) report.AggregateMetadata { + // First we need to map the ids in this layer into the ids in the underlying layer + _, mapped := m.render(rpt) // this maps from old -> new + inverted := map[string][]string{} // this maps from new -> old(s) + for k, v := range mapped { + existing := inverted[v] + existing = append(existing, k) + inverted[v] = existing + } + + // Now work out a slice of edges this edge is constructed from + oldEdges := []struct{ src, dst string }{} + for _, oldSrcID := range inverted[srcRenderableID] { + for _, oldDstID := range inverted[dstRenderableID] { + oldEdges = append(oldEdges, struct{ src, dst string }{oldSrcID, oldDstID}) + } + } + + // Now recurse for each old edge + output := report.AggregateMetadata{} + for _, edge := range oldEdges { + metadata := m.Renderer.AggregateMetadata(rpt, edge.src, edge.dst) + output.Merge(metadata) + } + return output +} + +// Render transforms a given Report into a set of RenderableNodes, which // the UI will render collectively as a graph. Note that a RenderableNode will // always be rendered with other nodes, and therefore contains limited detail. // -// RenderBy takes a a MapFunc, which defines how to group and label nodes. Npdes -// with the same mapped IDs will be merged. -func Topology(t report.Topology, mapFunc MapFunc, pseudoFunc PseudoFunc) RenderableNodes { +// Nodes with the same mapped IDs will be merged. +func (m LeafMap) Render(rpt report.Report) RenderableNodes { + t := m.Selector(rpt) nodes := RenderableNodes{} // Build a set of RenderableNodes for all non-pseudo probes, and an @@ -64,7 +146,7 @@ func Topology(t report.Topology, mapFunc MapFunc, pseudoFunc PseudoFunc) Rendera source2host = map[string]string{} // source node ID -> origin host ID ) for nodeID, metadata := range t.NodeMetadatas { - mapped, ok := mapFunc(metadata) + mapped, ok := m.Mapper(metadata) if !ok { continue } @@ -100,7 +182,7 @@ func Topology(t report.Topology, mapFunc MapFunc, pseudoFunc PseudoFunc) Rendera for _, dstNodeID := range dsts { dstRenderableID, ok := source2mapped[dstNodeID] if !ok { - pseudoNode, ok := pseudoFunc(srcNodeID, srcRenderableNode, dstNodeID) + pseudoNode, ok := m.Pseudo(srcNodeID, srcRenderableNode, dstNodeID) if !ok { continue } @@ -124,16 +206,12 @@ func Topology(t report.Topology, mapFunc MapFunc, pseudoFunc PseudoFunc) Rendera return nodes } -// AggregateMetadata produces an AggregateMetadata for a given edge -func (m Map) AggregateMetadata(rpt report.Report, localID, remoteID string) report.AggregateMetadata { - return edgeMetadata(m.Selector(rpt), m.Mapper, localID, remoteID).Transform() -} - -// EdgeMetadata gives the metadata of an edge from the perspective of the +// AggregateMetadata gives the metadata of an edge from the perspective of the // srcRenderableID. Since an edgeID can have multiple edges on the address // level, it uses the supplied mapping function to translate address IDs to // renderable node (mapped) IDs. -func edgeMetadata(t report.Topology, mapFunc MapFunc, srcRenderableID, dstRenderableID string) report.EdgeMetadata { +func (m LeafMap) AggregateMetadata(rpt report.Report, srcRenderableID, dstRenderableID string) report.AggregateMetadata { + t := m.Selector(rpt) metadata := report.EdgeMetadata{} for edgeID, edgeMeta := range t.EdgeMetadatas { src, dst, ok := report.ParseEdgeID(edgeID) @@ -142,16 +220,16 @@ func edgeMetadata(t report.Topology, mapFunc MapFunc, srcRenderableID, dstRender continue } if src != report.TheInternet { - mapped, _ := mapFunc(t.NodeMetadatas[src]) + mapped, _ := m.Mapper(t.NodeMetadatas[src]) src = mapped.ID } if dst != report.TheInternet { - mapped, _ := mapFunc(t.NodeMetadatas[dst]) + mapped, _ := m.Mapper(t.NodeMetadatas[dst]) dst = mapped.ID } if src == srcRenderableID && dst == dstRenderableID { metadata.Flatten(edgeMeta) } } - return metadata + return metadata.Transform() } diff --git a/render/render_test.go b/render/render_test.go index a144e6e75..c6e7a82a5 100644 --- a/render/render_test.go +++ b/render/render_test.go @@ -55,6 +55,109 @@ func TestReduceEdge(t *testing.T) { } } +func TestMapRender(t *testing.T) { + // 1. Check when we return false, the node gets filtered out + { + mapper := render.Map{ + MapFunc: func(nodes render.RenderableNode) (render.RenderableNode, bool) { + return render.RenderableNode{}, false + }, + Renderer: mockRenderer{RenderableNodes: render.RenderableNodes{ + "foo": {ID: "foo"}, + }}, + } + want := render.RenderableNodes{} + have := mapper.Render(report.MakeReport()) + if !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) + } + } + + // 2. Check we can remap two nodes into one + { + mapper := render.Map{ + MapFunc: func(nodes render.RenderableNode) (render.RenderableNode, bool) { + return render.RenderableNode{ID: "bar"}, true + }, + Renderer: mockRenderer{RenderableNodes: render.RenderableNodes{ + "foo": {ID: "foo"}, + "baz": {ID: "baz"}, + }}, + } + want := render.RenderableNodes{ + "bar": render.RenderableNode{ID: "bar"}, + } + have := mapper.Render(report.MakeReport()) + if !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) + } + } + + // 3. Check we can remap adjacencies + { + mapper := render.Map{ + MapFunc: func(nodes render.RenderableNode) (render.RenderableNode, bool) { + return render.RenderableNode{ID: "_" + nodes.ID}, true + }, + Renderer: mockRenderer{RenderableNodes: render.RenderableNodes{ + "foo": {ID: "foo", Adjacency: report.MakeIDList("baz")}, + "baz": {ID: "baz", Adjacency: report.MakeIDList("foo")}, + }}, + } + want := render.RenderableNodes{ + "_foo": {ID: "_foo", Adjacency: report.MakeIDList("_baz")}, + "_baz": {ID: "_baz", Adjacency: report.MakeIDList("_foo")}, + } + have := mapper.Render(report.MakeReport()) + if !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) + } + } +} + +func TestMapEdge(t *testing.T) { + selector := func(_ report.Report) report.Topology { + return report.Topology{ + NodeMetadatas: report.NodeMetadatas{ + "foo": report.NodeMetadata{"id": "foo"}, + "bar": report.NodeMetadata{"id": "bar"}, + }, + Adjacency: report.Adjacency{ + ">foo": report.MakeIDList("bar"), + ">bar": report.MakeIDList("foo"), + }, + EdgeMetadatas: report.EdgeMetadatas{ + "foo|bar": report.EdgeMetadata{WithBytes: true, BytesIngress: 1, BytesEgress: 2}, + "bar|foo": report.EdgeMetadata{WithBytes: true, BytesIngress: 3, BytesEgress: 4}, + }, + } + } + + identity := func(nmd report.NodeMetadata) (render.RenderableNode, bool) { + return render.NewRenderableNode(nmd["id"], "", "", ""), true + } + + mapper := render.Map{ + MapFunc: func(nodes render.RenderableNode) (render.RenderableNode, bool) { + return render.RenderableNode{ID: "_" + nodes.ID}, true + }, + Renderer: render.LeafMap{ + Selector: selector, + Mapper: identity, + Pseudo: nil, + }, + } + + want := report.AggregateMetadata{ + report.KeyBytesIngress: 1, + report.KeyBytesEgress: 2, + } + have := mapper.AggregateMetadata(report.MakeReport(), "_foo", "_bar") + if !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) + } +} + var ( clientHostID = "client.hostname.com" serverHostID = "server.hostname.com" @@ -287,7 +390,11 @@ func TestRenderByEndpointPID(t *testing.T) { Metadata: report.AggregateMetadata{}, }, } - have := render.Topology(rpt.Endpoint, render.ProcessPID, render.GenericPseudoNode) + have := render.LeafMap{ + Selector: report.SelectEndpoint, + Mapper: render.ProcessPID, + Pseudo: render.GenericPseudoNode, + }.Render(rpt) if !reflect.DeepEqual(want, have) { t.Error("\n" + diff(want, have)) } @@ -341,7 +448,11 @@ func TestRenderByEndpointPIDGrouped(t *testing.T) { Metadata: report.AggregateMetadata{}, }, } - have := render.Topology(rpt.Endpoint, render.ProcessName, render.GenericGroupedPseudoNode) + have := render.LeafMap{ + Selector: report.SelectEndpoint, + Mapper: render.ProcessName, + Pseudo: render.GenericGroupedPseudoNode, + }.Render(rpt) if !reflect.DeepEqual(want, have) { t.Error("\n" + diff(want, have)) } @@ -396,7 +507,11 @@ func TestRenderByNetworkHostname(t *testing.T) { Metadata: report.AggregateMetadata{}, }, } - have := render.Topology(rpt.Address, render.NetworkHostname, render.GenericPseudoNode) + have := render.LeafMap{ + Selector: report.SelectAddress, + Mapper: render.NetworkHostname, + Pseudo: render.GenericPseudoNode, + }.Render(rpt) if !reflect.DeepEqual(want, have) { t.Error("\n" + diff(want, have)) }