From e7dc25811968a2ba84eaed52ecc69ba6779e9aad Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Mon, 24 Aug 2015 12:52:57 +0000 Subject: [PATCH] Only group processes not in containers AND doing network IO into uncontained nodes. --- app/api_topology_test.go | 2 +- app/router.go | 4 +-- render/expected/expected.go | 23 ++++++++----- render/render.go | 66 +++++++++++++++++++++++++++++++++---- render/render_test.go | 19 +++++------ render/topologies.go | 18 ++++++++-- test/report_fixture.go | 10 ++++++ 7 files changed, 111 insertions(+), 31 deletions(-) diff --git a/app/api_topology_test.go b/app/api_topology_test.go index 56fea5b61..7edf67430 100644 --- a/app/api_topology_test.go +++ b/app/api_topology_test.go @@ -180,7 +180,7 @@ func TestAPITopologyWebsocket(t *testing.T) { if err := json.Unmarshal(p, &d); err != nil { t.Fatalf("JSON parse error: %s", err) } - equals(t, 6, len(d.Add)) + equals(t, 7, len(d.Add)) equals(t, 0, len(d.Update)) equals(t, 0, len(d.Remove)) } diff --git a/app/router.go b/app/router.go index e7480d43a..a8f7b8702 100644 --- a/app/router.go +++ b/app/router.go @@ -127,12 +127,12 @@ var topologyRegistry = map[string]topologyView{ "applications": { human: "Applications", parent: "", - renderer: render.FilterUnconnected{Renderer: render.ProcessWithContainerNameRenderer{}}, + renderer: render.FilterUnconnected(render.ProcessWithContainerNameRenderer{}), }, "applications-by-name": { human: "by name", parent: "applications", - renderer: render.FilterUnconnected{Renderer: render.ProcessNameRenderer}, + renderer: render.FilterUnconnected(render.ProcessNameRenderer), }, "containers": { human: "Containers", diff --git a/render/expected/expected.go b/render/expected/expected.go index 7349fdb7a..1b2ffb01b 100644 --- a/render/expected/expected.go +++ b/render/expected/expected.go @@ -59,6 +59,7 @@ var ( Adjacency: adjacency, Origins: report.MakeIDList( test.RandomClientNodeID, + test.GoogleEndpointNodeID, ), } } @@ -124,14 +125,15 @@ var ( }, nonContainerProcessID: { ID: nonContainerProcessID, - LabelMajor: "bash", + LabelMajor: test.NonContainerComm, LabelMinor: fmt.Sprintf("%s (%s)", test.ServerHostID, test.NonContainerPID), Rank: test.NonContainerComm, Pseudo: false, - Adjacency: report.MakeIDList(), + Adjacency: report.MakeIDList(render.TheInternetID), Origins: report.MakeIDList( test.NonContainerProcessNodeID, test.ServerHostNodeID, + test.NonContainerNodeID, ), NodeMetadata: report.MakeNodeMetadata(), EdgeMetadata: report.EdgeMetadata{}, @@ -180,16 +182,17 @@ var ( EgressByteCount: newu64(2100), }, }, - "bash": { - ID: "bash", - LabelMajor: "bash", + test.NonContainerComm: { + ID: test.NonContainerComm, + LabelMajor: test.NonContainerComm, LabelMinor: "1 process", - Rank: "bash", + Rank: test.NonContainerComm, Pseudo: false, - Adjacency: report.MakeIDList(), + Adjacency: report.MakeIDList(render.TheInternetID), Origins: report.MakeIDList( test.NonContainerProcessNodeID, test.ServerHostNodeID, + test.NonContainerNodeID, ), NodeMetadata: report.MakeNodeMetadata(), EdgeMetadata: report.EdgeMetadata{}, @@ -246,10 +249,11 @@ var ( LabelMinor: test.ServerHostName, Rank: "", Pseudo: true, - Adjacency: report.MakeIDList(), + Adjacency: report.MakeIDList(render.TheInternetID), Origins: report.MakeIDList( test.NonContainerProcessNodeID, test.ServerHostNodeID, + test.NonContainerNodeID, ), NodeMetadata: report.MakeNodeMetadata(), EdgeMetadata: report.EdgeMetadata{}, @@ -305,8 +309,9 @@ var ( LabelMinor: test.ServerHostName, Rank: "", Pseudo: true, - Adjacency: report.MakeIDList(), + Adjacency: report.MakeIDList(render.TheInternetID), Origins: report.MakeIDList( + test.NonContainerNodeID, test.NonContainerProcessNodeID, test.ServerHostNodeID, ), diff --git a/render/render.go b/render/render.go index 5990885d8..bc551ef52 100644 --- a/render/render.go +++ b/render/render.go @@ -269,27 +269,79 @@ func (m LeafMap) EdgeMetadata(rpt report.Report, srcRenderableID, dstRenderableI return metadata } -// FilterUnconnected is a Renderer which filters out unconnected nodes. -type FilterUnconnected struct { +// CustomRenderer allow for mapping functions that recived the entire topology +// in one call - useful for functions that need to consider the entire graph +type CustomRenderer struct { + RenderFunc func(RenderableNodes) RenderableNodes Renderer } -// Render produces a set of RenderableNodes given a Report -func (f FilterUnconnected) Render(rpt report.Report) RenderableNodes { - return OnlyConnected(f.Renderer.Render(rpt)) +// Render implements Renderer +func (c CustomRenderer) Render(rpt report.Report) RenderableNodes { + return c.RenderFunc(c.Renderer.Render(rpt)) } +// IsConnected is the key added to NodeMetadata by ColorConnected +// to indicate a node has an edge pointing to it or from it +const IsConnected = "is_connected" + // OnlyConnected filters out unconnected RenderedNodes func OnlyConnected(input RenderableNodes) RenderableNodes { output := RenderableNodes{} + for id, node := range ColorConnected(input) { + if _, ok := node.NodeMetadata.Metadata[IsConnected]; ok { + output[id] = node + } + } + return output +} + +// FilterUnconnected produces a renderer that filters unconnected nodes +// from the given renderer +func FilterUnconnected(r Renderer) Renderer { + return CustomRenderer{ + RenderFunc: OnlyConnected, + Renderer: r, + } +} + +// ColorConnected colors nodes with the IsConnected key if +// they have edges to or from them. +func ColorConnected(input RenderableNodes) RenderableNodes { + connected := map[string]struct{}{} + void := struct{}{} + for id, node := range input { if len(node.Adjacency) == 0 { continue } - output[id] = node + connected[id] = void for _, id := range node.Adjacency { - output[id] = input[id] + connected[id] = void + } + } + + for id := range connected { + node := input[id] + node.NodeMetadata.Metadata[IsConnected] = "true" + input[id] = node + } + return input +} + +// Filter removes nodes from a view based on a predicate. +type Filter struct { + Renderer + f func(RenderableNode) bool +} + +// Render implements Renderer +func (f Filter) Render(rpt report.Report) RenderableNodes { + output := RenderableNodes{} + for id, node := range f.Renderer.Render(rpt) { + if f.f(node) { + output[id] = node } } return output diff --git a/render/render_test.go b/render/render_test.go index 8371edaa6..bc69c90ff 100644 --- a/render/render_test.go +++ b/render/render_test.go @@ -149,18 +149,17 @@ func TestMapEdge(t *testing.T) { } func TestFilterRender(t *testing.T) { - renderer := render.FilterUnconnected{ - Renderer: mockRenderer{RenderableNodes: render.RenderableNodes{ - "foo": {ID: "foo", Adjacency: report.MakeIDList("bar")}, - "bar": {ID: "bar", Adjacency: report.MakeIDList("foo")}, - "baz": {ID: "baz", Adjacency: report.MakeIDList()}, - }}, - } + renderer := render.FilterUnconnected( + mockRenderer{RenderableNodes: render.RenderableNodes{ + "foo": {ID: "foo", Adjacency: report.MakeIDList("bar"), NodeMetadata: report.MakeNodeMetadata()}, + "bar": {ID: "bar", Adjacency: report.MakeIDList("foo"), NodeMetadata: report.MakeNodeMetadata()}, + "baz": {ID: "baz", Adjacency: report.MakeIDList(), NodeMetadata: report.MakeNodeMetadata()}, + }}) want := render.RenderableNodes{ - "foo": {ID: "foo", Adjacency: report.MakeIDList("bar")}, - "bar": {ID: "bar", Adjacency: report.MakeIDList("foo")}, + "foo": {ID: "foo", Adjacency: report.MakeIDList("bar"), NodeMetadata: report.MakeNodeMetadata()}, + "bar": {ID: "bar", Adjacency: report.MakeIDList("foo"), NodeMetadata: report.MakeNodeMetadata()}, } - have := renderer.Render(report.MakeReport()) + have := sterilize(renderer.Render(report.MakeReport()), true) if !reflect.DeepEqual(want, have) { t.Errorf("want %+v, have %+v", want, have) } diff --git a/render/topologies.go b/render/topologies.go index 6f6bb079e..f078dfeea 100644 --- a/render/topologies.go +++ b/render/topologies.go @@ -82,8 +82,22 @@ var ProcessNameRenderer = Map{ // graph by merging the process graph and the container topology. var ContainerRenderer = MakeReduce( Map{ - MapFunc: MapProcess2Container, - Renderer: ProcessRenderer, + MapFunc: MapProcess2Container, + + // We only want processes in container _or_ processes with network connections + // but we need to be careful to ensure we only include each edge once, by only + // including the ProcessRenderer once. + Renderer: Filter{ + f: func(n RenderableNode) bool { + _, inContainer := n.NodeMetadata.Metadata[docker.ContainerID] + _, isConnected := n.NodeMetadata.Metadata[IsConnected] + return inContainer || isConnected + }, + Renderer: CustomRenderer{ + RenderFunc: ColorConnected, + Renderer: ProcessRenderer, + }, + }, }, LeafMap{ Selector: report.SelectContainer, diff --git a/test/report_fixture.go b/test/report_fixture.go index cb9ac84a4..aad44c11e 100644 --- a/test/report_fixture.go +++ b/test/report_fixture.go @@ -28,6 +28,7 @@ var ( UnknownClient2IP = "10.10.10.10" UnknownClient3IP = "10.10.10.11" RandomClientIP = "51.52.53.54" + GoogleIP = "8.8.8.8" ClientHostName = ClientHostID ServerHostName = ServerHostID @@ -52,6 +53,8 @@ var ( UnknownClient2NodeID = report.MakeEndpointNodeID(ServerHostID, UnknownClient2IP, "54020") // to the same server, are deduped. UnknownClient3NodeID = report.MakeEndpointNodeID(ServerHostID, UnknownClient3IP, "54020") // Check this one isn't deduped RandomClientNodeID = report.MakeEndpointNodeID(ServerHostID, RandomClientIP, "12345") // this should become an internet node + NonContainerNodeID = report.MakeEndpointNodeID(ServerHostID, ServerIP, "46789") + GoogleEndpointNodeID = report.MakeEndpointNodeID(ServerHostID, GoogleIP, "80") ClientProcess1NodeID = report.MakeProcessNodeID(ClientHostID, Client1PID) ClientProcess2NodeID = report.MakeProcessNodeID(ClientHostID, Client2PID) @@ -85,6 +88,7 @@ var ( report.MakeAdjacencyID(UnknownClient2NodeID): report.MakeIDList(Server80NodeID), report.MakeAdjacencyID(UnknownClient3NodeID): report.MakeIDList(Server80NodeID), report.MakeAdjacencyID(RandomClientNodeID): report.MakeIDList(Server80NodeID), + report.MakeAdjacencyID(NonContainerNodeID): report.MakeIDList(GoogleEndpointNodeID), }, NodeMetadatas: report.NodeMetadatas{ // NodeMetadata is arbitrary. We're free to put only precisely what we @@ -108,6 +112,12 @@ var ( process.PID: ServerPID, report.HostNodeID: ServerHostNodeID, }), + NonContainerNodeID: report.MakeNodeMetadataWith(map[string]string{ + endpoint.Addr: ServerIP, + endpoint.Port: "46789", + process.PID: NonContainerPID, + report.HostNodeID: ServerHostNodeID, + }), }, EdgeMetadatas: report.EdgeMetadatas{ report.MakeEdgeID(Client54001NodeID, Server80NodeID): report.EdgeMetadata{