From e5117a652f2730eff9aa978dd6d28e9ca5fc1567 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sun, 17 Dec 2017 15:40:26 +0000 Subject: [PATCH 1/6] remove superfluous fixupAdjacencies calls fixupAdjacencies fixes up adjancencies of mapped nodes. The nodes at the call sites we are removing aren't mapped. --- render/container.go | 1 - render/process.go | 1 - 2 files changed, 2 deletions(-) diff --git a/render/container.go b/render/container.go index ff0745d96..6a06e91c0 100644 --- a/render/container.go +++ b/render/container.go @@ -100,7 +100,6 @@ func (c connectionJoin) Render(rpt report.Report) Nodes { } } ret.copyUnmatched(inputNodes) - ret.fixupAdjacencies(inputNodes) ret.fixupAdjacencies(endpoints) return ret.result() } diff --git a/render/process.go b/render/process.go index a4f2fdef2..963848113 100644 --- a/render/process.go +++ b/render/process.go @@ -116,7 +116,6 @@ func (e endpoints2Processes) Render(rpt report.Report) Nodes { } } ret.copyUnmatched(processes) - ret.fixupAdjacencies(processes) ret.fixupAdjacencies(endpoints) return ret.result() } From f16908aea968ca05f819b1f25dbb71433ca63b2f Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sun, 17 Dec 2017 15:55:21 +0000 Subject: [PATCH 2/6] refactor: fix up adjacencies as part of joinResult.result() --- render/container.go | 3 +-- render/host.go | 6 ++---- render/process.go | 6 ++---- render/render.go | 26 ++++++++++++-------------- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/render/container.go b/render/container.go index 6a06e91c0..9d743e181 100644 --- a/render/container.go +++ b/render/container.go @@ -100,8 +100,7 @@ func (c connectionJoin) Render(rpt report.Report) Nodes { } } ret.copyUnmatched(inputNodes) - ret.fixupAdjacencies(endpoints) - return ret.result() + return ret.result(endpoints) } // FilterEmpty is a Renderer which filters out nodes which have no children diff --git a/render/host.go b/render/host.go index 93a2c1ad6..85c3d9038 100644 --- a/render/host.go +++ b/render/host.go @@ -40,8 +40,7 @@ func nodes2Hosts(nodes Nodes) Nodes { }) } } - ret.fixupAdjacencies(nodes) - return ret.result() + return ret.result(nodes) } // endpoints2Hosts takes nodes from the endpoint topology and produces @@ -68,6 +67,5 @@ func (e endpoints2Hosts) Render(rpt report.Report) Nodes { }) } } - ret.fixupAdjacencies(endpoints) - return ret.result() + return ret.result(endpoints) } diff --git a/render/process.go b/render/process.go index 963848113..450ddbfa6 100644 --- a/render/process.go +++ b/render/process.go @@ -116,8 +116,7 @@ func (e endpoints2Processes) Render(rpt report.Report) Nodes { } } ret.copyUnmatched(processes) - ret.fixupAdjacencies(endpoints) - return ret.result() + return ret.result(endpoints) } // When there is more than one connection originating from a source @@ -161,6 +160,5 @@ func processes2Names(processes Nodes) Nodes { }) } } - ret.fixupAdjacencies(processes) - return ret.result() + return ret.result(processes) } diff --git a/render/render.go b/render/render.go index bcf4c7738..3f12102bb 100644 --- a/render/render.go +++ b/render/render.go @@ -202,13 +202,22 @@ func (ret *joinResults) addChildAndChildren(m report.Node, id string, create fun // Add a copy of n straight into the results func (ret *joinResults) passThrough(n report.Node) { - n.Adjacency = nil // fixupAdjacencies assumes all nodes start with blank lists + n.Adjacency = nil // result() assumes all nodes start with blank lists ret.nodes[n.ID] = n ret.mapped[n.ID] = n.ID } -// Rewrite Adjacency for new nodes in ret for original nodes in input -func (ret *joinResults) fixupAdjacencies(input Nodes) { +func (ret *joinResults) copyUnmatched(input Nodes) { + for _, n := range input.Nodes { + if _, found := ret.nodes[n.ID]; !found { + ret.nodes[n.ID] = n + } + } +} + +// Rewrite Adjacency of nodes in ret mapped from original nodes in +// input, and return the result. +func (ret *joinResults) result(input Nodes) Nodes { for _, n := range input.Nodes { outID, ok := ret.mapped[n.ID] if !ok { @@ -224,17 +233,6 @@ func (ret *joinResults) fixupAdjacencies(input Nodes) { } ret.nodes[outID] = out } -} - -func (ret *joinResults) copyUnmatched(input Nodes) { - for _, n := range input.Nodes { - if _, found := ret.nodes[n.ID]; !found { - ret.nodes[n.ID] = n - } - } -} - -func (ret *joinResults) result() Nodes { return Nodes{Nodes: ret.nodes} } From 4dde1ce7151e1ebd53e3f9d9bc23fb8bb7b16ca7 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sun, 17 Dec 2017 16:08:55 +0000 Subject: [PATCH 3/6] optimise and align endpoints2Hosts Avoid a reduce step by joining the host topology in endpoints2Hosts. This is similar to what we do in connectionJoin and endpoints2Processes. --- render/host.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/render/host.go b/render/host.go index 85c3d9038..552c0b194 100644 --- a/render/host.go +++ b/render/host.go @@ -9,12 +9,11 @@ import ( // // not memoised var HostRenderer = MakeReduce( - endpoints2Hosts{}, CustomRenderer{RenderFunc: nodes2Hosts, Renderer: ProcessRenderer}, CustomRenderer{RenderFunc: nodes2Hosts, Renderer: ContainerRenderer}, CustomRenderer{RenderFunc: nodes2Hosts, Renderer: ContainerImageRenderer}, CustomRenderer{RenderFunc: nodes2Hosts, Renderer: PodRenderer}, - SelectHost, + endpoints2Hosts{}, ) // nodes2Hosts maps any Nodes to host Nodes. @@ -50,6 +49,7 @@ type endpoints2Hosts struct { func (e endpoints2Hosts) Render(rpt report.Report) Nodes { local := LocalNetworks(rpt) + hosts := SelectHost.Render(rpt) endpoints := SelectEndpoint.Render(rpt) ret := newJoinResults() @@ -62,10 +62,16 @@ func (e endpoints2Hosts) Render(rpt report.Report) Nodes { } else { id := report.MakeHostNodeID(report.ExtractHostID(n)) ret.addChild(n, id, func(id string) report.Node { + if hostNode, found := hosts.Nodes[id]; found { + return hostNode + } + // we have a hostNodeID, but no matching host node; + // create a new one rather than dropping the data return report.MakeNode(id).WithTopology(report.Host). WithLatest(report.HostNodeID, timestamp, hostNodeID) }) } } + ret.copyUnmatched(hosts) return ret.result(endpoints) } From ac87c2b6e819661de02b59287704a9f08dde3b8c Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sun, 17 Dec 2017 16:32:04 +0000 Subject: [PATCH 4/6] optimise & simplify node propagation in joinResult Instead of copying unmatched nodes at the end, and matched nodes when we encounter them, copy *all* nodes at the beginning. --- render/container.go | 3 +-- render/host.go | 5 ++--- render/process.go | 5 ++--- render/render.go | 16 ++++++---------- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/render/container.go b/render/container.go index 9d743e181..540e284c0 100644 --- a/render/container.go +++ b/render/container.go @@ -71,7 +71,7 @@ func (c connectionJoin) Render(rpt report.Report) Nodes { } } } - ret := newJoinResults() + ret := newJoinResults(inputNodes.Nodes) // Now look at all the endpoints and see which map to IP nodes for _, m := range endpoints.Nodes { @@ -99,7 +99,6 @@ func (c connectionJoin) Render(rpt report.Report) Nodes { }) } } - ret.copyUnmatched(inputNodes) return ret.result(endpoints) } diff --git a/render/host.go b/render/host.go index 552c0b194..d8f39205f 100644 --- a/render/host.go +++ b/render/host.go @@ -26,7 +26,7 @@ var HostRenderer = MakeReduce( // not have enough info to do that, and the resulting graph must be // merged with a host graph to get that info. func nodes2Hosts(nodes Nodes) Nodes { - ret := newJoinResults() + ret := newJoinResults(nil) for _, n := range nodes.Nodes { if n.Topology == Pseudo { @@ -51,7 +51,7 @@ func (e endpoints2Hosts) Render(rpt report.Report) Nodes { local := LocalNetworks(rpt) hosts := SelectHost.Render(rpt) endpoints := SelectEndpoint.Render(rpt) - ret := newJoinResults() + ret := newJoinResults(hosts.Nodes) for _, n := range endpoints.Nodes { // Nodes without a hostid are treated as pseudo nodes @@ -72,6 +72,5 @@ func (e endpoints2Hosts) Render(rpt report.Report) Nodes { }) } } - ret.copyUnmatched(hosts) return ret.result(endpoints) } diff --git a/render/process.go b/render/process.go index 450ddbfa6..bd5c321dc 100644 --- a/render/process.go +++ b/render/process.go @@ -86,7 +86,7 @@ func (e endpoints2Processes) Render(rpt report.Report) Nodes { local := LocalNetworks(rpt) processes := SelectProcess.Render(rpt) endpoints := SelectEndpoint.Render(rpt) - ret := newJoinResults() + ret := newJoinResults(processes.Nodes) for _, n := range endpoints.Nodes { // Nodes without a hostid are treated as pseudo nodes @@ -115,7 +115,6 @@ func (e endpoints2Processes) Render(rpt report.Report) Nodes { }) } } - ret.copyUnmatched(processes) return ret.result(endpoints) } @@ -148,7 +147,7 @@ func hasMoreThanOneConnection(n report.Node, endpoints report.Nodes) bool { // processes2Names maps process Nodes to Nodes for each process name. func processes2Names(processes Nodes) Nodes { - ret := newJoinResults() + ret := newJoinResults(nil) for _, n := range processes.Nodes { if n.Topology == Pseudo { diff --git a/render/render.go b/render/render.go index 3f12102bb..72b3f8d30 100644 --- a/render/render.go +++ b/render/render.go @@ -166,8 +166,12 @@ type joinResults struct { mapped map[string]string // input node ID -> output node ID } -func newJoinResults() joinResults { - return joinResults{nodes: make(report.Nodes), mapped: map[string]string{}} +func newJoinResults(inputNodes report.Nodes) joinResults { + nodes := make(report.Nodes, len(inputNodes)) + for id, n := range inputNodes { + nodes[id] = n + } + return joinResults{nodes: nodes, mapped: map[string]string{}} } // Add m as a child of the node at id, creating a new result node if @@ -207,14 +211,6 @@ func (ret *joinResults) passThrough(n report.Node) { ret.mapped[n.ID] = n.ID } -func (ret *joinResults) copyUnmatched(input Nodes) { - for _, n := range input.Nodes { - if _, found := ret.nodes[n.ID]; !found { - ret.nodes[n.ID] = n - } - } -} - // Rewrite Adjacency of nodes in ret mapped from original nodes in // input, and return the result. func (ret *joinResults) result(input Nodes) Nodes { From a6f24fc151f7c1e20ca2bc0050735663e4c41ede Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sun, 17 Dec 2017 16:55:38 +0000 Subject: [PATCH 5/6] remove superfluous lookups Since we seed the joinResult with the nodes from the topology we are mapping to, we know the 'create' function is only called when there is no node with the specified id. This neatly makes the 'create' function only do what it says, i.e. return _new_ nodes. --- render/container.go | 5 ++--- render/host.go | 3 --- render/process.go | 6 ++---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/render/container.go b/render/container.go index 540e284c0..219a76d9c 100644 --- a/render/container.go +++ b/render/container.go @@ -94,9 +94,8 @@ func (c connectionJoin) Render(rpt report.Report) Nodes { id, found = ipNodes[report.MakeScopedEndpointNodeID(scope, addr, port)] } if found && id != "" { // not one we blanked out earlier - ret.addChild(m, id, func(id string) report.Node { - return inputNodes.Nodes[id] - }) + // We are guaranteed to find the id, so no need to pass a node constructor. + ret.addChild(m, id, nil) } } return ret.result(endpoints) diff --git a/render/host.go b/render/host.go index d8f39205f..180f1115d 100644 --- a/render/host.go +++ b/render/host.go @@ -62,9 +62,6 @@ func (e endpoints2Hosts) Render(rpt report.Report) Nodes { } else { id := report.MakeHostNodeID(report.ExtractHostID(n)) ret.addChild(n, id, func(id string) report.Node { - if hostNode, found := hosts.Nodes[id]; found { - return hostNode - } // we have a hostNodeID, but no matching host node; // create a new one rather than dropping the data return report.MakeNode(id).WithTopology(report.Host). diff --git a/render/process.go b/render/process.go index bd5c321dc..f1ca648e2 100644 --- a/render/process.go +++ b/render/process.go @@ -106,10 +106,8 @@ func (e endpoints2Processes) Render(rpt report.Report) Nodes { hostID, _, _ := report.ParseNodeID(hostNodeID) id := report.MakeProcessNodeID(hostID, pid) ret.addChild(n, id, func(id string) report.Node { - if processNode, found := processes.Nodes[id]; found { - return processNode - } - // we have a pid, but no matching process node; create a new one rather than dropping the data + // we have a pid, but no matching process node; + // create a new one rather than dropping the data return report.MakeNode(id).WithTopology(report.Process). WithLatest(process.PID, timestamp, pid) }) From ba7af78cfa6a926bf055811fc9be0cad8737cb41 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Sun, 17 Dec 2017 17:04:48 +0000 Subject: [PATCH 6/6] ensure result adjacencies start empty Any existing list would be mutated by result(), which is bad. Note that all the existing newJoinResults() call sites pass in nodes with no adjacencies, so this is purely a safety measure. --- render/render.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/render/render.go b/render/render.go index 72b3f8d30..b54115f42 100644 --- a/render/render.go +++ b/render/render.go @@ -169,6 +169,7 @@ type joinResults struct { func newJoinResults(inputNodes report.Nodes) joinResults { nodes := make(report.Nodes, len(inputNodes)) for id, n := range inputNodes { + n.Adjacency = nil // result() assumes all nodes start with no adjacencies nodes[id] = n } return joinResults{nodes: nodes, mapped: map[string]string{}} @@ -206,7 +207,7 @@ func (ret *joinResults) addChildAndChildren(m report.Node, id string, create fun // Add a copy of n straight into the results func (ret *joinResults) passThrough(n report.Node) { - n.Adjacency = nil // result() assumes all nodes start with blank lists + n.Adjacency = nil // result() assumes all nodes start with no adjacencies ret.nodes[n.ID] = n ret.mapped[n.ID] = n.ID }