From 8a116979578195dd7c8867c71cf3dd3e12430d9c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 18 Oct 2017 18:23:38 +0000 Subject: [PATCH 01/12] Add BenchmarkTopologyHosts() and BenchmarkTopologyContainers() To time a single topology, for more focused optimisation --- app/benchmark_internal_test.go | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/app/benchmark_internal_test.go b/app/benchmark_internal_test.go index 493244e29..9eae97bfb 100644 --- a/app/benchmark_internal_test.go +++ b/app/benchmark_internal_test.go @@ -32,19 +32,43 @@ func loadReport() (report.Report, error) { } func BenchmarkTopologyList(b *testing.B) { + benchmarkRender(b, func(report report.Report) { + request := &http.Request{ + Form: url.Values{}, + } + topologyRegistry.renderTopologies(report, request) + }) +} + +func benchmarkRender(b *testing.B, f func(report.Report)) { report, err := loadReport() if err != nil { b.Fatal(err) } b.ReportAllocs() b.ResetTimer() - request := &http.Request{ - Form: url.Values{}, - } for i := 0; i < b.N; i++ { b.StopTimer() render.ResetCache() b.StartTimer() - topologyRegistry.renderTopologies(report, request) + f(report) } } + +func BenchmarkTopologyHosts(b *testing.B) { + benchmarkOneTopology(b, "hosts") +} + +func BenchmarkTopologyContainers(b *testing.B) { + benchmarkOneTopology(b, "containers") +} + +func benchmarkOneTopology(b *testing.B, topologyID string) { + benchmarkRender(b, func(report report.Report) { + renderer, decorator, err := topologyRegistry.RendererForTopology(topologyID, url.Values{}, report) + if err != nil { + b.Fatal(err) + } + renderer.Render(report, decorator) + }) +} From d230846b95debc90305c52936349b9ad14c63823 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 17 Oct 2017 17:45:34 +0000 Subject: [PATCH 02/12] Refactor: extract externalNodeID function --- render/id.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/render/id.go b/render/id.go index 3f11bc563..e482f50bf 100644 --- a/render/id.go +++ b/render/id.go @@ -38,11 +38,20 @@ func NewDerivedPseudoNode(id string, node report.Node) report.Node { // NewDerivedExternalNode figures out if a node should be considered external and creates the corresponding pseudo node func NewDerivedExternalNode(n report.Node, addr string, local report.Networks) (report.Node, bool) { + id, ok := externalNodeID(n, addr, local) + if !ok { + return report.Node{}, false + } + return NewDerivedPseudoNode(id, n), true +} + +// figure out if a node should be considered external and returns an ID which can be used to create a pseudo node +func externalNodeID(n report.Node, addr string, local report.Networks) (string, bool) { // First, check if it's a known service and emit a a specific node if it // is. This needs to be done before checking IPs since known services can // live in the same network, see https://github.com/weaveworks/scope/issues/2163 if hostname, found := DNSFirstMatch(n, isKnownService); found { - return NewDerivedPseudoNode(ServiceNodeIDPrefix+hostname, n), true + return ServiceNodeIDPrefix + hostname, true } // If the dstNodeAddr is not in a network local to this report, we emit an @@ -50,13 +59,13 @@ func NewDerivedExternalNode(n report.Node, addr string, local report.Networks) ( if ip := net.ParseIP(addr); ip != nil && !local.Contains(ip) { // emit one internet node for incoming, one for outgoing if len(n.Adjacency) > 0 { - return NewDerivedPseudoNode(IncomingInternetID, n), true + return IncomingInternetID, true } - return NewDerivedPseudoNode(OutgoingInternetID, n), true + return OutgoingInternetID, true } // The node is not external - return report.Node{}, false + return "", false } // DNSFirstMatch returns the first DNS name where match() returns From 1b3e40ccb582a5ddee7320727813c274d1a2c250 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 17 Oct 2017 18:13:54 +0000 Subject: [PATCH 03/12] Refactor: extract pseudoNodeID function --- render/id.go | 15 +++++++++++++++ render/process.go | 12 +++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/render/id.go b/render/id.go index e482f50bf..441638d9d 100644 --- a/render/id.go +++ b/render/id.go @@ -45,6 +45,21 @@ func NewDerivedExternalNode(n report.Node, addr string, local report.Networks) ( return NewDerivedPseudoNode(id, n), true } +func pseudoNodeID(n report.Node, local report.Networks) (string, bool) { + _, addr, _, ok := report.ParseEndpointNodeID(n.ID) + if !ok { + return "", false + } + + if id, ok := externalNodeID(n, addr, local); ok { + return id, ok + } + + // due to https://github.com/weaveworks/scope/issues/1323 we are dropping + // all non-external pseudo nodes for now. + return "", false +} + // figure out if a node should be considered external and returns an ID which can be used to create a pseudo node func externalNodeID(n report.Node, addr string, local report.Networks) (string, bool) { // First, check if it's a known service and emit a a specific node if it diff --git a/render/process.go b/render/process.go index 5093ed3ec..724e20b47 100644 --- a/render/process.go +++ b/render/process.go @@ -91,18 +91,12 @@ var ProcessNameRenderer = ConditionalRenderer(renderProcesses, // MapEndpoint2Pseudo makes internet of host pesudo nodes from a endpoint node. func MapEndpoint2Pseudo(n report.Node, local report.Networks) report.Nodes { - _, addr, _, ok := report.ParseEndpointNodeID(n.ID) + id, ok := pseudoNodeID(n, local) if !ok { return report.Nodes{} } - - if externalNode, ok := NewDerivedExternalNode(n, addr, local); ok { - return report.Nodes{externalNode.ID: externalNode} - } - - // due to https://github.com/weaveworks/scope/issues/1323 we are dropping - // all non-external pseudo nodes for now. - return report.Nodes{} + externalNode := NewDerivedPseudoNode(id, n) + return report.Nodes{externalNode.ID: externalNode} } // MapEndpoint2Process maps endpoint Nodes to process From b684e3c6fc8197141e8d9ac5079faf392263015e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 17 Oct 2017 18:15:39 +0000 Subject: [PATCH 04/12] Rewrite MapEndpoint2Host as a Renderer This allows us to avoid creating a host of 'IP' type Nodes then discarding them after matching; instead we match directly and create just the result we want. --- render/host.go | 81 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/render/host.go b/render/host.go index 8983eb05a..7d374bff6 100644 --- a/render/host.go +++ b/render/host.go @@ -9,10 +9,7 @@ import ( // // not memoised var HostRenderer = MakeReduce( - MakeMap( - MapEndpoint2Host, - EndpointRenderer, - ), + endpoints2Hosts{}, MakeMap( MapX2Host, ColorConnectedProcessRenderer, @@ -64,18 +61,68 @@ func MapX2Host(n report.Node, _ report.Networks) report.Nodes { return result } -// MapEndpoint2Host takes nodes from the endpoint topology and produces +// endpoints2Hosts takes nodes from the endpoint topology and produces // host nodes or pseudo nodes. -func MapEndpoint2Host(n report.Node, local report.Networks) report.Nodes { - // Nodes without a hostid are treated as pseudo nodes - hostNodeID, timestamp, ok := n.Latest.LookupEntry(report.HostNodeID) - if !ok { - return MapEndpoint2Pseudo(n, local) - } - - id := report.MakeHostNodeID(report.ExtractHostID(n)) - result := NewDerivedNode(id, n).WithTopology(report.Host) - result.Latest = result.Latest.Set(report.HostNodeID, timestamp, hostNodeID) - result.Counters = result.Counters.Add(n.Topology, 1) - return report.Nodes{id: result} +type endpoints2Hosts struct { +} + +func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { + ns := SelectEndpoint.Render(rpt, dct) + local := LocalNetworks(rpt) + + var ret = make(report.Nodes) + var mapped = map[string]string{} // input node ID -> output node ID + for _, n := range ns { + var result report.Node + var exists bool + // Nodes without a hostid are treated as pseudo nodes + hostNodeID, timestamp, ok := n.Latest.LookupEntry(report.HostNodeID) + if !ok { + id, ok := pseudoNodeID(n, local) + if !ok { + continue + } + result, exists = ret[id] + if !exists { + result = report.MakeNode(id).WithTopology(Pseudo) + } + } else { + id := report.MakeHostNodeID(report.ExtractHostID(n)) + result, exists = ret[id] + if !exists { + result = report.MakeNode(id).WithTopology(report.Host) + result.Latest = result.Latest.Set(report.HostNodeID, timestamp, hostNodeID) + } + result.Children = result.Children.Merge(n.Children) + } + result.Children = result.Children.Add(n) + result.Counters = result.Counters.Add(n.Topology, 1) + ret[result.ID] = result + mapped[n.ID] = result.ID + } + fixupAdjancencies(ns, ret, mapped) + return ret +} + +// Rewrite Adjacency for new nodes in ret, original nodes in input, and mapping old->new IDs in mapped +func fixupAdjancencies(input, ret report.Nodes, mapped map[string]string) { + for _, n := range input { + outID, ok := mapped[n.ID] + if !ok { + continue + } + out := ret[outID] + // for each adjacency in the original node, find out what it maps to (if any), + // and add that to the new node + for _, a := range n.Adjacency { + if mappedDest, found := mapped[a]; found { + out.Adjacency = out.Adjacency.Add(mappedDest) + } + } + ret[outID] = out + } +} + +func (e endpoints2Hosts) Stats(rpt report.Report, _ Decorator) Stats { + return Stats{} // nothing to report } From e16aaf6c43ec92902928c552a744fc0294cdc9cb Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 19 Oct 2017 17:27:17 +0000 Subject: [PATCH 05/12] Merge nodeToIP, endpoints2Nodes and ipToNode into one Renderer This is much more efficient as we skip creating then merging all intermediate Nodes --- render/container.go | 125 +++++++++++++++++++++----------------------- render/host.go | 14 +++++ 2 files changed, 75 insertions(+), 64 deletions(-) diff --git a/render/container.go b/render/container.go index b832c35ce..6a5066132 100644 --- a/render/container.go +++ b/render/container.go @@ -39,85 +39,82 @@ var ContainerRenderer = Memoise(MakeFilter( ), )) -var mapEndpoint2IP = Memoise(MakeMap(endpoint2IP, SelectEndpoint)) - const originalNodeID = "original_node_id" // ConnectionJoin joins the given renderer with connections from the // endpoints topology, using the toIPs function to extract IPs from // the nodes. func ConnectionJoin(toIPs func(report.Node) []string, r Renderer) Renderer { - nodeToIP := func(n report.Node, _ report.Networks) report.Nodes { - result := report.Nodes{} - for _, ip := range toIPs(n) { - result[ip] = report.MakeNode(ip). - WithTopology(IP). - WithLatests(map[string]string{ - originalNodeID: n.ID, - }). - WithCounters(map[string]int{IP: 1}) - } - return result - } - - return MakeReduce( - MakeMap( - ipToNode, - MakeReduce( - MakeMap(nodeToIP, r), - mapEndpoint2IP, - ), - ), - r, - ) + return connectionJoin{toIPs: toIPs, r: r} } -func ipToNode(n report.Node, _ report.Networks) report.Nodes { - // propagate non-IP nodes - if n.Topology != IP { - return report.Nodes{n.ID: n} - } - // If an IP is shared between multiple nodes, we can't reliably - // attribute an connection based on its IP - if count, _ := n.Counters.Lookup(IP); count > 1 { - return report.Nodes{} - } - // If this node is not of the original type, exclude it. This - // excludes all the nodes we've dragged in from endpoint that we - // failed to join to a node. - id, ok := n.Latest.Lookup(originalNodeID) - if !ok { - return report.Nodes{} - } - - return report.Nodes{id: NewDerivedNode(id, n)} +type connectionJoin struct { + toIPs func(report.Node) []string + r Renderer } -// endpoint2IP maps endpoint nodes to their IP address, for joining -// with container nodes. -func endpoint2IP(m report.Node, local report.Networks) report.Nodes { - scope, addr, port, ok := report.ParseEndpointNodeID(m.ID) - if !ok { - return report.Nodes{} - } +func (c connectionJoin) Render(rpt report.Report, dct Decorator) report.Nodes { + local := LocalNetworks(rpt) + inputNodes := c.r.Render(rpt, dct) + endpoints := SelectEndpoint.Render(rpt, dct) - // Nodes without a hostid may be pseudo nodes - if _, ok := m.Latest.Lookup(report.HostNodeID); !ok { - if externalNode, ok := NewDerivedExternalNode(m, addr, local); ok { - return report.Nodes{externalNode.ID: externalNode} + // Collect all the IPs we are trying to map to, and which ID they map from + var ipNodes = map[string]string{} + for _, n := range inputNodes { + for _, ip := range c.toIPs(n) { + if _, exists := ipNodes[ip]; exists { + // If an IP is shared between multiple nodes, we can't reliably + // attribute an connection based on its IP + ipNodes[ip] = "" // blank out the mapping so we don't use it + } else { + ipNodes[ip] = n.ID + } } } + var ret = make(report.Nodes) + var mapped = map[string]string{} // input node ID -> output node ID - // We also allow for joining on ip:port pairs. This is useful for - // connections to the host IPs which have been port mapped to a - // container can only be unambiguously identified with the port. - // So we need to emit two nodes, for two different cases. - id := report.MakeScopedEndpointNodeID(scope, addr, "") - idWithPort := report.MakeScopedEndpointNodeID(scope, addr, port) - return report.Nodes{ - id: NewDerivedNode(id, m).WithTopology(IP), - idWithPort: NewDerivedNode(idWithPort, m).WithTopology(IP), + // Now look at all the endpoints and see which map to IP nodes + for _, m := range endpoints { + scope, addr, port, ok := report.ParseEndpointNodeID(m.ID) + if !ok { + continue + } + // Nodes without a hostid may be pseudo nodes - if so, pass through to result + if _, ok := m.Latest.Lookup(report.HostNodeID); !ok { + if id, ok := externalNodeID(m, addr, local); ok { + addToResults(m, id, ret, mapped, func() report.Node { + return report.MakeNode(id).WithTopology(Pseudo) + }) + continue + } + } + id, found := ipNodes[report.MakeScopedEndpointNodeID(scope, addr, "")] + // We also allow for joining on ip:port pairs. This is useful for + // connections to the host IPs which have been port mapped to a + // container can only be unambiguously identified with the port. + if !found { + id, found = ipNodes[report.MakeScopedEndpointNodeID(scope, addr, port)] + } + if found && id != "" { // not one we blanked out earlier + addToResults(m, id, ret, mapped, func() report.Node { + return inputNodes[id] + }) + } } + // Copy through any unmatched input nodes + for _, n := range inputNodes { + if _, found := ret[n.ID]; !found { + ret[n.ID] = n + } + } + fixupAdjancencies(inputNodes, ret, mapped) + fixupAdjancencies(endpoints, ret, mapped) + return ret +} + +func (c connectionJoin) Stats(rpt report.Report, _ Decorator) Stats { + return Stats{} // nothing to report } // FilterEmpty is a Renderer which filters out nodes which have no children diff --git a/render/host.go b/render/host.go index 7d374bff6..6a28135aa 100644 --- a/render/host.go +++ b/render/host.go @@ -104,6 +104,20 @@ func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { return ret } +// Add Node M to the result set ret under id, creating a new result +// node if not already there, and updating the old-id to new-id mapping +// Note we do not update any counters for child topologies here +func addToResults(m report.Node, id string, ret report.Nodes, mapped map[string]string, create func() report.Node) { + result, exists := ret[id] + if !exists { + result = create() + } + result.Children = result.Children.Add(m) + result.Children = result.Children.Merge(m.Children) + ret[result.ID] = result + mapped[m.ID] = result.ID +} + // Rewrite Adjacency for new nodes in ret, original nodes in input, and mapping old->new IDs in mapped func fixupAdjancencies(input, ret report.Nodes, mapped map[string]string) { for _, n := range input { From 77a8cac65a9121687f52a869dceffcf51de2944b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 5 Nov 2017 00:14:34 +0000 Subject: [PATCH 06/12] Use helper to add result nodes in endpoints2Hosts.Render This means we are no longer generating a Counter for the number of endpoint sub-nodes, but it seems that data was not used. --- render/host.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/render/host.go b/render/host.go index 6a28135aa..c6aab85c5 100644 --- a/render/host.go +++ b/render/host.go @@ -73,8 +73,6 @@ func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { var ret = make(report.Nodes) var mapped = map[string]string{} // input node ID -> output node ID for _, n := range ns { - var result report.Node - var exists bool // Nodes without a hostid are treated as pseudo nodes hostNodeID, timestamp, ok := n.Latest.LookupEntry(report.HostNodeID) if !ok { @@ -82,23 +80,16 @@ func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { if !ok { continue } - result, exists = ret[id] - if !exists { - result = report.MakeNode(id).WithTopology(Pseudo) - } + addToResults(n, id, ret, mapped, func() report.Node { + return report.MakeNode(id).WithTopology(Pseudo) + }) } else { id := report.MakeHostNodeID(report.ExtractHostID(n)) - result, exists = ret[id] - if !exists { - result = report.MakeNode(id).WithTopology(report.Host) - result.Latest = result.Latest.Set(report.HostNodeID, timestamp, hostNodeID) - } - result.Children = result.Children.Merge(n.Children) + addToResults(n, id, ret, mapped, func() report.Node { + return report.MakeNode(id).WithTopology(report.Host). + WithLatest(report.HostNodeID, timestamp, hostNodeID) + }) } - result.Children = result.Children.Add(n) - result.Counters = result.Counters.Add(n.Topology, 1) - ret[result.ID] = result - mapped[n.ID] = result.ID } fixupAdjancencies(ns, ret, mapped) return ret From cbcb5f19fcd675817d075f111140ee90b4a7b50e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 5 Nov 2017 16:54:30 +0000 Subject: [PATCH 07/12] Rewrite ProcessRenderer/MapEndpoint2Process as a single Renderer This is much more efficient as we skip creating then merging all intermediate Nodes --- render/process.go | 93 ++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/render/process.go b/render/process.go index 724e20b47..ed3d55030 100644 --- a/render/process.go +++ b/render/process.go @@ -26,15 +26,7 @@ var EndpointRenderer = SelectEndpoint // ProcessRenderer is a Renderer which produces a renderable process // graph by merging the endpoint graph and the process topology. -var ProcessRenderer = Memoise(ConditionalRenderer(renderProcesses, - MakeReduce( - MakeMap( - MapEndpoint2Process, - EndpointRenderer, - ), - SelectProcess, - ), -)) +var ProcessRenderer = Memoise(endpoints2Processes{}) // ColorConnectedProcessRenderer colors connected nodes from // ProcessRenderer. Since the process topology views only show @@ -99,40 +91,67 @@ func MapEndpoint2Pseudo(n report.Node, local report.Networks) report.Nodes { return report.Nodes{externalNode.ID: externalNode} } -// MapEndpoint2Process maps endpoint Nodes to process -// Nodes. -// -// If this function is given a pseudo node, then it will just return it; -// Pseudo nodes will never have pids in them, and therefore will never -// be able to be turned into a Process node. -// -// Otherwise, this function will produce a node with the correct ID -// format for a process, but without any Major or Minor labels. -// It does not have enough info to do that, and the resulting graph -// must be merged with a process graph to get that info. -func MapEndpoint2Process(n report.Node, local report.Networks) report.Nodes { - // Nodes without a hostid are treated as pseudo nodes - if _, ok := n.Latest.Lookup(report.HostNodeID); !ok { - return MapEndpoint2Pseudo(n, local) - } +// endpoints2Processes joins the endpoint topology to the process +// topology, matching on hostID and pid. +type endpoints2Processes struct { +} - pid, timestamp, ok := n.Latest.LookupEntry(process.PID) - if !ok { +func (e endpoints2Processes) Render(rpt report.Report, dct Decorator) report.Nodes { + if len(rpt.Process.Nodes) == 0 { return report.Nodes{} } + processes := SelectProcess.Render(rpt, dct) + endpoints := SelectEndpoint.Render(rpt, dct) + local := LocalNetworks(rpt) - if len(n.Adjacency) > 1 { - // We cannot be sure that the pid is associated with all the - // connections. It is better to drop such an endpoint than - // risk rendering bogus connections. - return report.Nodes{} + var ret = make(report.Nodes) + var mapped = map[string]string{} // input node ID -> output node ID + for _, n := range endpoints { + // Nodes without a hostid are treated as pseudo nodes + if hostNodeID, ok := n.Latest.Lookup(report.HostNodeID); !ok { + if id, ok := pseudoNodeID(n, local); ok { + addToResults(n, id, ret, mapped, func() report.Node { + return report.MakeNode(id).WithTopology(Pseudo) + }) + } + } else { + pid, timestamp, ok := n.Latest.LookupEntry(process.PID) + if !ok { + continue + } + + if len(n.Adjacency) > 1 { + // We cannot be sure that the pid is associated with all the + // connections. It is better to drop such an endpoint than + // risk rendering bogus connections. + continue + } + + hostID, _, _ := report.ParseNodeID(hostNodeID) + id := report.MakeProcessNodeID(hostID, pid) + addToResults(n, id, ret, mapped, func() report.Node { + if processNode, found := processes[id]; found { + return processNode + } + // we have a pid, but no matching process node; create a new one rather than dropping the data + return report.MakeNode(id).WithTopology(report.Process). + WithLatest(process.PID, timestamp, pid) + }) + } } + // Copy through any unmatched process nodes + for _, n := range processes { + if _, found := ret[n.ID]; !found { + ret[n.ID] = n + } + } + fixupAdjancencies(processes, ret, mapped) + fixupAdjancencies(endpoints, ret, mapped) + return ret +} - id := report.MakeProcessNodeID(report.ExtractHostID(n), pid) - node := NewDerivedNode(id, n).WithTopology(report.Process) - node.Latest = node.Latest.Set(process.PID, timestamp, pid) - node.Counters = node.Counters.Add(n.Topology, 1) - return report.Nodes{id: node} +func (e endpoints2Processes) Stats(rpt report.Report, _ Decorator) Stats { + return Stats{} // nothing to report } // MapProcess2Name maps process Nodes to Nodes From 322aa76e02000ca72ad2f47ef64a0683eee2a986 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 5 Nov 2017 21:48:05 +0000 Subject: [PATCH 08/12] Remove functions which are no longer called --- render/id.go | 9 --------- render/process.go | 10 ---------- 2 files changed, 19 deletions(-) diff --git a/render/id.go b/render/id.go index 441638d9d..ea955f30e 100644 --- a/render/id.go +++ b/render/id.go @@ -36,15 +36,6 @@ func NewDerivedPseudoNode(id string, node report.Node) report.Node { return output } -// NewDerivedExternalNode figures out if a node should be considered external and creates the corresponding pseudo node -func NewDerivedExternalNode(n report.Node, addr string, local report.Networks) (report.Node, bool) { - id, ok := externalNodeID(n, addr, local) - if !ok { - return report.Node{}, false - } - return NewDerivedPseudoNode(id, n), true -} - func pseudoNodeID(n report.Node, local report.Networks) (string, bool) { _, addr, _, ok := report.ParseEndpointNodeID(n.ID) if !ok { diff --git a/render/process.go b/render/process.go index ed3d55030..60d09f25e 100644 --- a/render/process.go +++ b/render/process.go @@ -81,16 +81,6 @@ var ProcessNameRenderer = ConditionalRenderer(renderProcesses, ), ) -// MapEndpoint2Pseudo makes internet of host pesudo nodes from a endpoint node. -func MapEndpoint2Pseudo(n report.Node, local report.Networks) report.Nodes { - id, ok := pseudoNodeID(n, local) - if !ok { - return report.Nodes{} - } - externalNode := NewDerivedPseudoNode(id, n) - return report.Nodes{externalNode.ID: externalNode} -} - // endpoints2Processes joins the endpoint topology to the process // topology, matching on hostID and pid. type endpoints2Processes struct { From cbba3c0fd3a3b5da6f0326b941076d04c75ab4cf Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 6 Nov 2017 21:22:33 +0000 Subject: [PATCH 09/12] Clarify use of 'id' in addToResults Pass 'id' through to the create function and expect that the result Node has that ID. Extract a function newPseudoNode for common calls. --- render/container.go | 6 ++---- render/host.go | 14 ++++++-------- render/id.go | 4 ++++ render/process.go | 6 ++---- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/render/container.go b/render/container.go index 6a5066132..f8f0dbd84 100644 --- a/render/container.go +++ b/render/container.go @@ -83,9 +83,7 @@ func (c connectionJoin) Render(rpt report.Report, dct Decorator) report.Nodes { // Nodes without a hostid may be pseudo nodes - if so, pass through to result if _, ok := m.Latest.Lookup(report.HostNodeID); !ok { if id, ok := externalNodeID(m, addr, local); ok { - addToResults(m, id, ret, mapped, func() report.Node { - return report.MakeNode(id).WithTopology(Pseudo) - }) + addToResults(m, id, ret, mapped, newPseudoNode) continue } } @@ -97,7 +95,7 @@ func (c connectionJoin) Render(rpt report.Report, dct Decorator) report.Nodes { id, found = ipNodes[report.MakeScopedEndpointNodeID(scope, addr, port)] } if found && id != "" { // not one we blanked out earlier - addToResults(m, id, ret, mapped, func() report.Node { + addToResults(m, id, ret, mapped, func(id string) report.Node { return inputNodes[id] }) } diff --git a/render/host.go b/render/host.go index c6aab85c5..384c47f5a 100644 --- a/render/host.go +++ b/render/host.go @@ -80,12 +80,10 @@ func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { if !ok { continue } - addToResults(n, id, ret, mapped, func() report.Node { - return report.MakeNode(id).WithTopology(Pseudo) - }) + addToResults(n, id, ret, mapped, newPseudoNode) } else { id := report.MakeHostNodeID(report.ExtractHostID(n)) - addToResults(n, id, ret, mapped, func() report.Node { + addToResults(n, id, ret, mapped, func(id string) report.Node { return report.MakeNode(id).WithTopology(report.Host). WithLatest(report.HostNodeID, timestamp, hostNodeID) }) @@ -98,15 +96,15 @@ func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { // Add Node M to the result set ret under id, creating a new result // node if not already there, and updating the old-id to new-id mapping // Note we do not update any counters for child topologies here -func addToResults(m report.Node, id string, ret report.Nodes, mapped map[string]string, create func() report.Node) { +func addToResults(m report.Node, id string, ret report.Nodes, mapped map[string]string, create func(string) report.Node) { result, exists := ret[id] if !exists { - result = create() + result = create(id) } result.Children = result.Children.Add(m) result.Children = result.Children.Merge(m.Children) - ret[result.ID] = result - mapped[m.ID] = result.ID + ret[id] = result + mapped[m.ID] = id } // Rewrite Adjacency for new nodes in ret, original nodes in input, and mapping old->new IDs in mapped diff --git a/render/id.go b/render/id.go index ea955f30e..471a38d0d 100644 --- a/render/id.go +++ b/render/id.go @@ -36,6 +36,10 @@ func NewDerivedPseudoNode(id string, node report.Node) report.Node { return output } +func newPseudoNode(id string) report.Node { + return report.MakeNode(id).WithTopology(Pseudo) +} + func pseudoNodeID(n report.Node, local report.Networks) (string, bool) { _, addr, _, ok := report.ParseEndpointNodeID(n.ID) if !ok { diff --git a/render/process.go b/render/process.go index 60d09f25e..e4288d3b5 100644 --- a/render/process.go +++ b/render/process.go @@ -100,9 +100,7 @@ func (e endpoints2Processes) Render(rpt report.Report, dct Decorator) report.Nod // Nodes without a hostid are treated as pseudo nodes if hostNodeID, ok := n.Latest.Lookup(report.HostNodeID); !ok { if id, ok := pseudoNodeID(n, local); ok { - addToResults(n, id, ret, mapped, func() report.Node { - return report.MakeNode(id).WithTopology(Pseudo) - }) + addToResults(n, id, ret, mapped, newPseudoNode) } } else { pid, timestamp, ok := n.Latest.LookupEntry(process.PID) @@ -119,7 +117,7 @@ func (e endpoints2Processes) Render(rpt report.Report, dct Decorator) report.Nod hostID, _, _ := report.ParseNodeID(hostNodeID) id := report.MakeProcessNodeID(hostID, pid) - addToResults(n, id, ret, mapped, func() report.Node { + addToResults(n, id, ret, mapped, func(id string) report.Node { if processNode, found := processes[id]; found { return processNode } From 4feb451760bb58ffeee16121b7364bc465b4113b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 6 Nov 2017 21:48:00 +0000 Subject: [PATCH 10/12] Refactor join-Renderer helper functions as methods New type joinResult is created to hold the nodes and ID mapping. The implementation move from host.go to render.go. --- render/container.go | 20 ++++++----------- render/host.go | 44 +++++--------------------------------- render/process.go | 20 ++++++----------- render/render.go | 52 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 65 deletions(-) diff --git a/render/container.go b/render/container.go index f8f0dbd84..5ee885d2d 100644 --- a/render/container.go +++ b/render/container.go @@ -71,8 +71,7 @@ func (c connectionJoin) Render(rpt report.Report, dct Decorator) report.Nodes { } } } - var ret = make(report.Nodes) - var mapped = map[string]string{} // input node ID -> output node ID + ret := newJoinResults() // Now look at all the endpoints and see which map to IP nodes for _, m := range endpoints { @@ -83,7 +82,7 @@ func (c connectionJoin) Render(rpt report.Report, dct Decorator) report.Nodes { // Nodes without a hostid may be pseudo nodes - if so, pass through to result if _, ok := m.Latest.Lookup(report.HostNodeID); !ok { if id, ok := externalNodeID(m, addr, local); ok { - addToResults(m, id, ret, mapped, newPseudoNode) + ret.addToResults(m, id, newPseudoNode) continue } } @@ -95,20 +94,15 @@ func (c connectionJoin) Render(rpt report.Report, dct Decorator) report.Nodes { id, found = ipNodes[report.MakeScopedEndpointNodeID(scope, addr, port)] } if found && id != "" { // not one we blanked out earlier - addToResults(m, id, ret, mapped, func(id string) report.Node { + ret.addToResults(m, id, func(id string) report.Node { return inputNodes[id] }) } } - // Copy through any unmatched input nodes - for _, n := range inputNodes { - if _, found := ret[n.ID]; !found { - ret[n.ID] = n - } - } - fixupAdjancencies(inputNodes, ret, mapped) - fixupAdjancencies(endpoints, ret, mapped) - return ret + ret.copyUnmatched(inputNodes) + ret.fixupAdjacencies(inputNodes) + ret.fixupAdjacencies(endpoints) + return ret.nodes } func (c connectionJoin) Stats(rpt report.Report, _ Decorator) Stats { diff --git a/render/host.go b/render/host.go index 384c47f5a..8ce144d0a 100644 --- a/render/host.go +++ b/render/host.go @@ -69,9 +69,8 @@ type endpoints2Hosts struct { func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { ns := SelectEndpoint.Render(rpt, dct) local := LocalNetworks(rpt) + ret := newJoinResults() - var ret = make(report.Nodes) - var mapped = map[string]string{} // input node ID -> output node ID for _, n := range ns { // Nodes without a hostid are treated as pseudo nodes hostNodeID, timestamp, ok := n.Latest.LookupEntry(report.HostNodeID) @@ -80,50 +79,17 @@ func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { if !ok { continue } - addToResults(n, id, ret, mapped, newPseudoNode) + ret.addToResults(n, id, newPseudoNode) } else { id := report.MakeHostNodeID(report.ExtractHostID(n)) - addToResults(n, id, ret, mapped, func(id string) report.Node { + ret.addToResults(n, id, func(id string) report.Node { return report.MakeNode(id).WithTopology(report.Host). WithLatest(report.HostNodeID, timestamp, hostNodeID) }) } } - fixupAdjancencies(ns, ret, mapped) - return ret -} - -// Add Node M to the result set ret under id, creating a new result -// node if not already there, and updating the old-id to new-id mapping -// Note we do not update any counters for child topologies here -func addToResults(m report.Node, id string, ret report.Nodes, mapped map[string]string, create func(string) report.Node) { - result, exists := ret[id] - if !exists { - result = create(id) - } - result.Children = result.Children.Add(m) - result.Children = result.Children.Merge(m.Children) - ret[id] = result - mapped[m.ID] = id -} - -// Rewrite Adjacency for new nodes in ret, original nodes in input, and mapping old->new IDs in mapped -func fixupAdjancencies(input, ret report.Nodes, mapped map[string]string) { - for _, n := range input { - outID, ok := mapped[n.ID] - if !ok { - continue - } - out := ret[outID] - // for each adjacency in the original node, find out what it maps to (if any), - // and add that to the new node - for _, a := range n.Adjacency { - if mappedDest, found := mapped[a]; found { - out.Adjacency = out.Adjacency.Add(mappedDest) - } - } - ret[outID] = out - } + ret.fixupAdjacencies(ns) + return ret.nodes } func (e endpoints2Hosts) Stats(rpt report.Report, _ Decorator) Stats { diff --git a/render/process.go b/render/process.go index e4288d3b5..f42db7c0a 100644 --- a/render/process.go +++ b/render/process.go @@ -93,14 +93,13 @@ func (e endpoints2Processes) Render(rpt report.Report, dct Decorator) report.Nod processes := SelectProcess.Render(rpt, dct) endpoints := SelectEndpoint.Render(rpt, dct) local := LocalNetworks(rpt) + ret := newJoinResults() - var ret = make(report.Nodes) - var mapped = map[string]string{} // input node ID -> output node ID for _, n := range endpoints { // Nodes without a hostid are treated as pseudo nodes if hostNodeID, ok := n.Latest.Lookup(report.HostNodeID); !ok { if id, ok := pseudoNodeID(n, local); ok { - addToResults(n, id, ret, mapped, newPseudoNode) + ret.addToResults(n, id, newPseudoNode) } } else { pid, timestamp, ok := n.Latest.LookupEntry(process.PID) @@ -117,7 +116,7 @@ func (e endpoints2Processes) Render(rpt report.Report, dct Decorator) report.Nod hostID, _, _ := report.ParseNodeID(hostNodeID) id := report.MakeProcessNodeID(hostID, pid) - addToResults(n, id, ret, mapped, func(id string) report.Node { + ret.addToResults(n, id, func(id string) report.Node { if processNode, found := processes[id]; found { return processNode } @@ -127,15 +126,10 @@ func (e endpoints2Processes) Render(rpt report.Report, dct Decorator) report.Nod }) } } - // Copy through any unmatched process nodes - for _, n := range processes { - if _, found := ret[n.ID]; !found { - ret[n.ID] = n - } - } - fixupAdjancencies(processes, ret, mapped) - fixupAdjancencies(endpoints, ret, mapped) - return ret + ret.copyUnmatched(processes) + ret.fixupAdjacencies(processes) + ret.fixupAdjacencies(endpoints) + return ret.nodes } func (e endpoints2Processes) Stats(rpt report.Report, _ Decorator) Stats { diff --git a/render/render.go b/render/render.go index 2e4357588..32205eb8a 100644 --- a/render/render.go +++ b/render/render.go @@ -207,3 +207,55 @@ func (c ConstantRenderer) Render(_ report.Report, _ Decorator) report.Nodes { func (c ConstantRenderer) Stats(_ report.Report, _ Decorator) Stats { return Stats{} } + +// joinResults is used by Renderers that join sets of nodes +type joinResults struct { + nodes report.Nodes + mapped map[string]string // input node ID -> output node ID +} + +func newJoinResults() joinResults { + return joinResults{nodes: make(report.Nodes), mapped: map[string]string{}} +} + +// Add Node M under id, creating a new result node if not already there +// and updating the mapping from old ID to new ID +// Note we do not update any counters for child topologies here, because addToResults +// is only ever called when m is an endpoint and we never look at endpoint counts +func (ret *joinResults) addToResults(m report.Node, id string, create func(string) report.Node) { + result, exists := ret.nodes[id] + if !exists { + result = create(id) + } + result.Children = result.Children.Add(m) + result.Children = result.Children.Merge(m.Children) + ret.nodes[id] = result + ret.mapped[m.ID] = id +} + +// Rewrite Adjacency for new nodes in ret for original nodes in input +func (ret *joinResults) fixupAdjacencies(input report.Nodes) { + for _, n := range input { + outID, ok := ret.mapped[n.ID] + if !ok { + continue + } + out := ret.nodes[outID] + // for each adjacency in the original node, find out what it maps to (if any), + // and add that to the new node + for _, a := range n.Adjacency { + if mappedDest, found := ret.mapped[a]; found { + out.Adjacency = out.Adjacency.Add(mappedDest) + } + } + ret.nodes[outID] = out + } +} + +func (ret *joinResults) copyUnmatched(input report.Nodes) { + for _, n := range input { + if _, found := ret.nodes[n.ID]; !found { + ret.nodes[n.ID] = n + } + } +} From ec0689b5aa3ae2d2326f3a751c33bf3632380d8b Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 6 Nov 2017 21:58:47 +0000 Subject: [PATCH 11/12] Code review: improve consistency of naming and ordering --- render/host.go | 6 +++--- render/process.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/render/host.go b/render/host.go index 8ce144d0a..f3c708458 100644 --- a/render/host.go +++ b/render/host.go @@ -67,11 +67,11 @@ type endpoints2Hosts struct { } func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { - ns := SelectEndpoint.Render(rpt, dct) local := LocalNetworks(rpt) + endpoints := SelectEndpoint.Render(rpt, dct) ret := newJoinResults() - for _, n := range ns { + for _, n := range endpoints { // Nodes without a hostid are treated as pseudo nodes hostNodeID, timestamp, ok := n.Latest.LookupEntry(report.HostNodeID) if !ok { @@ -88,7 +88,7 @@ func (e endpoints2Hosts) Render(rpt report.Report, dct Decorator) report.Nodes { }) } } - ret.fixupAdjacencies(ns) + ret.fixupAdjacencies(endpoints) return ret.nodes } diff --git a/render/process.go b/render/process.go index f42db7c0a..54d4a14b6 100644 --- a/render/process.go +++ b/render/process.go @@ -90,9 +90,9 @@ func (e endpoints2Processes) Render(rpt report.Report, dct Decorator) report.Nod if len(rpt.Process.Nodes) == 0 { return report.Nodes{} } + local := LocalNetworks(rpt) processes := SelectProcess.Render(rpt, dct) endpoints := SelectEndpoint.Render(rpt, dct) - local := LocalNetworks(rpt) ret := newJoinResults() for _, n := range endpoints { From 917ef980a41568f95cd00043cb8160d52cdf68c2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 7 Nov 2017 10:18:27 +0000 Subject: [PATCH 12/12] Memoise HostRenderer This shows a big improvement in BenchmarkTopologyList --- render/host.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/render/host.go b/render/host.go index f3c708458..bcb484b85 100644 --- a/render/host.go +++ b/render/host.go @@ -7,8 +7,7 @@ import ( // HostRenderer is a Renderer which produces a renderable host // graph from the host topology. // -// not memoised -var HostRenderer = MakeReduce( +var HostRenderer = Memoise(MakeReduce( endpoints2Hosts{}, MakeMap( MapX2Host, @@ -27,7 +26,7 @@ var HostRenderer = MakeReduce( PodRenderer, ), SelectHost, -) +)) // MapX2Host maps any Nodes to host Nodes. //