WIP -- deprecating host node id

It represents which probe the _thing_ was seen from, but in many cases
(container images, deployments, replicasets, services), it may have come
from several probes. We have previously conflated it to determine which
host a thing _lives on_, which it may not even have (deployments,
replica sets, services), or it may have multiple (container images).

The idea is to separate those two usages. We should convert HostNodeID
to a set of HostNodeIDs, and use that to determine which probes have
reported the thing. For determining which host a thing lives on we
should use the Parents field which we already have, but might need
extending to handle Endpoints/etc...

probe/endpoint/reporter.go

@@ -131,7 +131,7 @@ func (r *Reporter) Report() (report.Report, error) {
 			}
 
 			seenTuples[tuple.key()] = tuple
-			r.addConnection(&rpt, tuple, extraNodeInfo, extraNodeInfo)
+			r.addConnection(&rpt, tuple, extraNodeInfo, extraNodeInfo, report.Sets{}, report.Sets{})
 		})
 	}
 
@@ -148,12 +148,14 @@ func (r *Reporter) Report() (report.Report, error) {
 					conn.LocalPort,
 					conn.RemotePort,
 				}
-				toNodeInfo   = map[string]string{Procspied: "true"}
-				fromNodeInfo = map[string]string{Procspied: "true"}
+				toNodeInfo      = map[string]string{Procspied: "true"}
+				fromNodeInfo    = map[string]string{Procspied: "true"}
+				toNodeParents   report.Sets
+				fromNodeParents report.Sets
 			)
 			if conn.Proc.PID > 0 {
 				fromNodeInfo[process.PID] = strconv.FormatUint(uint64(conn.Proc.PID), 10)
-				fromNodeInfo[report.HostNodeID] = hostNodeID
+				fromNodeParents.Add(report.Host, report.MakeStringSet(hostNodeID))
 			}
 
 			// If we've already seen this connection, we should know the direction
@@ -164,8 +166,9 @@ func (r *Reporter) Report() (report.Report, error) {
 			if (ok && canonical != tuple) || (!ok && tuple.fromPort < tuple.toPort) {
 				tuple.reverse()
 				toNodeInfo, fromNodeInfo = fromNodeInfo, toNodeInfo
+				toNodeParents, fromNodeParents = fromNodeParents, toNodeParents
 			}
-			r.addConnection(&rpt, tuple, fromNodeInfo, toNodeInfo)
+			r.addConnection(&rpt, tuple, fromNodeInfo, toNodeInfo, fromNodeParents, toNodeParents)
 		}
 	}
 
@@ -173,7 +176,7 @@ func (r *Reporter) Report() (report.Report, error) {
 	return rpt, nil
 }
 
-func (r *Reporter) addConnection(rpt *report.Report, t fourTuple, extraFromNode, extraToNode map[string]string) {
+func (r *Reporter) addConnection(rpt *report.Report, t fourTuple, extraFromNode, extraToNode map[string]string, extraFromNodeParents, extraToNodeParents report.Sets) {
 	// Update endpoint topology
 	if !r.includeProcesses {
 		return
@@ -201,9 +204,11 @@ func (r *Reporter) addConnection(rpt *report.Report, t fourTuple, extraFromNode,
 	if extraFromNode != nil {
 		fromNode = fromNode.WithLatests(extraFromNode)
 	}
+	fromNode = fromNode.WithParents(extraFromNodeParents)
 	if extraToNode != nil {
 		toNode = toNode.WithLatests(extraToNode)
 	}
+	toNode = toNode.WithParents(extraToNodeParents)
 	rpt.Endpoint = rpt.Endpoint.AddNode(fromNode)
 	rpt.Endpoint = rpt.Endpoint.AddNode(toNode)
 }

probe/host/tagger.go

@@ -25,15 +25,21 @@ func (Tagger) Name() string { return "Host" }
 // Tag implements Tagger.
 func (t Tagger) Tag(r report.Report) (report.Report, error) {
 	var (
-		metadata = map[string]string{report.HostNodeID: t.hostNodeID}
-		parents  = report.EmptySets.Add(report.Host, report.MakeStringSet(t.hostNodeID))
+		sets    = report.EmptySets.Add(report.HostNodeIDs, report.MakeStringSet(t.hostNodeID))
+		parents = report.EmptySets.Add(report.Host, report.MakeStringSet(t.hostNodeID))
 	)
 
 	// Explicitly don't tag Endpoints and Addresses - These topologies include pseudo nodes,
 	// and as such do their own host tagging
 	for _, topology := range []report.Topology{r.Process, r.Container, r.ContainerImage, r.Host, r.Overlay, r.Pod} {
 		for _, node := range topology.Nodes {
-			topology.AddNode(node.WithLatests(metadata).WithParents(parents))
+			topology.AddNode(node.WithParents(parents))
+		}
+	}
+
+	for _, topology := range r.Topologies() {
+		for _, node := range topology.Nodes {
+			topology.AddNode(node.WithSets(sets))
 		}
 	}
 	return r, nil

render/container.go

@@ -256,20 +256,18 @@ func MapProcess2Container(n report.Node, _ report.Networks) report.Nodes {
 	// into an per-host "Uncontained" node.  If for whatever reason
 	// this node doesn't have a host id in their nodemetadata, it'll
 	// all get grouped into a single uncontained node.
-	var (
-		id   string
-		node report.Node
-	)
+	results := report.Nodes{}
 	if containerID, ok := n.Latest.Lookup(docker.ContainerID); ok {
-		id = report.MakeContainerNodeID(containerID)
-		node = NewDerivedNode(id, n).WithTopology(report.Container)
+		id := report.MakeContainerNodeID(containerID)
+		results[id] = NewDerivedNode(id, n).WithTopology(report.Container)
 	} else {
-		id = MakePseudoNodeID(UncontainedID, report.ExtractHostID(n))
-		node = NewDerivedPseudoNode(id, n)
-		node = propagateLatest(report.HostNodeID, n, node)
+		id := MakePseudoNodeID(UncontainedID, report.ExtractHostID(n))
+		node := NewDerivedPseudoNode(id, n)
+		node = propagateParents(report.Host, n, node)
 		node = propagateLatest(IsConnected, n, node)
+		results[id] = node
 	}
-	return report.Nodes{id: node}
+	return results
 }
 
 // MapContainer2ContainerImage maps container Nodes to container

render/host.go

@@ -44,30 +44,27 @@ func MapX2Host(n report.Node, _ report.Networks) report.Nodes {
 	if n.Topology == Pseudo {
 		return report.Nodes{}
 	}
-	hostNodeID, timestamp, ok := n.Latest.LookupEntry(report.HostNodeID)
-	if !ok {
-		return report.Nodes{}
+	ids, _ := n.Parents.Lookup(report.Host)
+	results := report.Nodes{}
+	for _, id := range ids {
+		result := NewDerivedNode(id, n).
+			WithTopology(report.Host).
+			WithSet(report.HostNodeIDs, report.MakeStringSet(id)).
+			WithCounters(map[string]int{n.Topology: 1})
+		result.Children = report.MakeNodeSet(n)
+		results[id] = result
 	}
-	id := report.MakeHostNodeID(report.ExtractHostID(n))
-	result := NewDerivedNode(id, n).WithTopology(report.Host)
-	result.Latest = result.Latest.Set(report.HostNodeID, timestamp, hostNodeID)
-	result.Counters = result.Counters.Add(n.Topology, 1)
-	result.Children = report.MakeNodeSet(n)
-	return report.Nodes{id: result}
+	return results
 }
 
 // MapEndpoint2Host takes nodes from the endpoint topology and produces
 // host nodes or pseudo nodes.
 func MapEndpoint2Host(n report.Node, local report.Networks) report.Nodes {
-	// Nodes without a hostid are treated as pseudo nodes
-	hostNodeID, timestamp, ok := n.Latest.LookupEntry(report.HostNodeID)
+	// Nodes without a host are treated as pseudo nodes
+	_, ok := n.Parents.Lookup(report.Host)
 	if !ok {
 		return MapEndpoint2Pseudo(n, local)
 	}
 
-	id := report.MakeHostNodeID(report.ExtractHostID(n))
-	result := NewDerivedNode(id, n).WithTopology(report.Host)
-	result.Latest = result.Latest.Set(report.HostNodeID, timestamp, hostNodeID)
-	result.Counters = result.Counters.Add(n.Topology, 1)
-	return report.Nodes{id: result}
+	return MapX2Host(n, local)
 }

render/process.go

@@ -112,8 +112,8 @@ func MapEndpoint2Pseudo(n report.Node, local report.Networks) report.Nodes {
 // It does not have enough info to do that, and the resulting graph
 // must be merged with a process graph to get that info.
 func MapEndpoint2Process(n report.Node, local report.Networks) report.Nodes {
-	// Nodes without a hostid are treated as pseudo nodes
-	if _, ok := n.Latest.Lookup(report.HostNodeID); !ok {
+	// Nodes without a host are treated as pseudo nodes
+	if _, ok := n.Parents.Lookup(report.Host); !ok {
 		return MapEndpoint2Pseudo(n, local)
 	}
 

render/render.go

@@ -159,6 +159,11 @@ func propagateLatest(key string, from, to report.Node) report.Node {
 	return to
 }
 
+func propagateParents(topology string, from, to report.Node) report.Node {
+	p, _ := from.Parents.Lookup(topology)
+	return to.WithParents(report.EmptySets.Add(topology, p))
+}
+
 // Condition is a predecate over the entire report that can evaluate to true or false.
 type Condition func(report.Report) bool
 

report/id.go

@@ -194,10 +194,14 @@ func ParseAddressNodeID(addressNodeID string) (hostID, address string, ok bool)
 	return fields[0], fields[1], true
 }
 
-// ExtractHostID extracts the host id from Node
+// ExtractHostID extracts the host id from Node. If a node has multiple host
+// parent nodes, we just return an empty string, as it's unclear.
 func ExtractHostID(m Node) string {
-	hostNodeID, _ := m.Latest.Lookup(HostNodeID)
-	hostID, _, _ := ParseNodeID(hostNodeID)
+	hostNodeIDs, _ := m.Sets.Lookup(HostNodeIDs)
+	if len(hostNodeIDs) != 1 {
+		return ""
+	}
+	hostID, _, _ := ParseNodeID(hostNodeIDs[0])
 	return hostID
 }
 

report/report.go

@@ -273,10 +273,10 @@ func (s Sampling) Merge(other Sampling) Sampling {
 }
 
 const (
-	// HostNodeID is a metadata foreign key, linking a node in any topology to
+	// HostNodeIDs is a metadata foreign key, linking a node in any topology to
 	// a node in the host topology. That host node is the origin host, where
 	// the node was originally detected.
-	HostNodeID = "host_node_id"
+	HostNodeIDs = "host_node_ids"
 	// ControlProbeID is the random ID of the probe which controls the specific node.
 	ControlProbeID = "control_probe_id"
 )