Make render module produce The Internet pseudo node.

app/api_topology_test.go

@@ -80,34 +80,35 @@ func TestAPITopologyHosts(t *testing.T) {
 			t.Fatal(err)
 		}
 		equals(t, 3, len(topo.Nodes))
-		node, ok := topo.Nodes["host:host-b"]
+		node, ok := topo.Nodes["host:hostB"]
 		if !ok {
-			t.Errorf("missing host:host-b node")
+			t.Errorf("missing host:hostB node")
+			t.Errorf("%+v", topo)
 		}
-		equals(t, report.MakeIDList("host:host-a"), node.Adjacency)
+		equals(t, report.MakeIDList("host:hostA"), node.Adjacency)
 		equals(t, report.MakeIDList(
 			report.MakeAddressNodeID("hostB", "192.168.1.2"),
 			report.MakeHostNodeID("hostB"),
 		), node.Origins)
-		equals(t, "host-b", node.LabelMajor)
-		equals(t, "", node.LabelMinor)
-		equals(t, "host-b", node.Rank)
+		equals(t, "node-b", node.LabelMajor)
+		equals(t, "local", node.LabelMinor)
+		equals(t, "local", node.Rank)
 		equals(t, false, node.Pseudo)
 	}
 	{
-		body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b")
+		body := getRawJSON(t, ts, "/api/topology/hosts/host:hostB")
 		var node APINode
 		if err := json.Unmarshal(body, &node); err != nil {
 			t.Fatal(err)
 		}
-		equals(t, "host:host-b", node.Node.ID)
-		equals(t, "host-b", node.Node.LabelMajor)
-		equals(t, "", node.Node.LabelMinor)
+		equals(t, "host:hostB", node.Node.ID)
+		equals(t, "node-b", node.Node.LabelMajor)
+		equals(t, "local", node.Node.LabelMinor)
 		equals(t, false, node.Node.Pseudo)
 		// Let's not unit-test the specific content of the detail tables
 	}
 	{
-		body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b/host:host-a")
+		body := getRawJSON(t, ts, "/api/topology/hosts/host:hostB/host:hostA")
 		var edge APIEdge
 		if err := json.Unmarshal(body, &edge); err != nil {
 			t.Fatalf("JSON parse error: %s", err)

app/mock_reporter_test.go

@@ -142,11 +142,11 @@ func (s StaticReport) Report() report.Report {
 			},
 			NodeMetadatas: report.NodeMetadatas{
 				report.MakeAddressNodeID("hostA", "192.168.1.1"): report.NodeMetadata{
-					"name":            "host-a",
+					"addr":            "192.168.1.1",
 					report.HostNodeID: report.MakeHostNodeID("hostA"),
 				},
 				report.MakeAddressNodeID("hostB", "192.168.1.2"): report.NodeMetadata{
-					"name":            "host-b",
+					"addr":            "192.168.1.2",
 					report.HostNodeID: report.MakeHostNodeID("hostB"),
 				},
 			},
@@ -157,15 +157,17 @@ func (s StaticReport) Report() report.Report {
 			EdgeMetadatas: report.EdgeMetadatas{},
 			NodeMetadatas: report.NodeMetadatas{
 				report.MakeHostNodeID("hostA"): report.NodeMetadata{
-					"host_name":      "node-a.local",
-					"os":             "Linux",
-					"local_networks": localNet.String(),
-					"load":           "3.14 2.71 1.61",
+					"host_name":       "node-a.local",
+					"os":              "Linux",
+					"local_networks":  localNet.String(),
+					"load":            "3.14 2.71 1.61",
+					report.HostNodeID: report.MakeHostNodeID("hostA"),
 				},
 				report.MakeHostNodeID("hostB"): report.NodeMetadata{
-					"host_name":      "node-b.local",
-					"os":             "Linux",
-					"local_networks": localNet.String(),
+					"host_name":       "node-b.local",
+					"os":              "Linux",
+					"local_networks":  localNet.String(),
+					report.HostNodeID: report.MakeHostNodeID("hostB"),
 				},
 			},
 		},

app/router.go

@@ -6,7 +6,6 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/weaveworks/scope/render"
-	"github.com/weaveworks/scope/report"
 )
 
 // Router gives of the HTTP dispatcher. It will always use the embedded HTML
@@ -70,7 +69,7 @@ var topologyRegistry = map[string]topologyView{
 	"hosts": {
 		human:    "Hosts",
 		parent:   "",
-		renderer: render.LeafMap{Selector: report.SelectAddress, Mapper: render.NetworkHostname, Pseudo: render.GenericPseudoNode},
+		renderer: render.HostRenderer,
 	},
 }
 

experimental/bridge/main.go

@@ -12,6 +12,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/weaveworks/scope/render"
 	"github.com/weaveworks/scope/report"
 	"github.com/weaveworks/scope/xfer"
 )
@@ -112,34 +113,6 @@ func makeAvoid(fixed []string) map[string]struct{} {
 	return avoid
 }
 
-// LocalNetworks returns a superset of the networks (think: CIDRs) that are
-// "local" from the perspective of each host represented in the report. It's
-// used to determine which nodes in the report are "remote", i.e. outside of
-// our infrastructure.
-func LocalNetworks(r report.Report) []*net.IPNet {
-	var ipNets []*net.IPNet
-	for _, md := range r.Host.NodeMetadatas {
-		val, ok := md["local_networks"]
-		if !ok {
-			continue
-		}
-	outer:
-		for _, s := range strings.Fields(val) {
-			_, ipNet, err := net.ParseCIDR(s)
-			if err != nil {
-				continue
-			}
-			for _, existing := range ipNets {
-				if ipNet.String() == existing.String() {
-					continue outer
-				}
-			}
-			ipNets = append(ipNets, ipNet)
-		}
-	}
-	return ipNets
-}
-
 // discover reads reports from a collector and republishes them on the
 // publisher, while scanning the reports for IPs to connect to. Only addresses
 // in the network topology of the report are considered. IPs listed in fixed
@@ -155,7 +128,7 @@ func discover(c collector, p publisher, fixed []string) {
 
 		var (
 			now       = time.Now()
-			localNets = LocalNetworks(r)
+			localNets = render.LocalNetworks(r)
 		)
 
 		for _, adjacent := range r.Address.Adjacency {

experimental/graphviz/handle.go

@@ -105,7 +105,7 @@ func engine(r *http.Request) string {
 func mapFunc(r *http.Request) render.LeafMapFunc {
 	switch strings.ToLower(r.FormValue("map_func")) {
 	case "hosts", "networkhost", "networkhostname":
-		return render.NetworkHostname
+		return render.MapAddressIdentity
 	}
 	return render.MapProcessIdentity
 }

probe/main.go

@@ -115,13 +115,15 @@ func main() {
 			select {
 			case <-pubTick:
 				publishTicks.WithLabelValues().Add(1)
-				r.Host = hostTopology(hostID, hostName)
 				publisher.Publish(r)
 				r = report.MakeReport()
 
 			case <-spyTick:
 				r.Merge(spy(hostID, hostName, *spyProcs))
 
+				// Do this every tick so it gets tagged by the OriginHostTagger
+				r.Host = hostTopology(hostID, hostName)
+
 				// TODO abstract PIDTree to a process provider, and provide an
 				// alternate implementation for Darwin.
 				if runtime.GOOS == linux {

probe/spy.go

@@ -54,6 +54,7 @@ func addConnection(
 	if _, ok := r.Address.NodeMetadatas[scopedLocal]; !ok {
 		r.Address.NodeMetadatas[scopedLocal] = report.NodeMetadata{
 			"name": hostName,
+			"addr": c.LocalAddress.String(),
 		}
 	}
 

render/detailed_node.go

@@ -121,12 +121,9 @@ func endpointOriginTable(nmd report.NodeMetadata) (Table, bool) {
 
 func addressOriginTable(nmd report.NodeMetadata) (Table, bool) {
 	rows := []Row{}
-	if val, ok := nmd["address"]; ok {
+	if val, ok := nmd["addr"]; ok {
 		rows = append(rows, Row{"Address", val, ""})
 	}
-	if val, ok := nmd["host_name"]; ok {
-		rows = append(rows, Row{"Host name", val, ""})
-	}
 	return Table{
 		Title:   "Origin Address",
 		Numeric: false,

render/detailed_node_test.go

@@ -25,7 +25,7 @@ func TestOriginTable(t *testing.T) {
 			Title:   "Origin Address",
 			Numeric: false,
 			Rows: []render.Row{
-				{"Host name", clientHostName, ""},
+				{"Address", clientIP, ""},
 			},
 		},
 		serverProcessNodeID: {

render/mapping.go

@@ -2,6 +2,7 @@ package render
 
 import (
 	"fmt"
+	"net"
 	"strings"
 
 	"github.com/weaveworks/scope/report"
@@ -12,7 +13,8 @@ const (
 	UncontainedID    = "uncontained"
 	UncontainedMajor = "Uncontained"
 
-	humanTheInternet = "the Internet"
+	TheInternetID    = "theinternet"
+	TheInternetMajor = "The Internet"
 )
 
 // LeafMapFunc is anything which can take an arbitrary NodeMetadata, which is
@@ -31,7 +33,7 @@ type LeafMapFunc func(report.NodeMetadata) (RenderableNode, bool)
 // The srcNode renderable node is essentially from MapFunc, representing one of
 // the rendered nodes this pseudo node refers to. srcNodeID and dstNodeID are
 // node IDs prior to mapping.
-type PseudoFunc func(srcNodeID string, srcNode RenderableNode, dstNodeID string) (RenderableNode, bool)
+type PseudoFunc func(srcNodeID string, srcNode RenderableNode, dstNodeID string, local Networks) (RenderableNode, bool)
 
 // MapFunc is anything which can take an arbitrary RenderableNode and
 // return another RenderableNode.
@@ -100,6 +102,40 @@ func MapContainerImageIdentity(m report.NodeMetadata) (RenderableNode, bool) {
 	return NewRenderableNode(id, major, "", rank, m), true
 }
 
+// MapAddressIdentity maps a address topology node to address RenderableNode
+// node. As it is only ever run on address topology nodes, we can safely
+// assume the presence of certain keys.
+func MapAddressIdentity(m report.NodeMetadata) (RenderableNode, bool) {
+	var (
+		id    = fmt.Sprintf("address:%s:%s", report.ExtractHostID(m), m["addr"])
+		major = m["addr"]
+		minor = report.ExtractHostID(m)
+		rank  = major
+	)
+
+	return NewRenderableNode(id, major, minor, rank, m), true
+}
+
+// MapHostIdentity maps a host topology node to host RenderableNode
+// node. As it is only ever run on host topology nodes, we can safely
+// assume the presence of certain keys.
+func MapHostIdentity(m report.NodeMetadata) (RenderableNode, bool) {
+	var (
+		id                 = fmt.Sprintf("host:%s", report.ExtractHostID(m))
+		hostname           = m["host_name"]
+		parts              = strings.SplitN(hostname, ".", 2)
+		major, minor, rank = "", "", ""
+	)
+
+	if len(parts) == 2 {
+		major, minor, rank = parts[0], parts[1], parts[1]
+	} else {
+		major = hostname
+	}
+
+	return NewRenderableNode(id, major, minor, rank, m), true
+}
+
 // MapEndpoint2Process maps endpoint RenderableNodes to process
 // RenderableNodes.
 //
@@ -138,6 +174,13 @@ func MapEndpoint2Process(n RenderableNode) (RenderableNode, bool) {
 // It does not have enough info to do that, and the resulting graph
 // must be merged with a container graph to get that info.
 func MapProcess2Container(n RenderableNode) (RenderableNode, bool) {
+	// Propogate the internet pseudo node
+	if n.ID == TheInternetID {
+		return n, true
+	}
+
+	// Otherwise, if the process is not in a container, group it
+	// into an "Uncontained" node
 	id, ok := n.NodeMetadata["docker_container_id"]
 	if !ok || n.Pseudo {
 		return newDerivedPseudoNode(UncontainedID, UncontainedMajor, n), true
@@ -181,6 +224,13 @@ func MapProcess2Name(n RenderableNode) (RenderableNode, bool) {
 // It does not have enough info to do that, and the resulting graph
 // must be merged with a container graph to get that info.
 func MapContainer2ContainerImage(n RenderableNode) (RenderableNode, bool) {
+	// Propogate the internet pseudo node
+	if n.ID == TheInternetID {
+		return n, true
+	}
+
+	// Otherwise, if the process is not in a container, group it
+	// into an "Uncontained" node
 	id, ok := n.NodeMetadata["docker_image_id"]
 	if !ok || n.Pseudo {
 		return newDerivedPseudoNode(UncontainedID, UncontainedMajor, n), true
@@ -189,69 +239,46 @@ func MapContainer2ContainerImage(n RenderableNode) (RenderableNode, bool) {
 	return newDerivedNode(id, n), true
 }
 
-// NetworkHostname takes a node NodeMetadata and returns a representation
-// based on the hostname. Major label is the hostname, the minor label is the
-// domain, if any.
-func NetworkHostname(m report.NodeMetadata) (RenderableNode, bool) {
-	var (
-		name   = m["name"]
-		domain = ""
-		parts  = strings.SplitN(name, ".", 2)
-	)
-
-	if len(parts) == 2 {
-		domain = parts[1]
+// MapAddress2Host maps address RenderableNodes to host RenderableNodes.
+//
+// Otherthan pseudo nodes, we can assume all nodes have a HostID
+func MapAddress2Host(n RenderableNode) (RenderableNode, bool) {
+	if n.Pseudo {
+		return n, true
 	}
 
-	return NewRenderableNode(fmt.Sprintf("host:%s", name), parts[0], domain, parts[0], m), name != ""
+	id := fmt.Sprintf("host:%s", report.ExtractHostID(n.NodeMetadata))
+	return newDerivedNode(id, n), true
 }
 
-// GenericPseudoNode contains heuristics for building sensible pseudo nodes.
-// It should go away.
-func GenericPseudoNode(src string, srcMapped RenderableNode, dst string) (RenderableNode, bool) {
-	var maj, min, outputID string
+// GenericPseudoNode makes a PseudoFunc given an addresser.  The returned
+// PseudoFunc will produce Internet pseudo nodes for addresses not in
+// the report's local networks.  Otherwise, the returned function will
+// produce a single pseudo node per (dst address, src address, src port).
+func GenericPseudoNode(addresser func(id string) net.IP) PseudoFunc {
+	return func(src string, srcMapped RenderableNode, dst string, local Networks) (RenderableNode, bool) {
+		// Use the addresser to extract the destination IP
+		dstNodeAddr := addresser(dst)
 
-	if dst == report.TheInternet {
-		outputID = dst
-		maj, min = humanTheInternet, ""
-	} else {
-		// Rule for non-internet psuedo nodes; emit 1 new node for each
+		// If the dstNodeAddr is not in a network local to this report, we emit an
+		// internet node
+		if !local.Contains(dstNodeAddr) {
+			return newPseudoNode(TheInternetID, TheInternetMajor, ""), true
+		}
+
+		// Otherwise, the rule for non-internet psuedo nodes; emit 1 new node for each
 		// dstNodeAddr, srcNodeAddr, srcNodePort.
 		srcNodeAddr, srcNodePort := trySplitAddr(src)
-		dstNodeAddr, _ := trySplitAddr(dst)
 
-		outputID = report.MakePseudoNodeID(dstNodeAddr, srcNodeAddr, srcNodePort)
-		maj, min = dstNodeAddr, ""
+		outputID := report.MakePseudoNodeID(dstNodeAddr.String(), srcNodeAddr, srcNodePort)
+		major := dstNodeAddr.String()
+		return newPseudoNode(outputID, major, ""), true
 	}
-
-	return newPseudoNode(outputID, maj, min), true
 }
 
-// GenericGroupedPseudoNode contains heuristics for building sensible pseudo nodes.
-// It should go away.
-func GenericGroupedPseudoNode(src string, srcMapped RenderableNode, dst string) (RenderableNode, bool) {
-	var maj, min, outputID string
-
-	if dst == report.TheInternet {
-		outputID = dst
-		maj, min = humanTheInternet, ""
-	} else {
-		// When grouping, emit one pseudo node per (srcNodeAddress, dstNodeAddr)
-		dstNodeAddr, _ := trySplitAddr(dst)
-
-		outputID = report.MakePseudoNodeID(dstNodeAddr, srcMapped.ID)
-		maj, min = dstNodeAddr, ""
-	}
-
-	return newPseudoNode(outputID, maj, min), true
-}
-
-// InternetOnlyPseudoNode never creates a pseudo node, unless it's the Internet.
-func InternetOnlyPseudoNode(_ string, _ RenderableNode, dst string) (RenderableNode, bool) {
-	if dst == report.TheInternet {
-		return newPseudoNode(report.TheInternet, humanTheInternet, ""), true
-	}
-	return RenderableNode{}, false
+// PanicPseudoNode just panics; it is for Topologies without edges
+func PanicPseudoNode(src string, srcMapped RenderableNode, dst string, local Networks) (RenderableNode, bool) {
+	panic(dst)
 }
 
 // trySplitAddr is basically ParseArbitraryNodeID, since its callsites

render/mapping_test.go

@@ -1,67 +0,0 @@
-package render_test
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/weaveworks/scope/render"
-	"github.com/weaveworks/scope/report"
-)
-
-func TestUngroupedMapping(t *testing.T) {
-	for i, c := range []struct {
-		f                                      render.LeafMapFunc
-		id                                     string
-		meta                                   report.NodeMetadata
-		wantOK                                 bool
-		wantID, wantMajor, wantMinor, wantRank string
-	}{
-		{
-			f:  render.NetworkHostname,
-			id: report.MakeAddressNodeID("", "1.2.3.4"),
-			meta: report.NodeMetadata{
-				"name": "my.host",
-			},
-			wantOK:    true,
-			wantID:    "host:my.host",
-			wantMajor: "my",
-			wantMinor: "host",
-			wantRank:  "my",
-		},
-		{
-			f:  render.NetworkHostname,
-			id: report.MakeAddressNodeID("", "1.2.3.4"),
-			meta: report.NodeMetadata{
-				"name": "localhost",
-			},
-			wantOK:    true,
-			wantID:    "host:localhost",
-			wantMajor: "localhost",
-			wantMinor: "",
-			wantRank:  "localhost",
-		},
-	} {
-		identity := fmt.Sprintf("(%d %s %v)", i, c.id, c.meta)
-
-		m, haveOK := c.f(c.meta)
-		if want, have := c.wantOK, haveOK; want != have {
-			t.Errorf("%s: map OK error: want %v, have %v", identity, want, have)
-		}
-		if want, have := c.wantID, m.ID; want != have {
-			t.Errorf("%s: map ID error: want %#v, have %#v", identity, want, have)
-		}
-		if want, have := c.wantMajor, m.LabelMajor; want != have {
-			t.Errorf("%s: map major label: want %#v, have %#v", identity, want, have)
-		}
-		if want, have := c.wantMinor, m.LabelMinor; want != have {
-			t.Errorf("%s: map minor label: want %#v, have %#v", identity, want, have)
-		}
-		if want, have := c.wantRank, m.Rank; want != have {
-			t.Errorf("%s: map rank: want %#v, have %#v", identity, want, have)
-		}
-	}
-}
-
-func TestGroupedMapping(t *testing.T) {
-	t.Skipf("not yet implemented") // TODO
-}

render/render.go

@@ -144,8 +144,11 @@ func (m Map) AggregateMetadata(rpt report.Report, srcRenderableID, dstRenderable
 //
 // Nodes with the same mapped IDs will be merged.
 func (m LeafMap) Render(rpt report.Report) RenderableNodes {
-	t := m.Selector(rpt)
-	nodes := RenderableNodes{}
+	var (
+		t             = m.Selector(rpt)
+		nodes         = RenderableNodes{}
+		localNetworks = LocalNetworks(rpt)
+	)
 
 	// Build a set of RenderableNodes for all non-pseudo probes, and an
 	// addressID to nodeID lookup map. Multiple addressIDs can map to the same
@@ -189,7 +192,7 @@ func (m LeafMap) Render(rpt report.Report) RenderableNodes {
 		for _, dstNodeID := range dsts {
 			dstRenderableID, ok := source2mapped[dstNodeID]
 			if !ok {
-				pseudoNode, ok := m.Pseudo(srcNodeID, srcRenderableNode, dstNodeID)
+				pseudoNode, ok := m.Pseudo(srcNodeID, srcRenderableNode, dstNodeID, localNetworks)
 				if !ok {
 					continue
 				}

render/theinternet.go

@@ -0,0 +1,51 @@
+package render
+
+import (
+	"net"
+	"strings"
+
+	"github.com/weaveworks/scope/report"
+)
+
+// Networks represent a set of subnets local to a report.
+type Networks []*net.IPNet
+
+// LocalNetworks returns a superset of the networks (think: CIDRs) that are
+// "local" from the perspective of each host represented in the report. It's
+// used to determine which nodes in the report are "remote", i.e. outside of
+// our infrastructure.
+func LocalNetworks(r report.Report) Networks {
+	var (
+		result   = Networks{}
+		networks = map[string]struct{}{}
+	)
+
+	for _, md := range r.Host.NodeMetadatas {
+		val, ok := md["local_networks"]
+		if !ok {
+			continue
+		}
+		for _, s := range strings.Fields(val) {
+			_, ipNet, err := net.ParseCIDR(s)
+			if err != nil {
+				continue
+			}
+			_, ok := networks[ipNet.String()]
+			if !ok {
+				result = append(result, ipNet)
+				networks[ipNet.String()] = struct{}{}
+			}
+		}
+	}
+	return result
+}
+
+// Contains returns true if IP is in Networks.
+func (n Networks) Contains(ip net.IP) bool {
+	for _, net := range n {
+		if net.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}

render/theinternet_test.go

@@ -0,0 +1,50 @@
+package render_test
+
+import (
+	"net"
+	"reflect"
+	"testing"
+
+	"github.com/weaveworks/scope/render"
+	"github.com/weaveworks/scope/report"
+	"github.com/weaveworks/scope/test"
+)
+
+func TestReportLocalNetworks(t *testing.T) {
+	r := report.MakeReport()
+	r.Merge(report.Report{Host: report.Topology{NodeMetadatas: report.NodeMetadatas{
+		"nonets": {},
+		"foo":    {"local_networks": "10.0.0.1/8 192.168.1.1/24 10.0.0.1/8 badnet/33"},
+	}}})
+	want := render.Networks([]*net.IPNet{
+		mustParseCIDR("10.0.0.1/8"),
+		mustParseCIDR("192.168.1.1/24"),
+	})
+	have := render.LocalNetworks(r)
+	if !reflect.DeepEqual(want, have) {
+		t.Errorf("%s", test.Diff(want, have))
+	}
+}
+
+func TestContains(t *testing.T) {
+	networks := render.Networks([]*net.IPNet{
+		mustParseCIDR("10.0.0.1/8"),
+		mustParseCIDR("192.168.1.1/24"),
+	})
+
+	if networks.Contains(net.ParseIP("52.52.52.52")) {
+		t.Errorf("52.52.52.52 not in %v", networks)
+	}
+
+	if !networks.Contains(net.ParseIP("10.0.0.1")) {
+		t.Errorf("10.0.0.1 in %v", networks)
+	}
+}
+
+func mustParseCIDR(s string) *net.IPNet {
+	_, ipNet, err := net.ParseCIDR(s)
+	if err != nil {
+		panic(err)
+	}
+	return ipNet
+}

render/topologies.go

@@ -8,7 +8,7 @@ import (
 var EndpointRenderer = LeafMap{
 	Selector: report.SelectEndpoint,
 	Mapper:   MapEndpointIdentity,
-	Pseudo:   GenericPseudoNode,
+	Pseudo:   GenericPseudoNode(report.EndpointIDAddresser),
 }
 
 // ProcessRenderer is a Renderer which produces a renderable process
@@ -21,7 +21,7 @@ var ProcessRenderer = MakeReduce(
 	LeafMap{
 		Selector: report.SelectProcess,
 		Mapper:   MapProcessIdentity,
-		Pseudo:   GenericPseudoNode,
+		Pseudo:   PanicPseudoNode,
 	},
 )
 
@@ -42,7 +42,7 @@ var ContainerRenderer = MakeReduce(
 	LeafMap{
 		Selector: report.SelectContainer,
 		Mapper:   MapContainerIdentity,
-		Pseudo:   GenericPseudoNode,
+		Pseudo:   PanicPseudoNode,
 	},
 )
 
@@ -56,6 +56,28 @@ var ContainerImageRenderer = MakeReduce(
 	LeafMap{
 		Selector: report.SelectContainerImage,
 		Mapper:   MapContainerImageIdentity,
-		Pseudo:   GenericPseudoNode,
+		Pseudo:   PanicPseudoNode,
+	},
+)
+
+// AddressRenderer is a Renderer which produces a renderable address
+// graph from the address topology.
+var AddressRenderer = LeafMap{
+	Selector: report.SelectAddress,
+	Mapper:   MapAddressIdentity,
+	Pseudo:   GenericPseudoNode(report.AddressIDAddresser),
+}
+
+// HostRenderer is a Renderer which produces a renderable host
+// graph from the host topology and address graph.
+var HostRenderer = MakeReduce(
+	Map{
+		MapFunc:  MapAddress2Host,
+		Renderer: AddressRenderer,
+	},
+	LeafMap{
+		Selector: report.SelectHost,
+		Mapper:   MapHostIdentity,
+		Pseudo:   PanicPseudoNode,
 	},
 )

render/topologies_test.go

@@ -10,10 +10,12 @@ import (
 	"github.com/weaveworks/scope/test"
 )
 
+// This is an example Report:
+//   2 hosts with probes installed - client & server.
+
 var (
 	clientHostID  = "client.hostname.com"
 	serverHostID  = "server.hostname.com"
-	randomHostID  = "random.hostname.com"
 	unknownHostID = ""
 
 	clientIP        = "10.10.10.20"
@@ -31,19 +33,14 @@ var (
 
 	clientHostNodeID = report.MakeHostNodeID(clientHostID)
 	serverHostNodeID = report.MakeHostNodeID(serverHostID)
-	randomHostNodeID = report.MakeHostNodeID(randomHostID)
 
 	client54001NodeID    = report.MakeEndpointNodeID(clientHostID, clientIP, clientPort54001) // curl (1)
 	client54002NodeID    = report.MakeEndpointNodeID(clientHostID, clientIP, clientPort54002) // curl (2)
+	server80NodeID       = report.MakeEndpointNodeID(serverHostID, serverIP, serverPort)      // apache
 	unknownClient1NodeID = report.MakeEndpointNodeID(serverHostID, "10.10.10.10", "54010")    // we want to ensure two unknown clients, connnected
 	unknownClient2NodeID = report.MakeEndpointNodeID(serverHostID, "10.10.10.10", "54020")    // to the same server, are deduped.
 	unknownClient3NodeID = report.MakeEndpointNodeID(serverHostID, "10.10.10.11", "54020")    // Check this one isn't deduped
-	server80NodeID       = report.MakeEndpointNodeID(serverHostID, serverIP, serverPort)      // apache
-
-	clientAddressNodeID  = report.MakeAddressNodeID(clientHostID, "10.10.10.20")
-	serverAddressNodeID  = report.MakeAddressNodeID(serverHostID, "192.168.1.1")
-	randomAddressNodeID  = report.MakeAddressNodeID(randomHostID, "172.16.11.9") // only in Address topology
-	unknownAddressNodeID = report.MakeAddressNodeID(unknownHostID, "10.10.10.10")
+	randomClientNodeID   = report.MakeEndpointNodeID(serverHostID, "51.52.53.54", "12345")    // this should become an internet node
 
 	clientProcessNodeID       = report.MakeProcessNodeID(clientHostID, clientPID)
 	serverProcessNodeID       = report.MakeProcessNodeID(serverHostID, serverPID)
@@ -58,6 +55,33 @@ var (
 	serverContainerImageID     = "imageid456"
 	clientContainerImageNodeID = report.MakeContainerNodeID(clientHostID, clientContainerImageID)
 	serverContainerImageNodeID = report.MakeContainerNodeID(serverHostID, serverContainerImageID)
+
+	clientAddressNodeID   = report.MakeAddressNodeID(clientHostID, "10.10.10.20")
+	serverAddressNodeID   = report.MakeAddressNodeID(serverHostID, "192.168.1.1")
+	unknownAddress1NodeID = report.MakeAddressNodeID(serverHostID, "10.10.10.10")
+	unknownAddress2NodeID = report.MakeAddressNodeID(serverHostID, "10.10.10.11")
+	randomAddressNodeID   = report.MakeAddressNodeID(serverHostID, "51.52.53.54") // this should become an internet node
+
+	unknownPseudoNode1ID = "pseudo;10.10.10.10;192.168.1.1;80"
+	unknownPseudoNode2ID = "pseudo;10.10.10.11;192.168.1.1;80"
+	unknownPseudoNode1   = render.RenderableNode{
+		ID:                unknownPseudoNode1ID,
+		LabelMajor:        "10.10.10.10",
+		Pseudo:            true,
+		AggregateMetadata: render.AggregateMetadata{},
+	}
+	unknownPseudoNode2 = render.RenderableNode{
+		ID:                unknownPseudoNode2ID,
+		LabelMajor:        "10.10.10.11",
+		Pseudo:            true,
+		AggregateMetadata: render.AggregateMetadata{},
+	}
+	theInternetNode = render.RenderableNode{
+		ID:                render.TheInternetID,
+		LabelMajor:        render.TheInternetMajor,
+		Pseudo:            true,
+		AggregateMetadata: render.AggregateMetadata{},
+	}
 )
 
 var (
@@ -66,7 +90,9 @@ var (
 			Adjacency: report.Adjacency{
 				report.MakeAdjacencyID(client54001NodeID): report.MakeIDList(server80NodeID),
 				report.MakeAdjacencyID(client54002NodeID): report.MakeIDList(server80NodeID),
-				report.MakeAdjacencyID(server80NodeID):    report.MakeIDList(client54001NodeID, client54002NodeID, unknownClient1NodeID, unknownClient2NodeID, unknownClient3NodeID),
+				report.MakeAdjacencyID(server80NodeID): report.MakeIDList(
+					client54001NodeID, client54002NodeID, unknownClient1NodeID, unknownClient2NodeID,
+					unknownClient3NodeID, randomClientNodeID),
 			},
 			NodeMetadatas: report.NodeMetadatas{
 				// NodeMetadata is arbitrary. We're free to put only precisely what we
@@ -186,21 +212,16 @@ var (
 		Address: report.Topology{
 			Adjacency: report.Adjacency{
 				report.MakeAdjacencyID(clientAddressNodeID): report.MakeIDList(serverAddressNodeID),
-				report.MakeAdjacencyID(randomAddressNodeID): report.MakeIDList(serverAddressNodeID),
-				report.MakeAdjacencyID(serverAddressNodeID): report.MakeIDList(clientAddressNodeID, unknownAddressNodeID), // no backlink to random
+				report.MakeAdjacencyID(serverAddressNodeID): report.MakeIDList(
+					clientAddressNodeID, unknownAddress1NodeID, unknownAddress2NodeID, randomAddressNodeID), // no backlinks to unknown/random
 			},
 			NodeMetadatas: report.NodeMetadatas{
 				clientAddressNodeID: report.NodeMetadata{
-					"name":            "client.hostname.com", // hostname
-					"host_name":       "client.hostname.com",
+					"addr":            clientIP,
 					report.HostNodeID: clientHostNodeID,
 				},
-				randomAddressNodeID: report.NodeMetadata{
-					"name":            "random.hostname.com", // hostname
-					report.HostNodeID: randomHostNodeID,
-				},
 				serverAddressNodeID: report.NodeMetadata{
-					"name":            "server.hostname.com", // hostname
+					"addr":            serverIP,
 					report.HostNodeID: serverHostNodeID,
 				},
 			},
@@ -209,28 +230,28 @@ var (
 					WithConnCountTCP: true,
 					MaxConnCountTCP:  3,
 				},
-				report.MakeEdgeID(randomAddressNodeID, serverAddressNodeID): report.EdgeMetadata{
-					WithConnCountTCP: true,
-					MaxConnCountTCP:  20, // dangling connections, weird but possible
-				},
 				report.MakeEdgeID(serverAddressNodeID, clientAddressNodeID): report.EdgeMetadata{
 					WithConnCountTCP: true,
 					MaxConnCountTCP:  3,
 				},
-				report.MakeEdgeID(serverAddressNodeID, unknownAddressNodeID): report.EdgeMetadata{
-					WithConnCountTCP: true,
-					MaxConnCountTCP:  7,
-				},
 			},
 		},
 		Host: report.Topology{
 			Adjacency: report.Adjacency{},
 			NodeMetadatas: report.NodeMetadatas{
+				clientHostNodeID: report.NodeMetadata{
+					"host_name":       clientHostName,
+					"local_networks":  "10.10.10.0/24",
+					"os":              "Linux",
+					"load":            "0.01 0.01 0.01",
+					report.HostNodeID: clientHostNodeID,
+				},
 				serverHostNodeID: report.NodeMetadata{
-					"host_name":      serverHostName,
-					"local_networks": "10.10.10.0/24",
-					"os":             "Linux",
-					"load":           "0.01 0.01 0.01",
+					"host_name":       serverHostName,
+					"local_networks":  "10.10.10.0/24",
+					"os":              "Linux",
+					"load":            "0.01 0.01 0.01",
+					report.HostNodeID: serverHostNodeID,
 				},
 			},
 			EdgeMetadatas: report.EdgeMetadatas{},
@@ -282,8 +303,9 @@ func TestProcessRenderer(t *testing.T) {
 			Pseudo:     false,
 			Adjacency: report.MakeIDList(
 				clientProcessID,
-				"pseudo;10.10.10.10;192.168.1.1;80",
-				"pseudo;10.10.10.11;192.168.1.1;80",
+				unknownPseudoNode1ID,
+				unknownPseudoNode2ID,
+				render.TheInternetID,
 			),
 			Origins: report.MakeIDList(server80NodeID, serverProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{
@@ -301,18 +323,9 @@ func TestProcessRenderer(t *testing.T) {
 			Origins:           report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{},
 		},
-		"pseudo;10.10.10.10;192.168.1.1;80": {
-			ID:                "pseudo;10.10.10.10;192.168.1.1;80",
-			LabelMajor:        "10.10.10.10",
-			Pseudo:            true,
-			AggregateMetadata: render.AggregateMetadata{},
-		},
-		"pseudo;10.10.10.11;192.168.1.1;80": {
-			ID:                "pseudo;10.10.10.11;192.168.1.1;80",
-			LabelMajor:        "10.10.10.11",
-			Pseudo:            true,
-			AggregateMetadata: render.AggregateMetadata{},
-		},
+		unknownPseudoNode1ID: unknownPseudoNode1,
+		unknownPseudoNode2ID: unknownPseudoNode2,
+		render.TheInternetID: theInternetNode,
 	}
 	have := render.ProcessRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
@@ -347,8 +360,9 @@ func TestProcessNameRenderer(t *testing.T) {
 			Pseudo:     false,
 			Adjacency: report.MakeIDList(
 				"curl",
-				"pseudo;10.10.10.10;192.168.1.1;80",
-				"pseudo;10.10.10.11;192.168.1.1;80",
+				unknownPseudoNode1ID,
+				unknownPseudoNode2ID,
+				render.TheInternetID,
 			),
 			Origins: report.MakeIDList(server80NodeID, serverProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{
@@ -365,18 +379,9 @@ func TestProcessNameRenderer(t *testing.T) {
 			Origins:           report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{},
 		},
-		"pseudo;10.10.10.10;192.168.1.1;80": {
-			ID:                "pseudo;10.10.10.10;192.168.1.1;80",
-			LabelMajor:        "10.10.10.10",
-			Pseudo:            true,
-			AggregateMetadata: render.AggregateMetadata{},
-		},
-		"pseudo;10.10.10.11;192.168.1.1;80": {
-			ID:                "pseudo;10.10.10.11;192.168.1.1;80",
-			LabelMajor:        "10.10.10.11",
-			Pseudo:            true,
-			AggregateMetadata: render.AggregateMetadata{},
-		},
+		unknownPseudoNode1ID: unknownPseudoNode1,
+		unknownPseudoNode2ID: unknownPseudoNode2,
+		render.TheInternetID: theInternetNode,
 	}
 	have := render.ProcessNameRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
@@ -406,7 +411,7 @@ func TestContainerRenderer(t *testing.T) {
 			LabelMinor: serverHostName,
 			Rank:       serverContainerImageID,
 			Pseudo:     false,
-			Adjacency:  report.MakeIDList(clientContainerID, render.UncontainedID),
+			Adjacency:  report.MakeIDList(clientContainerID, render.UncontainedID, render.TheInternetID),
 			Origins:    report.MakeIDList(serverContainerNodeID, server80NodeID, serverProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{
 				render.KeyBytesIngress: 150,
@@ -422,6 +427,7 @@ func TestContainerRenderer(t *testing.T) {
 			Origins:           report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{},
 		},
+		render.TheInternetID: theInternetNode,
 	}
 	have := render.ContainerRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
@@ -451,7 +457,7 @@ func TestContainerImageRenderer(t *testing.T) {
 			LabelMinor: "",
 			Rank:       serverContainerImageID,
 			Pseudo:     false,
-			Adjacency:  report.MakeIDList(clientContainerImageID, render.UncontainedID),
+			Adjacency:  report.MakeIDList(clientContainerImageID, render.UncontainedID, render.TheInternetID),
 			Origins:    report.MakeIDList(serverContainerImageNodeID, serverContainerNodeID, server80NodeID, serverProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{
 				render.KeyBytesIngress: 150,
@@ -467,6 +473,7 @@ func TestContainerImageRenderer(t *testing.T) {
 			Origins:           report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID),
 			AggregateMetadata: render.AggregateMetadata{},
 		},
+		render.TheInternetID: theInternetNode,
 	}
 	have := render.ContainerImageRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
@@ -475,60 +482,47 @@ func TestContainerImageRenderer(t *testing.T) {
 	}
 }
 
-func TestRenderByNetworkHostname(t *testing.T) {
+func TestHostRenderer(t *testing.T) {
 	want := render.RenderableNodes{
-		"host:client.hostname.com": {
-			ID:         "host:client.hostname.com",
-			LabelMajor: "client",       // before first .
-			LabelMinor: "hostname.com", // after first .
-			Rank:       "client",
-			Pseudo:     false,
-			Adjacency:  report.MakeIDList("host:server.hostname.com"),
-			Origins:    report.MakeIDList(report.MakeHostNodeID("client.hostname.com"), report.MakeAddressNodeID("client.hostname.com", "10.10.10.20")),
-			AggregateMetadata: render.AggregateMetadata{
-				render.KeyMaxConnCountTCP: 3,
-			},
-		},
-		"host:random.hostname.com": {
-			ID:         "host:random.hostname.com",
-			LabelMajor: "random",       // before first .
-			LabelMinor: "hostname.com", // after first .
-			Rank:       "random",
-			Pseudo:     false,
-			Adjacency:  report.MakeIDList("host:server.hostname.com"),
-			Origins:    report.MakeIDList(report.MakeHostNodeID("random.hostname.com"), report.MakeAddressNodeID("random.hostname.com", "172.16.11.9")),
-			AggregateMetadata: render.AggregateMetadata{
-				render.KeyMaxConnCountTCP: 20,
-			},
-		},
 		"host:server.hostname.com": {
 			ID:         "host:server.hostname.com",
 			LabelMajor: "server",       // before first .
 			LabelMinor: "hostname.com", // after first .
-			Rank:       "server",
+			Rank:       "hostname.com",
 			Pseudo:     false,
-			Adjacency:  report.MakeIDList("host:client.hostname.com", "pseudo;10.10.10.10;192.168.1.1;"),
-			Origins:    report.MakeIDList(report.MakeHostNodeID("server.hostname.com"), report.MakeAddressNodeID("server.hostname.com", "192.168.1.1")),
+			Adjacency:  report.MakeIDList("host:client.hostname.com", render.TheInternetID, "pseudo;10.10.10.10;192.168.1.1;", "pseudo;10.10.10.11;192.168.1.1;"),
+			Origins:    report.MakeIDList(serverHostNodeID, serverAddressNodeID),
 			AggregateMetadata: render.AggregateMetadata{
-				render.KeyMaxConnCountTCP: 10,
+				render.KeyMaxConnCountTCP: 3,
+			},
+		},
+		"host:client.hostname.com": {
+			ID:         "host:client.hostname.com",
+			LabelMajor: "client",       // before first .
+			LabelMinor: "hostname.com", // after first .
+			Rank:       "hostname.com",
+			Pseudo:     false,
+			Adjacency:  report.MakeIDList("host:server.hostname.com"),
+			Origins:    report.MakeIDList(clientHostNodeID, clientAddressNodeID),
+			AggregateMetadata: render.AggregateMetadata{
+				render.KeyMaxConnCountTCP: 3,
 			},
 		},
 		"pseudo;10.10.10.10;192.168.1.1;": {
 			ID:                "pseudo;10.10.10.10;192.168.1.1;",
 			LabelMajor:        "10.10.10.10",
-			LabelMinor:        "", // after first .
-			Rank:              "",
 			Pseudo:            true,
-			Adjacency:         nil,
-			Origins:           nil,
 			AggregateMetadata: render.AggregateMetadata{},
 		},
+		"pseudo;10.10.10.11;192.168.1.1;": {
+			ID:                "pseudo;10.10.10.11;192.168.1.1;",
+			LabelMajor:        "10.10.10.11",
+			Pseudo:            true,
+			AggregateMetadata: render.AggregateMetadata{},
+		},
+		render.TheInternetID: theInternetNode,
 	}
-	have := render.LeafMap{
-		Selector: report.SelectAddress,
-		Mapper:   render.NetworkHostname,
-		Pseudo:   render.GenericPseudoNode,
-	}.Render(rpt)
+	have := render.HostRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
 	if !reflect.DeepEqual(want, have) {
 		t.Error("\n" + test.Diff(want, have))

report/id.go

@@ -1,7 +1,6 @@
 package report
 
 import (
-	"fmt"
 	"net"
 	"strings"
 )
@@ -131,13 +130,6 @@ func AddressIDAddresser(id string) net.IP {
 	return net.ParseIP(fields[1])
 }
 
-// PanicIDAddresser will panic if it's ever called. It's used in topologies
-// where there are never any edges, and so it's nonsensical to try and extract
-// IPs from the node IDs.
-func PanicIDAddresser(id string) net.IP {
-	panic(fmt.Sprintf("PanicIDAddresser called on %q", id))
-}
-
 func isLoopback(address string) bool {
 	ip := net.ParseIP(address)
 	return ip != nil && ip.IsLoopback()

report/id_test.go

@@ -126,17 +126,3 @@ func TestAddressIDAddresser(t *testing.T) {
 		t.Errorf("want %s, have %s", want, have)
 	}
 }
-
-func TestPanicIDAddresser(t *testing.T) {
-	if panicked := func() (recovered bool) {
-		defer func() {
-			if r := recover(); r != nil {
-				recovered = true
-			}
-		}()
-		report.PanicIDAddresser("irrelevant")
-		return false
-	}(); !panicked {
-		t.Errorf("expected panic, didn't get it")
-	}
-}

report/report.go

@@ -58,11 +58,6 @@ func SelectProcess(r Report) Topology {
 	return r.Process
 }
 
-// SelectAddress selects the address topology.
-func SelectAddress(r Report) Topology {
-	return r.Address
-}
-
 // SelectContainer selects the container topology.
 func SelectContainer(r Report) Topology {
 	return r.Container
@@ -73,6 +68,16 @@ func SelectContainerImage(r Report) Topology {
 	return r.ContainerImage
 }
 
+// SelectAddress selects the address topology.
+func SelectAddress(r Report) Topology {
+	return r.Address
+}
+
+// SelectHost selects the address topology.
+func SelectHost(r Report) Topology {
+	return r.Host
+}
+
 // MakeReport makes a clean report, ready to Merge() other reports into.
 func MakeReport() Report {
 	return Report{