Merge pull request #254 from tomwilkie/230-container-image-topology

Add ContainerImage topology and use it to build the Container By Image graph.

app/router.go

@@ -65,7 +65,7 @@ var topologyRegistry = map[string]topologyView{
 	"containers-by-image": {
 		human:    "by image",
 		parent:   "containers",
-		renderer: render.LeafMap{Selector: report.SelectEndpoint, Mapper: render.ProcessContainerImage, Pseudo: render.InternetOnlyPseudoNode},
+		renderer: render.ContainerImageRenderer,
 	},
 	"hosts": {
 		human:    "Hosts",

probe/main.go

@@ -130,6 +130,7 @@ func main() {
 
 				if dockerTagger != nil {
 					r.Container.Merge(dockerTagger.ContainerTopology(hostID))
+					r.ContainerImage.Merge(dockerTagger.ContainerImageTopology(hostID))
 				}
 
 				if weaveTagger != nil {

probe/tag/docker_tagger.go

@@ -272,7 +272,6 @@ func (t *DockerTagger) Containers() []*docker.Container {
 // Tag implements Tagger.
 func (t *DockerTagger) Tag(r report.Report) report.Report {
 	t.tag(&r.Process)
-	t.tag(&r.Endpoint)
 	return r
 }
 
@@ -313,15 +312,6 @@ func (t *DockerTagger) tag(topology *report.Topology) {
 
 		md := report.NodeMetadata{
 			ContainerID: container.ID,
-			ImageID:     container.Image,
-		}
-
-		t.RLock()
-		image, ok := t.images[container.Image]
-		t.RUnlock()
-
-		if ok && len(image.RepoTags) > 0 {
-			md[ImageName] = image.RepoTags[0]
 		}
 
 		topology.NodeMetadatas[nodeID].Merge(md)
@@ -341,11 +331,6 @@ func (t *DockerTagger) ContainerTopology(scope string) report.Topology {
 			ImageID:       container.Image,
 		}
 
-		image, ok := t.images[container.Image]
-		if ok && len(image.RepoTags) > 0 {
-			nmd[ImageName] = image.RepoTags[0]
-		}
-
 		nmd.Merge(container.getStats())
 
 		nodeID := report.MakeContainerNodeID(scope, container.ID)
@@ -353,3 +338,27 @@ func (t *DockerTagger) ContainerTopology(scope string) report.Topology {
 	}
 	return result
 }
+
+// ContainerImageTopology produces a Toplogy of Container Images
+func (t *DockerTagger) ContainerImageTopology(scope string) report.Topology {
+	t.RLock()
+	defer t.RUnlock()
+
+	result := report.NewTopology()
+
+	// Loop over containers so we only emit images for running containers.
+	for _, container := range t.containers {
+		nmd := report.NodeMetadata{
+			ImageID: container.Image,
+		}
+
+		image, ok := t.images[container.Image]
+		if ok && len(image.RepoTags) > 0 {
+			nmd[ImageName] = image.RepoTags[0]
+		}
+
+		nodeID := report.MakeContainerNodeID(scope, container.Image)
+		result.NodeMetadatas[nodeID] = nmd
+	}
+	return result
+}

probe/tag/docker_tagger_test.go

@@ -8,6 +8,7 @@ import (
 
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/weaveworks/scope/report"
+	"github.com/weaveworks/scope/test"
 )
 
 type mockDockerClient struct {
@@ -67,18 +68,31 @@ func TestDockerTagger(t *testing.T) {
 	}
 
 	var (
-		pid1NodeID           = report.MakeProcessNodeID("somehost.com", "1")
-		pid2NodeID           = report.MakeProcessNodeID("somehost.com", "2")
-		endpointNodeMetadata = report.NodeMetadata{
-			ContainerID: "foo",
-			ImageID:     "baz",
-			ImageName:   "bang",
-		}
+		pid1NodeID          = report.MakeProcessNodeID("somehost.com", "1")
+		pid2NodeID          = report.MakeProcessNodeID("somehost.com", "2")
 		processNodeMetadata = report.NodeMetadata{
-			ContainerID:   "foo",
-			ContainerName: "bar",
-			ImageID:       "baz",
-			ImageName:     "bang",
+			ContainerID: "foo",
+		}
+		wantContainerTopology = report.Topology{
+			Adjacency:     report.Adjacency{},
+			EdgeMetadatas: report.EdgeMetadatas{},
+			NodeMetadatas: report.NodeMetadatas{
+				report.MakeContainerNodeID("", "foo"): report.NodeMetadata{
+					ContainerID:   "foo",
+					ContainerName: "bar",
+					ImageID:       "baz",
+				},
+			},
+		}
+		wantContainerImageTopology = report.Topology{
+			Adjacency:     report.Adjacency{},
+			EdgeMetadatas: report.EdgeMetadatas{},
+			NodeMetadatas: report.NodeMetadatas{
+				report.MakeContainerNodeID("", "baz"): report.NodeMetadata{
+					ImageID:   "baz",
+					ImageName: "bang",
+				},
+			},
 		}
 	)
 
@@ -89,7 +103,7 @@ func TestDockerTagger(t *testing.T) {
 	dockerTagger, _ := NewDockerTagger("/irrelevant", 10*time.Second)
 	runtime.Gosched()
 	for _, nodeID := range []string{pid1NodeID, pid2NodeID} {
-		want := endpointNodeMetadata.Copy()
+		want := processNodeMetadata.Copy()
 		have := dockerTagger.Tag(r).Process.NodeMetadatas[nodeID].Copy()
 		delete(have, "pid")
 		if !reflect.DeepEqual(want, have) {
@@ -97,10 +111,13 @@ func TestDockerTagger(t *testing.T) {
 		}
 	}
 
-	wantTopology := report.NewTopology()
-	wantTopology.NodeMetadatas[report.MakeContainerNodeID("", "foo")] = processNodeMetadata
-	haveTopology := dockerTagger.ContainerTopology("")
-	if !reflect.DeepEqual(wantTopology, haveTopology) {
-		t.Errorf("toplog want %+v, have %+v", wantTopology, haveTopology)
+	haveContainerTopology := dockerTagger.ContainerTopology("")
+	if !reflect.DeepEqual(wantContainerTopology, haveContainerTopology) {
+		t.Errorf("%s", test.Diff(wantContainerTopology, haveContainerTopology))
+	}
+
+	haveContainerImageTopology := dockerTagger.ContainerImageTopology("")
+	if !reflect.DeepEqual(wantContainerImageTopology, haveContainerImageTopology) {
+		t.Errorf("%s", test.Diff(wantContainerImageTopology, haveContainerImageTopology))
 	}
 }

render/detailed_node.go

@@ -93,6 +93,9 @@ func OriginTable(r report.Report, originID string) (Table, bool) {
 	if nmd, ok := r.Container.NodeMetadatas[originID]; ok {
 		return containerOriginTable(nmd)
 	}
+	if nmd, ok := r.ContainerImage.NodeMetadatas[originID]; ok {
+		return containerImageOriginTable(nmd)
+	}
 	if nmd, ok := r.Host.NodeMetadatas[originID]; ok {
 		return hostOriginTable(nmd)
 	}
@@ -155,7 +158,6 @@ func containerOriginTable(nmd report.NodeMetadata) (Table, bool) {
 		{"docker_container_id", "Container ID"},
 		{"docker_container_name", "Container name"},
 		{"docker_image_id", "Container image ID"},
-		{"docker_image_name", "Container image name"},
 	} {
 		if val, ok := nmd[tuple.key]; ok {
 			rows = append(rows, Row{Key: tuple.human, ValueMajor: val, ValueMinor: ""})
@@ -168,6 +170,23 @@ func containerOriginTable(nmd report.NodeMetadata) (Table, bool) {
 	}, len(rows) > 0
 }
 
+func containerImageOriginTable(nmd report.NodeMetadata) (Table, bool) {
+	rows := []Row{}
+	for _, tuple := range []struct{ key, human string }{
+		{"docker_image_id", "Container image ID"},
+		{"docker_image_name", "Container image name"},
+	} {
+		if val, ok := nmd[tuple.key]; ok {
+			rows = append(rows, Row{Key: tuple.human, ValueMajor: val, ValueMinor: ""})
+		}
+	}
+	return Table{
+		Title:   "Origin Container Image",
+		Numeric: false,
+		Rows:    rows,
+	}, len(rows) > 0
+}
+
 func hostOriginTable(nmd report.NodeMetadata) (Table, bool) {
 	rows := []Row{}
 	if val, ok := nmd["host_name"]; ok {

render/detailed_node_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/weaveworks/scope/render"
+	"github.com/weaveworks/scope/test"
 )
 
 func TestOriginTable(t *testing.T) {
@@ -51,7 +52,7 @@ func TestOriginTable(t *testing.T) {
 			continue
 		}
 		if !reflect.DeepEqual(want, have) {
-			t.Errorf("%q: %s", originID, diff(want, have))
+			t.Errorf("%q: %s", originID, test.Diff(want, have))
 		}
 	}
 }
@@ -95,6 +96,7 @@ func TestMakeDetailedNode(t *testing.T) {
 				Rows: []render.Row{
 					{"Container ID", "5e4d3c2b1a", ""},
 					{"Container name", "server", ""},
+					{"Container image ID", "imageid456", ""},
 				},
 			},
 			{
@@ -109,6 +111,6 @@ func TestMakeDetailedNode(t *testing.T) {
 		},
 	}
 	if !reflect.DeepEqual(want, have) {
-		t.Errorf("%s", diff(want, have))
+		t.Errorf("%s", test.Diff(want, have))
 	}
 }

render/mapping.go

@@ -87,6 +87,19 @@ func MapContainerIdentity(m report.NodeMetadata) (RenderableNode, bool) {
 	return NewRenderableNode(id, major, minor, rank, m), true
 }
 
+// MapContainerImageIdentity maps a container image topology node to container
+// image RenderableNode node. As it is only ever run on container image
+// topology nodes, we can safely assume the presences of certain keys.
+func MapContainerImageIdentity(m report.NodeMetadata) (RenderableNode, bool) {
+	var (
+		id    = m["docker_image_id"]
+		major = m["docker_image_name"]
+		rank  = m["docker_image_id"]
+	)
+
+	return NewRenderableNode(id, major, "", rank, m), true
+}
+
 // MapEndpoint2Process maps endpoint RenderableNodes to process
 // RenderableNodes.
 //
@@ -156,18 +169,24 @@ func MapProcess2Name(n RenderableNode) (RenderableNode, bool) {
 	return node, true
 }
 
-// ProcessContainerImage maps topology nodes to the container images they run
-// on. If no container metadata is found, nodes are grouped into the
-// Uncontained node.
-func ProcessContainerImage(m report.NodeMetadata) (RenderableNode, bool) {
-	var id, major, minor, rank string
-	if m["docker_image_id"] == "" {
-		id, major, minor, rank = UncontainedID, UncontainedMajor, "", UncontainedID
-	} else {
-		id, major, minor, rank = m["docker_image_id"], m["docker_image_name"], "", m["docker_image_id"]
+// MapContainer2ContainerImage maps container RenderableNodes to container
+// image RenderableNodes.
+//
+// If this function is given a node without a docker_image_id
+// (including other pseudo nodes), it will produce an "Uncontained"
+// pseudo node.
+//
+// Otherwise, this function will produce a node with the correct ID
+// format for a container, but without any Major or Minor labels.
+// It does not have enough info to do that, and the resulting graph
+// must be merged with a container graph to get that info.
+func MapContainer2ContainerImage(n RenderableNode) (RenderableNode, bool) {
+	id, ok := n.NodeMetadata["docker_image_id"]
+	if !ok || n.Pseudo {
+		return newDerivedPseudoNode(UncontainedID, UncontainedMajor, n), true
 	}
 
-	return NewRenderableNode(id, major, minor, rank, m), true
+	return newDerivedNode(id, n), true
 }
 
 // NetworkHostname takes a node NodeMetadata and returns a representation

render/topologies.go

@@ -45,3 +45,17 @@ var ContainerRenderer = MakeReduce(
 		Pseudo:   GenericPseudoNode,
 	},
 )
+
+// ContainerImageRenderer is a Renderer which produces a renderable container
+// image graph by merging the container graph and the container image topology.
+var ContainerImageRenderer = MakeReduce(
+	Map{
+		MapFunc:  MapContainer2ContainerImage,
+		Renderer: ContainerRenderer,
+	},
+	LeafMap{
+		Selector: report.SelectContainerImage,
+		Mapper:   MapContainerImageIdentity,
+		Pseudo:   GenericPseudoNode,
+	},
+)

render/topologies_test.go

@@ -5,17 +5,11 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/davecgh/go-spew/spew"
-	"github.com/pmezard/go-difflib/difflib"
-
 	"github.com/weaveworks/scope/render"
 	"github.com/weaveworks/scope/report"
+	"github.com/weaveworks/scope/test"
 )
 
-func init() {
-	spew.Config.SortKeys = true // :\
-}
-
 var (
 	clientHostID  = "client.hostname.com"
 	serverHostID  = "server.hostname.com"
@@ -59,6 +53,11 @@ var (
 	serverContainerID     = "5e4d3c2b1a"
 	clientContainerNodeID = report.MakeContainerNodeID(clientHostID, clientContainerID)
 	serverContainerNodeID = report.MakeContainerNodeID(serverHostID, serverContainerID)
+
+	clientContainerImageID     = "imageid123"
+	serverContainerImageID     = "imageid456"
+	clientContainerImageNodeID = report.MakeContainerNodeID(clientHostID, clientContainerImageID)
+	serverContainerImageNodeID = report.MakeContainerNodeID(serverHostID, serverContainerImageID)
 )
 
 var (
@@ -159,15 +158,31 @@ var (
 				clientContainerNodeID: report.NodeMetadata{
 					"docker_container_id":   clientContainerID,
 					"docker_container_name": "client",
+					"docker_image_id":       clientContainerImageID,
 					report.HostNodeID:       clientHostNodeID,
 				},
 				serverContainerNodeID: report.NodeMetadata{
 					"docker_container_id":   serverContainerID,
 					"docker_container_name": "server",
+					"docker_image_id":       serverContainerImageID,
 					report.HostNodeID:       serverHostNodeID,
 				},
 			},
 		},
+		ContainerImage: report.Topology{
+			NodeMetadatas: report.NodeMetadatas{
+				clientContainerImageNodeID: report.NodeMetadata{
+					"docker_image_id":   clientContainerImageID,
+					"docker_image_name": "client_image",
+					report.HostNodeID:   clientHostNodeID,
+				},
+				serverContainerImageNodeID: report.NodeMetadata{
+					"docker_image_id":   serverContainerImageID,
+					"docker_image_name": "server_image",
+					report.HostNodeID:   serverHostNodeID,
+				},
+			},
+		},
 		Address: report.Topology{
 			Adjacency: report.Adjacency{
 				report.MakeAdjacencyID(clientAddressNodeID): report.MakeIDList(serverAddressNodeID),
@@ -302,7 +317,7 @@ func TestProcessRenderer(t *testing.T) {
 	have := render.ProcessRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
 	if !reflect.DeepEqual(want, have) {
-		t.Error("\n" + diff(want, have))
+		t.Error("\n" + test.Diff(want, have))
 	}
 }
 
@@ -366,20 +381,17 @@ func TestProcessNameRenderer(t *testing.T) {
 	have := render.ProcessNameRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
 	if !reflect.DeepEqual(want, have) {
-		t.Error("\n" + diff(want, have))
+		t.Error("\n" + test.Diff(want, have))
 	}
 }
 
 func TestContainerRenderer(t *testing.T) {
-	// For grouped, I've somewhat arbitrarily chosen to squash together all
-	// processes with the same name by removing the PID and domain (host)
-	// dimensions from the ID. That could be changed.
 	want := render.RenderableNodes{
 		clientContainerID: {
 			ID:         clientContainerID,
 			LabelMajor: "client",
 			LabelMinor: clientHostName,
-			Rank:       "",
+			Rank:       clientContainerImageID,
 			Pseudo:     false,
 			Adjacency:  report.MakeIDList(serverContainerID),
 			Origins:    report.MakeIDList(clientContainerNodeID, client54001NodeID, client54002NodeID, clientProcessNodeID, clientHostNodeID),
@@ -392,7 +404,7 @@ func TestContainerRenderer(t *testing.T) {
 			ID:         serverContainerID,
 			LabelMajor: "server",
 			LabelMinor: serverHostName,
-			Rank:       "",
+			Rank:       serverContainerImageID,
 			Pseudo:     false,
 			Adjacency:  report.MakeIDList(clientContainerID, render.UncontainedID),
 			Origins:    report.MakeIDList(serverContainerNodeID, server80NodeID, serverProcessNodeID, serverHostNodeID),
@@ -414,7 +426,52 @@ func TestContainerRenderer(t *testing.T) {
 	have := render.ContainerRenderer.Render(rpt)
 	have = trimNodeMetadata(have)
 	if !reflect.DeepEqual(want, have) {
-		t.Error("\n" + diff(want, have))
+		t.Error("\n" + test.Diff(want, have))
+	}
+}
+
+func TestContainerImageRenderer(t *testing.T) {
+	want := render.RenderableNodes{
+		clientContainerImageID: {
+			ID:         clientContainerImageID,
+			LabelMajor: "client_image",
+			LabelMinor: "",
+			Rank:       clientContainerImageID,
+			Pseudo:     false,
+			Adjacency:  report.MakeIDList(serverContainerImageID),
+			Origins:    report.MakeIDList(clientContainerImageNodeID, clientContainerNodeID, client54001NodeID, client54002NodeID, clientProcessNodeID, clientHostNodeID),
+			AggregateMetadata: render.AggregateMetadata{
+				render.KeyBytesIngress: 300,
+				render.KeyBytesEgress:  30,
+			},
+		},
+		serverContainerImageID: {
+			ID:         serverContainerImageID,
+			LabelMajor: "server_image",
+			LabelMinor: "",
+			Rank:       serverContainerImageID,
+			Pseudo:     false,
+			Adjacency:  report.MakeIDList(clientContainerImageID, render.UncontainedID),
+			Origins:    report.MakeIDList(serverContainerImageNodeID, serverContainerNodeID, server80NodeID, serverProcessNodeID, serverHostNodeID),
+			AggregateMetadata: render.AggregateMetadata{
+				render.KeyBytesIngress: 150,
+				render.KeyBytesEgress:  1500,
+			},
+		},
+		render.UncontainedID: {
+			ID:                render.UncontainedID,
+			LabelMajor:        render.UncontainedMajor,
+			LabelMinor:        "",
+			Rank:              "",
+			Pseudo:            true,
+			Origins:           report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID),
+			AggregateMetadata: render.AggregateMetadata{},
+		},
+	}
+	have := render.ContainerImageRenderer.Render(rpt)
+	have = trimNodeMetadata(have)
+	if !reflect.DeepEqual(want, have) {
+		t.Error("\n" + test.Diff(want, have))
 	}
 }
 
@@ -474,17 +531,6 @@ func TestRenderByNetworkHostname(t *testing.T) {
 	}.Render(rpt)
 	have = trimNodeMetadata(have)
 	if !reflect.DeepEqual(want, have) {
-		t.Error("\n" + diff(want, have))
+		t.Error("\n" + test.Diff(want, have))
 	}
 }
-
-func diff(want, have interface{}) string {
-	text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
-		A:        difflib.SplitLines(spew.Sdump(want)),
-		B:        difflib.SplitLines(spew.Sdump(have)),
-		FromFile: "want",
-		ToFile:   "have",
-		Context:  3,
-	})
-	return "\n" + text
-}

render/topology_diff_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/weaveworks/scope/render"
+	"github.com/weaveworks/scope/test"
 )
 
 // ByID is a sort interface for a RenderableNode slice.
@@ -87,7 +88,7 @@ func TestTopoDiff(t *testing.T) {
 		sort.Sort(ByID(c.have.Add))
 		sort.Sort(ByID(c.have.Update))
 		if !reflect.DeepEqual(c.want, c.have) {
-			t.Errorf("%s - %s", c.label, diff(c.want, c.have))
+			t.Errorf("%s - %s", c.label, test.Diff(c.want, c.have))
 		}
 	}
 }

report/merge.go

@@ -10,6 +10,7 @@ func (r *Report) Merge(other Report) {
 	r.Address.Merge(other.Address)
 	r.Process.Merge(other.Process)
 	r.Container.Merge(other.Container)
+	r.ContainerImage.Merge(other.ContainerImage)
 	r.Host.Merge(other.Host)
 	r.Overlay.Merge(other.Overlay)
 }

report/report.go

@@ -23,10 +23,15 @@ type Report struct {
 	Process Topology
 
 	// Container nodes represent all Docker containers on hosts running probes.
-	// Metadata includes things like Docker image, name etc.
+	// Metadata includes things like containter id, name, image id etc.
 	// Edges are not present.
 	Container Topology
 
+	// ContainerImages nodes represent all Docker containers images on
+	// hosts running probes. Metadata includes things like image id, name etc.
+	// Edges are not present.
+	ContainerImage Topology
+
 	// Host nodes are physical hosts that run probes. Metadata includes things
 	// like operating system, load, etc. The information is scraped by the
 	// probes with each published report. Edges are not present.
@@ -68,15 +73,21 @@ func SelectContainer(r Report) Topology {
 	return r.Container
 }
 
+// SelectContainerImage selects the container image topology.
+func SelectContainerImage(r Report) Topology {
+	return r.ContainerImage
+}
+
 // MakeReport makes a clean report, ready to Merge() other reports into.
 func MakeReport() Report {
 	return Report{
-		Endpoint:  NewTopology(),
-		Address:   NewTopology(),
-		Process:   NewTopology(),
-		Container: NewTopology(),
-		Host:      NewTopology(),
-		Overlay:   NewTopology(),
+		Endpoint:       NewTopology(),
+		Address:        NewTopology(),
+		Process:        NewTopology(),
+		Container:      NewTopology(),
+		ContainerImage: NewTopology(),
+		Host:           NewTopology(),
+		Overlay:        NewTopology(),
 	}
 }
 
@@ -88,6 +99,7 @@ func (r Report) Squash() Report {
 	r.Address = r.Address.Squash(AddressIDAddresser, localNetworks)
 	r.Process = r.Process.Squash(PanicIDAddresser, localNetworks)
 	r.Container = r.Container.Squash(PanicIDAddresser, localNetworks)
+	r.ContainerImage = r.ContainerImage.Squash(PanicIDAddresser, localNetworks)
 	r.Host = r.Host.Squash(PanicIDAddresser, localNetworks)
 	r.Overlay = r.Overlay.Squash(PanicIDAddresser, localNetworks)
 	return r
@@ -123,7 +135,8 @@ func (r Report) LocalNetworks() []*net.IPNet {
 
 // Topologies returns a slice of Topologies in this report
 func (r Report) Topologies() []Topology {
-	return []Topology{r.Endpoint, r.Address, r.Process, r.Container, r.Host}
+	return []Topology{r.Endpoint, r.Address, r.Process, r.Container,
+		r.ContainerImage, r.Host, r.Overlay}
 }
 
 // Validate checks the report for various inconsistencies.

test/diff.go

@@ -0,0 +1,22 @@
+package test
+
+import (
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pmezard/go-difflib/difflib"
+)
+
+func init() {
+	spew.Config.SortKeys = true // :\
+}
+
+// Diff diff diff
+func Diff(want, have interface{}) string {
+	text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
+		A:        difflib.SplitLines(spew.Sdump(want)),
+		B:        difflib.SplitLines(spew.Sdump(have)),
+		FromFile: "want",
+		ToFile:   "have",
+		Context:  3,
+	})
+	return "\n" + text
+}