Basic Kubernetes integration

* Added helper for installing scope on gcloud
* Added topologies Pods and Pods-by-Service
* Uses k8s.io/kubernetes/pkg/client/cache for the client
* Filter kube-system nodes by default
* Only show the k8s topologies if we've received a non-empty k8s report

render/detailed_node_test.go

@@ -61,6 +61,7 @@ func TestOriginTable(t *testing.T) {
 				{fmt.Sprintf(`Label %q`, render.AmazonECSContainerNameLabel), `server`, "", false},
 				{`Label "foo1"`, `bar1`, "", false},
 				{`Label "foo2"`, `bar2`, "", false},
+				{`Label "io.kubernetes.pod.name"`, "ping/pong-b", "", false},
 			},
 		},
 	} {
@@ -162,6 +163,7 @@ func TestMakeDetailedContainerNode(t *testing.T) {
 					{fmt.Sprintf(`Label %q`, render.AmazonECSContainerNameLabel), `server`, "", false},
 					{`Label "foo1"`, `bar1`, "", false},
 					{`Label "foo2"`, `bar2`, "", false},
+					{`Label "io.kubernetes.pod.name"`, "ping/pong-b", "", false},
 				},
 			},
 			{

render/expected/expected.go

@@ -368,6 +368,141 @@ var (
 			Origins:      report.MakeIDList(test.RandomAddressNodeID),
 		},
 	}).Prune()
+
+	RenderedPods = (render.RenderableNodes{
+		"ping/pong-a": {
+			ID:         "ping/pong-a",
+			LabelMajor: "pong-a",
+			LabelMinor: "1 container",
+			Rank:       "ping/pong-a",
+			Pseudo:     false,
+			Origins: report.MakeIDList(
+				test.Client54001NodeID,
+				test.Client54002NodeID,
+				test.ClientProcess1NodeID,
+				test.ClientProcess2NodeID,
+				test.ClientHostNodeID,
+				test.ClientContainerNodeID,
+				test.ClientContainerImageNodeID,
+				test.ClientPodNodeID,
+			),
+			Node: report.MakeNode().WithAdjacent("ping/pong-b"),
+			EdgeMetadata: report.EdgeMetadata{
+				EgressPacketCount: newu64(30),
+				EgressByteCount:   newu64(300),
+			},
+		},
+		"ping/pong-b": {
+			ID:         "ping/pong-b",
+			LabelMajor: "pong-b",
+			LabelMinor: "1 container",
+			Rank:       "ping/pong-b",
+			Pseudo:     false,
+			Origins: report.MakeIDList(
+				test.Server80NodeID,
+				test.ServerPodNodeID,
+				test.ServerProcessNodeID,
+				test.ServerContainerNodeID,
+				test.ServerHostNodeID,
+				test.ServerContainerImageNodeID,
+			),
+			Node: report.MakeNode(),
+			EdgeMetadata: report.EdgeMetadata{
+				IngressPacketCount: newu64(210),
+				IngressByteCount:   newu64(2100),
+			},
+		},
+		uncontainedServerID: {
+			ID:         uncontainedServerID,
+			LabelMajor: render.UncontainedMajor,
+			LabelMinor: test.ServerHostName,
+			Rank:       "",
+			Pseudo:     true,
+			Origins: report.MakeIDList(
+				test.ServerHostNodeID,
+				test.NonContainerProcessNodeID,
+				test.NonContainerNodeID,
+			),
+			Node:         report.MakeNode().WithAdjacent(render.TheInternetID),
+			EdgeMetadata: report.EdgeMetadata{},
+		},
+		render.TheInternetID: {
+			ID:         render.TheInternetID,
+			LabelMajor: render.TheInternetMajor,
+			Pseudo:     true,
+			Node:       report.MakeNode().WithAdjacent("ping/pong-b"),
+			EdgeMetadata: report.EdgeMetadata{
+				EgressPacketCount: newu64(60),
+				EgressByteCount:   newu64(600),
+			},
+			Origins: report.MakeIDList(
+				test.RandomClientNodeID,
+				test.GoogleEndpointNodeID,
+			),
+		},
+	}).Prune()
+
+	RenderedPodServices = (render.RenderableNodes{
+		"ping/pongservice": {
+			ID:         test.ServiceID,
+			LabelMajor: "pongservice",
+			LabelMinor: "2 pods",
+			Rank:       test.ServiceID,
+			Pseudo:     false,
+			Origins: report.MakeIDList(
+				test.Client54001NodeID,
+				test.Client54002NodeID,
+				test.ClientProcess1NodeID,
+				test.ClientProcess2NodeID,
+				test.ClientHostNodeID,
+				test.ClientContainerNodeID,
+				test.ClientContainerImageNodeID,
+				test.ClientPodNodeID,
+				test.Server80NodeID,
+				test.ServerPodNodeID,
+				test.ServiceNodeID,
+				test.ServerProcessNodeID,
+				test.ServerContainerNodeID,
+				test.ServerHostNodeID,
+				test.ServerContainerImageNodeID,
+			),
+			Node: report.MakeNode().WithAdjacent(test.ServiceID), // ?? Shouldn't be adjacent to itself?
+			EdgeMetadata: report.EdgeMetadata{
+				EgressPacketCount:  newu64(30),
+				EgressByteCount:    newu64(300),
+				IngressPacketCount: newu64(210),
+				IngressByteCount:   newu64(2100),
+			},
+		},
+		uncontainedServerID: {
+			ID:         uncontainedServerID,
+			LabelMajor: render.UncontainedMajor,
+			LabelMinor: test.ServerHostName,
+			Rank:       "",
+			Pseudo:     true,
+			Origins: report.MakeIDList(
+				test.ServerHostNodeID,
+				test.NonContainerProcessNodeID,
+				test.NonContainerNodeID,
+			),
+			Node:         report.MakeNode().WithAdjacent(render.TheInternetID),
+			EdgeMetadata: report.EdgeMetadata{},
+		},
+		render.TheInternetID: {
+			ID:         render.TheInternetID,
+			LabelMajor: render.TheInternetMajor,
+			Pseudo:     true,
+			Node:       report.MakeNode().WithAdjacent(test.ServiceID),
+			EdgeMetadata: report.EdgeMetadata{
+				EgressPacketCount: newu64(60),
+				EgressByteCount:   newu64(600),
+			},
+			Origins: report.MakeIDList(
+				test.RandomClientNodeID,
+				test.GoogleEndpointNodeID,
+			),
+		},
+	}).Prune()
 )
 
 func newu64(value uint64) *uint64 { return &value }

render/mapping.go

@@ -9,6 +9,7 @@ import (
 	"github.com/weaveworks/scope/probe/docker"
 	"github.com/weaveworks/scope/probe/endpoint"
 	"github.com/weaveworks/scope/probe/host"
+	"github.com/weaveworks/scope/probe/kubernetes"
 	"github.com/weaveworks/scope/probe/process"
 	"github.com/weaveworks/scope/report"
 )
@@ -22,7 +23,9 @@ const (
 	TheInternetMajor = "The Internet"
 
 	containersKey = "containers"
+	podsKey       = "pods"
 	processesKey  = "processes"
+	servicesKey   = "services"
 
 	AmazonECSContainerNameLabel = "com.amazonaws.ecs.container-name"
 )
@@ -170,6 +173,40 @@ func MapContainerImageIdentity(m RenderableNode, _ report.Networks) RenderableNo
 	return RenderableNodes{id: NewRenderableNodeWith(id, major, "", rank, m)}
 }
 
+// MapPodIdentity maps a pod topology node to pod renderable node. As it is
+// only ever run on pod topology nodes, we expect that certain keys
+// are present.
+func MapPodIdentity(m RenderableNode, _ report.Networks) RenderableNodes {
+	id, ok := m.Metadata[kubernetes.PodID]
+	if !ok {
+		return RenderableNodes{}
+	}
+
+	var (
+		major = m.Metadata[kubernetes.PodName]
+		rank  = m.Metadata[kubernetes.PodID]
+	)
+
+	return RenderableNodes{id: NewRenderableNodeWith(id, major, "", rank, m)}
+}
+
+// MapServiceIdentity maps a service topology node to service renderable node. As it is
+// only ever run on service topology nodes, we expect that certain keys
+// are present.
+func MapServiceIdentity(m RenderableNode, _ report.Networks) RenderableNodes {
+	id, ok := m.Metadata[kubernetes.ServiceID]
+	if !ok {
+		return RenderableNodes{}
+	}
+
+	var (
+		major = m.Metadata[kubernetes.ServiceName]
+		rank  = m.Metadata[kubernetes.ServiceID]
+	)
+
+	return RenderableNodes{id: NewRenderableNodeWith(id, major, "", rank, m)}
+}
+
 // MapAddressIdentity maps an address topology node to an address renderable
 // node. As it is only ever run on address topology nodes, we expect that
 // certain keys are present.
@@ -430,6 +467,38 @@ func MapContainer2ContainerImage(n RenderableNode, _ report.Networks) Renderable
 	return RenderableNodes{id: result}
 }
 
+// MapPod2Service maps pod RenderableNodes to service RenderableNodes.
+//
+// If this function is given a node without a kubernetes_pod_id
+// (including other pseudo nodes), it will produce an "Uncontained"
+// pseudo node.
+//
+// Otherwise, this function will produce a node with the correct ID
+// format for a container, but without any Major or Minor labels.
+// It does not have enough info to do that, and the resulting graph
+// must be merged with a pod graph to get that info.
+func MapPod2Service(n RenderableNode, _ report.Networks) RenderableNodes {
+	// Propogate all pseudo nodes
+	if n.Pseudo {
+		return RenderableNodes{n.ID: n}
+	}
+
+	// Otherwise, if some some reason the pod doesn't have a service_ids (maybe
+	// slightly out of sync reports, or its not in a service), just drop it
+	ids, ok := n.Node.Metadata[kubernetes.ServiceIDs]
+	if !ok {
+		return RenderableNodes{}
+	}
+
+	result := RenderableNodes{}
+	for _, id := range strings.Fields(ids) {
+		n := NewDerivedNode(id, n)
+		n.Node.Counters[podsKey] = 1
+		result[id] = n
+	}
+	return result
+}
+
 func imageNameWithoutVersion(name string) string {
 	parts := strings.SplitN(name, ":", 2)
 	if len(parts) == 2 {
@@ -463,6 +532,45 @@ func MapContainerImage2Name(n RenderableNode, _ report.Networks) RenderableNodes
 	return RenderableNodes{name: node}
 }
 
+// MapContainer2Pod maps container RenderableNodes to pod
+// RenderableNodes.
+//
+// If this function is given a node without a kubernetes_pod_id
+// (including other pseudo nodes), it will produce an "Unmanaged"
+// pseudo node.
+//
+// Otherwise, this function will produce a node with the correct ID
+// format for a container, but without any Major or Minor labels.
+// It does not have enough info to do that, and the resulting graph
+// must be merged with a container graph to get that info.
+func MapContainer2Pod(n RenderableNode, _ report.Networks) RenderableNodes {
+	// Propogate all pseudo nodes
+	if n.Pseudo {
+		return RenderableNodes{n.ID: n}
+	}
+
+	// Otherwise, if some some reason the container doesn't have a pod_id (maybe
+	// slightly out of sync reports, or its not in a pod), just drop it
+	id, ok := n.Node.Metadata["docker_label_io.kubernetes.pod.name"]
+	if !ok {
+		return RenderableNodes{}
+	}
+
+	// Add container-<id> key to NMD, which will later be counted to produce the
+	// minor label
+	result := NewRenderableNodeWith(id, "", "", id, n)
+	result.Node.Counters[containersKey] = 1
+	// Due to a bug in kubernetes, addon pods on the master node are not returned
+	// from the API. This is a workaround until
+	// https://github.com/kubernetes/kubernetes/issues/14738 is fixed.
+	if s := strings.SplitN(id, "/", 2); len(s) == 2 {
+		result.LabelMajor = s[1]
+		result.Node.Metadata[kubernetes.Namespace] = s[0]
+		result.Node.Metadata[kubernetes.PodName] = s[1]
+	}
+	return RenderableNodes{id: result}
+}
+
 // MapCountContainers maps 1:1 container image nodes, counting
 // the number of containers grouped together and putting
 // that info in the minor label.
@@ -480,6 +588,22 @@ func MapCountContainers(n RenderableNode, _ report.Networks) RenderableNodes {
 	return RenderableNodes{n.ID: n}
 }
 
+// MapCountPods maps 1:1 service nodes, counting the number of pods grouped
+// together and putting that info in the minor label.
+func MapCountPods(n RenderableNode, _ report.Networks) RenderableNodes {
+	if n.Pseudo {
+		return RenderableNodes{n.ID: n}
+	}
+
+	pods := n.Node.Counters[podsKey]
+	if pods == 1 {
+		n.LabelMinor = "1 pod"
+	} else {
+		n.LabelMinor = fmt.Sprintf("%d pods", pods)
+	}
+	return RenderableNodes{n.ID: n}
+}
+
 // MapAddress2Host maps address RenderableNodes to host RenderableNodes.
 //
 // Otherthan pseudo nodes, we can assume all nodes have a HostID

render/mapping_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/weaveworks/scope/probe/docker"
 	"github.com/weaveworks/scope/probe/endpoint"
+	"github.com/weaveworks/scope/probe/kubernetes"
 	"github.com/weaveworks/scope/probe/process"
 	"github.com/weaveworks/scope/render"
 	"github.com/weaveworks/scope/report"
@@ -72,6 +73,24 @@ func TestMapHostIdentity(t *testing.T) {
 	}
 }
 
+func TestMapPodIdentity(t *testing.T) {
+	for _, input := range []testcase{
+		{nrn(report.MakeNode()), false},
+		{nrn(report.MakeNodeWith(map[string]string{kubernetes.PodID: "ping/pong", kubernetes.PodName: "pong"})), true},
+	} {
+		testMap(t, render.MapPodIdentity, input)
+	}
+}
+
+func TestMapServiceIdentity(t *testing.T) {
+	for _, input := range []testcase{
+		{nrn(report.MakeNode()), false},
+		{nrn(report.MakeNodeWith(map[string]string{kubernetes.ServiceID: "ping/pong", kubernetes.ServiceName: "pong"})), true},
+	} {
+		testMap(t, render.MapServiceIdentity, input)
+	}
+}
+
 type testcase struct {
 	md render.RenderableNode
 	ok bool

render/render.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/weaveworks/scope/probe/docker"
+	"github.com/weaveworks/scope/probe/kubernetes"
 	"github.com/weaveworks/scope/report"
 )
 
@@ -291,6 +292,12 @@ func FilterSystem(r Renderer) Renderer {
 			if node.Metadata[docker.LabelPrefix+"works.weave.role"] == "system" {
 				return false
 			}
+			if node.Metadata[kubernetes.Namespace] == "kube-system" {
+				return false
+			}
+			if strings.HasPrefix(node.Metadata[docker.LabelPrefix+"io.kubernetes.pod.name"], "kube-system/") {
+				return false
+			}
 			return true
 		},
 	}

render/selectors.go

@@ -91,4 +91,14 @@ var (
 	SelectHost = TopologySelector(func(r report.Report) RenderableNodes {
 		return MakeRenderableNodes(r.Host)
 	})
+
+	// SelectPod selects the pod topology.
+	SelectPod = TopologySelector(func(r report.Report) RenderableNodes {
+		return MakeRenderableNodes(r.Pod)
+	})
+
+	// SelectService selects the service topology.
+	SelectService = TopologySelector(func(r report.Report) RenderableNodes {
+		return MakeRenderableNodes(r.Service)
+	})
 )

render/topologies.go

@@ -190,3 +190,35 @@ var HostRenderer = MakeReduce(
 		Renderer: SelectHost,
 	},
 )
+
+// PodRenderer is a Renderer which produces a renderable kubernetes
+// graph by merging the container graph and the pods topology.
+var PodRenderer = Map{
+	MapFunc: MapCountContainers,
+	Renderer: MakeReduce(
+		Map{
+			MapFunc:  MapPodIdentity,
+			Renderer: SelectPod,
+		},
+		Map{
+			MapFunc:  MapContainer2Pod,
+			Renderer: ContainerRenderer,
+		},
+	),
+}
+
+// PodsServiceRenderer is a Renderer which produces a renderable kubernetes services
+// graph by merging the pods graph and the services topology.
+var PodServiceRenderer = Map{
+	MapFunc: MapCountPods,
+	Renderer: MakeReduce(
+		Map{
+			MapFunc:  MapPod2Service,
+			Renderer: PodRenderer,
+		},
+		Map{
+			MapFunc:  MapServiceIdentity,
+			Renderer: SelectService,
+		},
+	),
+}

render/topologies_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/weaveworks/scope/probe/docker"
+	"github.com/weaveworks/scope/probe/kubernetes"
 	"github.com/weaveworks/scope/render"
 	"github.com/weaveworks/scope/render/expected"
 	"github.com/weaveworks/scope/test"
@@ -62,3 +63,36 @@ func TestHostRenderer(t *testing.T) {
 		t.Error(test.Diff(want, have))
 	}
 }
+
+func TestPodRenderer(t *testing.T) {
+	have := render.PodRenderer.Render(test.Report).Prune()
+	want := expected.RenderedPods
+	if !reflect.DeepEqual(want, have) {
+		t.Error(test.Diff(want, have))
+	}
+}
+
+func TestPodFilterRenderer(t *testing.T) {
+	// tag on containers or pod namespace in the topology and ensure
+	// it is filtered out correctly.
+	input := test.Report.Copy()
+	input.Pod.Nodes[test.ClientPodNodeID].Metadata[kubernetes.PodID] = "kube-system/foo"
+	input.Pod.Nodes[test.ClientPodNodeID].Metadata[kubernetes.Namespace] = "kube-system"
+	input.Pod.Nodes[test.ClientPodNodeID].Metadata[kubernetes.PodName] = "foo"
+	input.Container.Nodes[test.ClientContainerNodeID].Metadata[docker.LabelPrefix+"io.kubernetes.pod.name"] = "kube-system/foo"
+	have := render.FilterSystem(render.PodRenderer).Render(input).Prune()
+	want := expected.RenderedPods.Copy()
+	delete(want, test.ClientPodID)
+	delete(want, test.ClientContainerID)
+	if !reflect.DeepEqual(want, have) {
+		t.Error(test.Diff(want, have))
+	}
+}
+
+func TestPodServiceRenderer(t *testing.T) {
+	have := render.PodServiceRenderer.Render(test.Report).Prune()
+	want := expected.RenderedPodServices
+	if !reflect.DeepEqual(want, have) {
+		t.Error(test.Diff(want, have))
+	}
+}