Merge pull request #776 from weaveworks/tracer

Tracer!
2026-07-15 19:41:02 +00:00 · 2015-12-17 13:55:10 +00:00
parent 4ea854519b 6dc65e2481
commit 2faa58aa41
31 changed files with 2004 additions and 11 deletions
--- a/experimental/example/Makefile
+++ b/experimental/example/Makefile
@@ -1,7 +1,7 @@
 CC=gcc
 CFLAGS=-g -lpthread

-all: qotd.marker app.marker client.marker searchapp.marker shout.marker frontend.marker
+all: qotd.marker app.marker client.marker searchapp.marker shout.marker frontend.marker echo.marker

 qotd/qotd: qotd/qotd.o
 	gcc -o $@ $< $(CFLAGS)
@@ -21,6 +21,7 @@ client.marker: client/*
 searchapp.marker: searchapp/* searchapp/searchapp
 shout.marker: shout/* shout/shout
 frontend.marker: frontend/*
+echo.marker: echo/*
 %.marker:
 	docker build -t tomwilkie/$(<D) $(<D)/
 	touch $@
--- a/experimental/example/app/app.py
+++ b/experimental/example/app/app.py
@@ -37,6 +37,10 @@ def do_search():
  r = sessions.session.get(random.choice(searchapps))
  return r.text

+def do_echo(text):
+  r = requests.get("http://echo/", data=text)
+  return r.text
+
 def ignore_error(f):
  try:
    return str(f())
@@ -44,14 +48,24 @@ def ignore_error(f):
    logging.error("Error executing function", exc_info=sys.exc_info())
  return "Error"

-@app.route('/')
+# this root is for the tracing demo
+@app.route('/hello')
 def hello():
+  qotd_msg = do_qotd()
+  qotd_msg = do_echo(qotd_msg)
+  return qotd_msg
+
+# this is for normal demos
+@app.route('/')
+def root():
  counter_future = pool.submit(do_redis)
  search_future = pool.submit(do_search)
  qotd_future = pool.submit(do_qotd)
+  echo_future = pool.submit(lambda: do_echo("foo"))
  result = 'Hello World! I have been seen %s times.' % ignore_error(counter_future.result)
  result += ignore_error(search_future.result)
  result += ignore_error(qotd_future.result)
+  result += ignore_error(echo_future.result)
  return result

 if __name__ == "__main__":
--- a/experimental/example/echo/Dockerfile
+++ b/experimental/example/echo/Dockerfile
@@ -0,0 +1,8 @@
+FROM python:2.7
+MAINTAINER Weaveworks Inc <help@weave.works>
+WORKDIR /home/weave
+ADD requirements.txt /home/weave/
+RUN pip install -r /home/weave/requirements.txt
+ADD echo.py /home/weave/
+EXPOSE 5000
+ENTRYPOINT ["python", "/home/weave/echo.py"]
--- a/experimental/example/echo/echo.py
+++ b/experimental/example/echo/echo.py
@@ -0,0 +1,27 @@
+import os
+import socket
+import sys
+import random
+import time
+import threading
+import logging
+
+
+from flask import Flask
+from flask import request
+from werkzeug.serving import WSGIRequestHandler
+
+app = Flask(__name__)
+
+@app.route('/')
+def hello():
+  if random.random() > 0.6:
+    time.sleep(2)
+  else:
+    time.sleep(0.5)
+  return request.data
+
+if __name__ == "__main__":
+  logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:%(lineno)d - %(message)s', level=logging.INFO)
+  WSGIRequestHandler.protocol_version = "HTTP/1.0"
+  app.run(host="0.0.0.0", port=80, debug=True)
--- a/experimental/example/echo/requirements.txt
+++ b/experimental/example/echo/requirements.txt
@@ -0,0 +1,2 @@
+flask
+
--- a/experimental/example/qotd/qotd.c
+++ b/experimental/example/qotd/qotd.c
@@ -8,17 +8,13 @@
 /* this function is run by the  thread */
 void *thread_func(void *sock) {
  struct sockaddr_in dest;
-  char buffer[1024];
+  char in_buffer[1024];
+  char out_buffer[1024];
  int sockfd = (int)sock;
  int clientfd;

  printf("I'm thread %d\n", syscall(SYS_gettid));

-  if (read(sockfd, buffer, sizeof(buffer)) < 0) {
-    perror("ERROR reading from socket");
-    return;
-  }
-
  clientfd = socket(AF_INET, SOCK_STREAM, 0);
  if (clientfd < 0) {
    perror("ERROR opening socket");
@@ -37,10 +33,11 @@ void *thread_func(void *sock) {
    return;
  }

-  int readbytes = recv(clientfd, buffer, sizeof(buffer), 0);
+  int readbytes = recv(clientfd, in_buffer, sizeof(in_buffer), 0);
  close(clientfd);

-  if (write(sockfd, buffer, read) < 0) {
+  int writtenbytes = snprintf(out_buffer, sizeof(out_buffer), "{\"qotd\": %s}\n", in_buffer);
+  if (write(sockfd, out_buffer, writtenbytes) < 0) {
    perror("ERROR writing to socket");
    return;
  }
--- a/experimental/example/run.sh
+++ b/experimental/example/run.sh
@@ -23,6 +23,8 @@ start_container 1 elasticsearch elasticsearch
 start_container 2 tomwilkie/searchapp searchapp
 start_container 1 redis redis
 start_container 1 tomwilkie/qotd qotd
+start_container 1 tomwilkie/echo echo
 start_container 2 tomwilkie/app app
 start_container 2 tomwilkie/frontend frontend --add-host=dns.weave.local:$(weave docker-bridge-ip)
 start_container 1 tomwilkie/client client
+
--- a/experimental/tracer/.gitignore
+++ b/experimental/tracer/.gitignore
@@ -0,0 +1,3 @@
+main/main
+tracer.tar
+main/static.go
--- a/experimental/tracer/Makefile
+++ b/experimental/tracer/Makefile
@@ -0,0 +1,14 @@
+tracer.tar: main/main main/Dockerfile
+	docker build -t weaveworks/tracer main/
+	docker save weaveworks/tracer:latest >$@
+
+main/main: main/*.go main/static.go ptrace/*.go
+	go get -tags netgo ./$(@D)
+	go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D)
+
+main/static.go: ui/*
+	esc -o main/static.go -prefix ui ui
+
+clean:
+	go clean ./..
+	rm -f main/static.go tracer.tar main/main
--- a/experimental/tracer/README.md
+++ b/experimental/tracer/README.md
@@ -0,0 +1,11 @@
+Run tracer:
+- make
+- ./tracer.sh start
+
+TODO:
+- <s>need to stich traces together</s>
+- deal with persistant connections
+- make it work for goroutines
+- test with jvm based app
+- find way to get local ip address for an incoming connection
+- make the container/process trace start/stop more reliable
--- a/experimental/tracer/main/Dockerfile
+++ b/experimental/tracer/main/Dockerfile
@@ -0,0 +1,6 @@
+FROM gliderlabs/alpine
+MAINTAINER Weaveworks Inc <help@weave.works>
+WORKDIR /home/weave
+COPY ./main /home/weave/
+EXPOSE 4050
+ENTRYPOINT ["/home/weave/main"]
--- a/experimental/tracer/main/http.go
+++ b/experimental/tracer/main/http.go
@@ -0,0 +1,150 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"mime"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+
+	"github.com/weaveworks/scope/probe/docker"
+	"github.com/weaveworks/scope/probe/process"
+)
+
+func respondWith(w http.ResponseWriter, code int, response interface{}) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Add("Cache-Control", "no-cache")
+	w.WriteHeader(code)
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		log.Printf("Error handling http request: %v", err.Error())
+	}
+}
+
+func (t *tracer) pidsForContainer(id string) ([]int, error) {
+	var container docker.Container
+	t.docker.WalkContainers(func(c docker.Container) {
+		if c.ID() == id {
+			container = c
+		}
+	})
+
+	if container == nil {
+		return []int{}, fmt.Errorf("Not Found")
+	}
+
+	pidTree, err := process.NewTree(process.NewWalker("/proc"))
+	if err != nil {
+		return []int{}, err
+	}
+
+	return pidTree.GetChildren(container.PID())
+}
+
+// Container is the type exported by the HTTP API.
+type Container struct {
+	ID   string
+	Name string
+	PIDs []int
+}
+
+func (t *tracer) http(port int) {
+	router := mux.NewRouter()
+
+	router.Methods("GET").Path("/container").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		pidTree, err := process.NewTree(process.NewWalker("/proc"))
+		if err != nil {
+			respondWith(w, http.StatusBadRequest, err.Error())
+			return
+		}
+
+		containers := []Container{}
+		t.docker.WalkContainers(func(container docker.Container) {
+			children, _ := pidTree.GetChildren(container.PID())
+			out := Container{
+				Name: strings.TrimPrefix(container.Container().Name, "/"),
+				ID:   container.ID(),
+				PIDs: children,
+			}
+			containers = append(containers, out)
+		})
+
+		respondWith(w, http.StatusOK, containers)
+	})
+
+	router.Methods("POST").Path("/container/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		id := mux.Vars(r)["id"]
+		children, err := t.pidsForContainer(id)
+		if err != nil {
+			respondWith(w, http.StatusBadRequest, err.Error())
+			return
+		}
+
+		for _, pid := range children {
+			t.ptrace.TraceProcess(pid)
+		}
+		w.WriteHeader(204)
+	})
+
+	router.Methods("DELETE").Path("/container/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		id := mux.Vars(r)["id"]
+		children, err := t.pidsForContainer(id)
+		if err != nil {
+			respondWith(w, http.StatusBadRequest, err.Error())
+			return
+		}
+
+		for _, pid := range children {
+			t.ptrace.StopTracing(pid)
+		}
+		w.WriteHeader(204)
+	})
+
+	router.Methods("GET").Path("/pid").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		respondWith(w, http.StatusOK, t.ptrace.AttachedPIDs())
+	})
+
+	router.Methods("POST").Path("/pid/{pid:\\d+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		pid, err := strconv.Atoi(mux.Vars(r)["pid"])
+		if err != nil {
+			respondWith(w, http.StatusBadRequest, err.Error())
+			return
+		}
+
+		t.ptrace.TraceProcess(pid)
+		w.WriteHeader(204)
+	})
+
+	router.Methods("DELETE").Path("/pid/{pid:\\d+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		pid, err := strconv.Atoi(mux.Vars(r)["pid"])
+		if err != nil {
+			respondWith(w, http.StatusBadRequest, err.Error())
+			return
+		}
+
+		t.ptrace.StopTracing(pid)
+		w.WriteHeader(204)
+	})
+
+	router.Methods("GET").Path("/traces").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		respondWith(w, http.StatusOK, t.store.Traces())
+	})
+
+	mime.AddExtensionType(".svg", "image/svg+xml")
+
+	router.Methods("GET").PathPrefix("/").Handler(http.FileServer(FS(false))) // everything else is static
+
+	log.Printf("Launching HTTP API on port %d", port)
+	srv := &http.Server{
+		Addr:    fmt.Sprintf(":%d", port),
+		Handler: router,
+	}
+
+	if err := srv.ListenAndServe(); err != nil {
+		log.Printf("Unable to create http listener: %v", err)
+	}
+}
--- a/experimental/tracer/main/main.go
+++ b/experimental/tracer/main/main.go
@@ -0,0 +1,70 @@
+package main
+
+import (
+	"log"
+	_ "net/http/pprof"
+	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/weaveworks/scope/experimental/tracer/ptrace"
+	"github.com/weaveworks/scope/probe/docker"
+)
+
+const (
+	procRoot     = "/proc"
+	pollInterval = 10 * time.Second
+)
+
+type tracer struct {
+	ptrace ptrace.PTracer
+	store  *store
+	docker docker.Registry
+}
+
+func (t *tracer) Stop() {
+	log.Printf("Shutting down...")
+	t.ptrace.Stop()
+	t.docker.Stop()
+	log.Printf("Done.")
+}
+
+func main() {
+	dockerRegistry, err := docker.NewRegistry(pollInterval, nil)
+	if err != nil {
+		log.Fatalf("Could start docker watcher: %v", err)
+	}
+
+	store := newStore()
+	tracer := tracer{
+		store:  store,
+		ptrace: ptrace.NewPTracer(store),
+		docker: dockerRegistry,
+	}
+	defer tracer.Stop()
+
+	go tracer.http(6060)
+	<-handleSignals()
+}
+
+func handleSignals() chan struct{} {
+	quit := make(chan struct{}, 10)
+	go func() {
+		sigs := make(chan os.Signal, 1)
+		signal.Notify(sigs, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGTERM)
+		buf := make([]byte, 1<<20)
+		for {
+			sig := <-sigs
+			switch sig {
+			case syscall.SIGINT, syscall.SIGTERM:
+				quit <- struct{}{}
+			case syscall.SIGQUIT:
+				stacklen := runtime.Stack(buf, true)
+				log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
+			}
+		}
+	}()
+	return quit
+}
--- a/experimental/tracer/main/store.go
+++ b/experimental/tracer/main/store.go
@@ -0,0 +1,172 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"sync"
+
+	"github.com/msackman/skiplist"
+
+	"github.com/weaveworks/scope/experimental/tracer/ptrace"
+)
+
+const epsilon = int64(5) * 1000 // milliseconds
+
+// Traces are indexed by from addr, from port, and start time.
+type key struct {
+	fromAddr  uint32
+	fromPort  uint16
+	startTime int64
+}
+
+func (k key) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf("\"%x.%x.%x\"", k.fromAddr, k.fromPort, k.startTime)), nil
+}
+
+type trace struct {
+	PID           int
+	Key           key
+	ServerDetails *ptrace.ConnectionDetails
+	ClientDetails *ptrace.ConnectionDetails
+	Children      []*trace
+	Level         int
+}
+
+type byKey []*trace
+
+func (a byKey) Len() int           { return len(a) }
+func (a byKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byKey) Less(i, j int) bool { return a[i].Key.startTime < a[j].Key.startTime }
+
+type store struct {
+	sync.RWMutex
+	traces *skiplist.SkipList
+}
+
+func newKey(fd *ptrace.Fd) key {
+	var fromAddr uint32
+	for _, b := range fd.FromAddr.To4() {
+		fromAddr <<= 8
+		fromAddr |= uint32(b)
+	}
+
+	return key{fromAddr, fd.FromPort, fd.Start}
+}
+
+func (k key) LessThan(other skiplist.Comparable) bool {
+	r := other.(key)
+
+	if k.fromAddr != r.fromAddr {
+		return k.fromAddr > r.fromAddr
+	}
+
+	if k.fromPort != r.fromPort {
+		return k.fromPort < r.fromPort
+	}
+
+	if k.Equal(other) {
+		return false
+	}
+
+	return k.startTime < r.startTime
+}
+
+func (k key) Equal(other skiplist.Comparable) bool {
+	r := other.(key)
+	if k.fromAddr != r.fromAddr || k.fromPort != r.fromPort {
+		return false
+	}
+
+	diff := k.startTime - r.startTime
+	return -epsilon < diff && diff < epsilon
+}
+
+func newStore() *store {
+	return &store{traces: skiplist.New(rand.New(rand.NewSource(0)))}
+}
+
+func (t *trace) addChild(child *trace) {
+	// find the child we're supposed to be replacing
+	for i, candidate := range t.Children {
+		if !candidate.Key.Equal(skiplist.Comparable(child.Key)) {
+			continue
+		}
+
+		// Fix up some fields
+		child.ClientDetails = candidate.ClientDetails
+		IncrementLevel(child, t.Level+1)
+
+		// Overwrite old record
+		t.Children[i] = child
+		return
+	}
+}
+
+func (s *store) RecordConnection(pid int, connection *ptrace.Fd) {
+	s.Lock()
+	defer s.Unlock()
+
+	newTrace := &trace{
+		PID:           pid,
+		Key:           newKey(connection),
+		ServerDetails: &connection.ConnectionDetails,
+	}
+	for _, child := range connection.Children {
+		newTrace.Children = append(newTrace.Children, &trace{
+			Level:         1,
+			Key:           newKey(child),
+			ClientDetails: &child.ConnectionDetails,
+		})
+	}
+
+	// First, see if this new conneciton is a child of an existing connection.
+	// This indicates we have a parent connection to attach to.
+	// If not, insert this connection.
+	if parentNode := s.traces.Get(newTrace.Key); parentNode != nil {
+		parentTrace := parentNode.Value.(*trace)
+		parentTrace.addChild(newTrace)
+		parentNode.Remove()
+	} else {
+		s.traces.Insert(newTrace.Key, newTrace)
+	}
+
+	// Next, see if we already know about the child connections
+	// If not, insert each of our children.
+	for _, child := range newTrace.Children {
+		if childNode := s.traces.Get(child.Key); childNode != nil {
+			childTrace := childNode.Value.(*trace)
+			newTrace.addChild(childTrace)
+			childNode.Remove()
+		} else {
+			s.traces.Insert(child.Key, newTrace)
+		}
+	}
+}
+
+// IncrementLevel ...
+func IncrementLevel(trace *trace, increment int) {
+	trace.Level += increment
+	for _, child := range trace.Children {
+		IncrementLevel(child, increment)
+	}
+}
+
+func (s *store) Traces() []*trace {
+	s.RLock()
+	defer s.RUnlock()
+
+	traces := []*trace{}
+	var cur = s.traces.First()
+	for cur != nil {
+		key := cur.Key.(key)
+		trace := cur.Value.(*trace)
+		if trace.Key == key {
+			traces = append(traces, trace)
+		}
+		cur = cur.Next()
+	}
+
+	sort.Sort(byKey(traces))
+	return traces
+}
--- a/experimental/tracer/ptrace/fd.go
+++ b/experimental/tracer/ptrace/fd.go
@@ -0,0 +1,198 @@
+package ptrace
+
+import (
+	"bufio"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"os"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+const (
+	listening = iota
+	incoming
+	outgoing
+)
+
+const (
+	socketPattern = `^socket:\[(\d+)\]$`
+	tcpPattern    = `^\s*(?P<fd>\d+): (?P<localaddr>[A-F0-9]{8}):(?P<localport>[A-F0-9]{4}) ` +
+		`(?P<remoteaddr>[A-F0-9]{8}):(?P<remoteport>[A-F0-9]{4}) (?:[A-F0-9]{2}) (?:[A-F0-9]{8}):(?:[A-F0-9]{8}) ` +
+		`(?:[A-F0-9]{2}):(?:[A-F0-9]{8}) (?:[A-F0-9]{8}) \s+(?:\d+) \s+(?:\d+) (?P<inode>\d+)`
+)
+
+var (
+	socketRegex = regexp.MustCompile(socketPattern)
+	tcpRegexp   = regexp.MustCompile(tcpPattern)
+)
+
+// ConnectionDetails ...
+type ConnectionDetails struct {
+	direction int
+
+	Start    int64
+	Stop     int64
+	sent     int64
+	received int64
+
+	FromAddr net.IP
+	FromPort uint16
+	ToAddr   net.IP
+	ToPort   uint16
+}
+
+// Fd represents a connect and subsequent connections caused by it.
+type Fd struct {
+	fd     int
+	closed bool
+
+	ConnectionDetails
+
+	// Fds are connections, and can have a causal-link to other Fds
+	Children []*Fd
+}
+
+func getLocalAddr(pid, fd int) (addr net.IP, port uint16, err error) {
+	var (
+		socket    string
+		match     []string
+		inode     int
+		tcpFile   *os.File
+		scanner   *bufio.Scanner
+		candidate int
+		port64    int64
+	)
+
+	socket, err = os.Readlink(fmt.Sprintf("/proc/%d/fd/%d", pid, fd))
+	if err != nil {
+		return
+	}
+
+	match = socketRegex.FindStringSubmatch(socket)
+	if match == nil {
+		err = fmt.Errorf("Fd %d not a socket", fd)
+		return
+	}
+
+	inode, err = strconv.Atoi(match[1])
+	if err != nil {
+		return
+	}
+
+	tcpFile, err = os.Open(fmt.Sprintf("/proc/%d/net/tcp", pid))
+	if err != nil {
+		return
+	}
+	defer tcpFile.Close()
+
+	scanner = bufio.NewScanner(tcpFile)
+	for scanner.Scan() {
+		match = tcpRegexp.FindStringSubmatch(scanner.Text())
+		if match == nil {
+			continue
+		}
+
+		candidate, err = strconv.Atoi(match[6])
+		if err != nil {
+			return
+		}
+		if candidate != inode {
+			continue
+		}
+
+		addr = make([]byte, 4)
+		if _, err = hex.Decode(addr, []byte(match[2])); err != nil {
+			return
+		}
+		addr[0], addr[1], addr[2], addr[3] = addr[3], addr[2], addr[1], addr[0]
+
+		// use a 32 bit int for target, at the result is a uint16
+		port64, err = strconv.ParseInt(match[3], 16, 32)
+		if err != nil {
+			return
+		}
+		port = uint16(port64)
+
+		return
+	}
+
+	if err = scanner.Err(); err != nil {
+		return
+	}
+
+	err = fmt.Errorf("Fd %d not found for proc %d", fd, pid)
+	return
+}
+
+// in milliseconds
+func now() int64 {
+	return time.Now().UnixNano() / 1000000
+}
+
+// We want to get the listening address from /proc
+func newListeningFd(pid, fd int) (*Fd, error) {
+	localAddr, localPort, err := getLocalAddr(pid, fd)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Fd{
+		fd: fd,
+
+		ConnectionDetails: ConnectionDetails{
+			direction: listening,
+			Start:     now(),
+			ToAddr:    localAddr,
+			ToPort:    uint16(localPort),
+		},
+	}, nil
+}
+
+// We intercepted a connect syscall
+func newConnectionFd(pid, fd int, remoteAddr net.IP, remotePort uint16) (*Fd, error) {
+	localAddr, localPort, err := getLocalAddr(pid, fd)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Fd{
+		fd: fd,
+
+		ConnectionDetails: ConnectionDetails{
+			direction: outgoing,
+			Start:     now(),
+			FromAddr:  localAddr,
+			FromPort:  uint16(localPort),
+			ToAddr:    remoteAddr,
+			ToPort:    remotePort,
+		},
+	}, nil
+}
+
+// We got a new connection on a listening socket
+func (fd *Fd) newConnection(addr net.IP, port uint16, newFd int) (*Fd, error) {
+	if fd.direction != listening {
+		return nil, fmt.Errorf("New connection on non-listening fd!")
+	}
+
+	return &Fd{
+		fd: newFd,
+
+		ConnectionDetails: ConnectionDetails{
+			direction: incoming,
+			Start:     now(),
+			ToAddr:    fd.ToAddr,
+			ToPort:    fd.ToPort,
+			FromAddr:  addr,
+			FromPort:  port,
+		},
+	}, nil
+}
+
+func (fd *Fd) close() {
+	fd.closed = true
+	fd.Stop = now()
+}
--- a/experimental/tracer/ptrace/process.go
+++ b/experimental/tracer/ptrace/process.go
@@ -0,0 +1,99 @@
+package ptrace
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strconv"
+	"sync"
+)
+
+type process struct {
+	sync.Mutex
+	pid int
+
+	detaching bool
+	detached  chan struct{}
+
+	tracer  *PTracer
+	threads map[int]*thread
+	fds     map[int]*Fd
+}
+
+func newProcess(pid int, tracer *PTracer) *process {
+	return &process{
+		pid:      pid,
+		tracer:   tracer,
+		threads:  make(map[int]*thread),
+		fds:      make(map[int]*Fd),
+		detached: make(chan struct{}),
+	}
+}
+
+func (p *process) trace() {
+	go p.loop()
+}
+
+// This doesn't actually guarantees we follow all the threads.  Oops.
+func (p *process) loop() {
+	var (
+		attached int
+	)
+	log.Printf("Tracing process %d", p.pid)
+
+	for {
+		ps, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/task", p.pid))
+		if err != nil {
+			log.Printf("ReadDir failed, pid=%d, err=%v", p.pid, err)
+			return
+		}
+
+		attached = 0
+		for _, file := range ps {
+			pid, err := strconv.Atoi(file.Name())
+			if err != nil {
+				log.Printf("'%s' is not a pid: %v", file.Name(), err)
+				attached++
+				continue
+			}
+
+			p.Lock()
+			t, ok := p.threads[pid]
+			if !ok {
+				t = p.tracer.traceThread(pid, p)
+				p.threads[pid] = t
+			}
+			p.Unlock()
+
+			if !t.attached {
+				continue
+			}
+
+			attached++
+		}
+
+		// When we successfully attach to all threads
+		// we can be sure to catch new clones, so we
+		// can quit.
+		if attached == len(ps) {
+			break
+		}
+	}
+
+	log.Printf("Successfully attached to %d threads", attached)
+}
+
+func (p *process) newThread(thread *thread) {
+	p.Lock()
+	defer p.Unlock()
+	p.threads[thread.tid] = thread
+}
+
+func (p *process) newFd(fd *Fd) error {
+	_, ok := p.fds[fd.fd]
+	if ok {
+		return fmt.Errorf("New fd %d, alread exists!", fd.fd)
+	}
+	p.fds[fd.fd] = fd
+	return nil
+}
--- a/experimental/tracer/ptrace/ptracer.go
+++ b/experimental/tracer/ptrace/ptracer.go
@@ -0,0 +1,336 @@
+package ptrace
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+const (
+	ptraceOptions         = syscall.PTRACE_O_TRACESYSGOOD | syscall.PTRACE_O_TRACECLONE
+	ptraceTracesysgoodBit = 0x80
+)
+
+// Store ...
+type Store interface {
+	RecordConnection(int, *Fd)
+}
+
+// PTracer ptrace processed and threads
+type PTracer struct {
+
+	// All ptrace calls must come from the
+	// same thread.  So we wait on a separate
+	// thread.
+	ops           chan func()
+	stopped       chan stopped
+	quit          chan struct{}
+	childAttached chan struct{} //used to signal the wait loop
+
+	threads   map[int]*thread
+	processes map[int]*process
+	store     Store
+}
+
+type stopped struct {
+	pid    int
+	status syscall.WaitStatus
+}
+
+// NewPTracer creates a new ptracer.
+func NewPTracer(store Store) PTracer {
+	t := PTracer{
+		ops:           make(chan func()),
+		stopped:       make(chan stopped),
+		quit:          make(chan struct{}),
+		childAttached: make(chan struct{}),
+
+		threads:   make(map[int]*thread),
+		processes: make(map[int]*process),
+		store:     store,
+	}
+	go t.waitLoop()
+	go t.loop()
+	return t
+}
+
+// Stop stop stop
+func (t *PTracer) Stop() {
+	out := make(chan []int)
+	t.ops <- func() {
+		pids := []int{}
+		for pid := range t.processes {
+			pids = append(pids, pid)
+		}
+		out <- pids
+	}
+	for _, pid := range <-out {
+		t.StopTracing(pid)
+	}
+	t.quit <- struct{}{}
+}
+
+// TraceProcess starts tracing the given pid
+func (t *PTracer) TraceProcess(pid int) *process {
+	result := make(chan *process)
+	t.ops <- func() {
+		process := newProcess(pid, t)
+		t.processes[pid] = process
+		process.trace()
+		result <- process
+	}
+	return <-result
+}
+
+// StopTracing stops tracing all threads for the given pid
+func (t *PTracer) StopTracing(pid int) error {
+	log.Printf("Detaching from %d", pid)
+
+	errors := make(chan error)
+	processes := make(chan *process)
+
+	t.ops <- func() {
+		// send sigstop to all threads
+		process, ok := t.processes[pid]
+		if !ok {
+			errors <- fmt.Errorf("PID %d not found", pid)
+			return
+		}
+
+		// This flag tells the thread to detach when it next stops
+		process.detaching = true
+
+		// Now send sigstop to all threads.
+		for _, thread := range process.threads {
+			log.Printf("sending SIGSTOP to %d", thread.tid)
+			if err := syscall.Tgkill(pid, thread.tid, syscall.SIGSTOP); err != nil {
+				errors <- err
+				return
+			}
+		}
+
+		processes <- process
+	}
+
+	select {
+	case err := <-errors:
+		return err
+	case process := <-processes:
+		<-process.detached
+		return nil
+	}
+}
+
+// AttachedPIDs list the currently attached processes.
+func (t *PTracer) AttachedPIDs() []int {
+	result := make(chan []int)
+	t.ops <- func() {
+		var pids []int
+		for pid := range t.processes {
+			pids = append(pids, pid)
+		}
+		result <- pids
+	}
+	return <-result
+}
+
+func (t *PTracer) traceThread(pid int, process *process) *thread {
+	result := make(chan *thread)
+	t.ops <- func() {
+		thread := newThread(pid, process, t)
+		t.threads[pid] = thread
+
+		err := syscall.PtraceAttach(pid)
+		if err != nil {
+			log.Printf("Attach %d failed: %v", pid, err)
+			return
+		}
+
+		result <- thread
+
+		select {
+		case t.childAttached <- struct{}{}:
+		default:
+		}
+	}
+	return <-result
+}
+
+func (t *PTracer) waitLoop() {
+	var (
+		status syscall.WaitStatus
+		pid    int
+		err    error
+	)
+
+	for {
+		//log.Printf("Waiting...")
+		pid, err = syscall.Wait4(-1, &status, syscall.WALL, nil)
+		if err != nil && err.(syscall.Errno) == syscall.ECHILD {
+			//log.Printf( "No children to wait4")
+			<-t.childAttached
+			continue
+		}
+		if err != nil {
+			log.Printf(" Wait failed: %v %d", err, err.(syscall.Errno))
+			return
+		}
+		//log.Printf(" PID %d stopped with signal %#x", pid, status)
+		t.stopped <- stopped{pid, status}
+	}
+}
+
+func (t *PTracer) loop() {
+	runtime.LockOSThread()
+
+	for {
+		select {
+		case op := <-t.ops:
+			op()
+		case stopped := <-t.stopped:
+			t.handleStopped(stopped.pid, stopped.status)
+		case <-t.quit:
+			return
+		}
+	}
+}
+
+func (t *PTracer) handleStopped(pid int, status syscall.WaitStatus) {
+	signal := syscall.Signal(0)
+	target, err := t.thread(pid)
+	if err != nil {
+		log.Printf("thread failed: %v", err)
+		return
+	}
+
+	if !target.attached {
+		target.attached = true
+
+		err = syscall.PtraceSetOptions(pid, ptraceOptions)
+		if err != nil {
+			log.Printf("SetOptions failed, pid=%d, err=%v", pid, err)
+			return
+		}
+
+	} else if status.Stopped() && status.StopSignal() == syscall.SIGTRAP|ptraceTracesysgoodBit {
+		// pid entered Syscall-enter-stop or syscall-exit-stop
+		target.syscallStopped()
+
+	} else if status.Stopped() && status.StopSignal() == syscall.SIGTRAP {
+		// pid entered PTRACE_EVENT stop
+		switch status.TrapCause() {
+		case syscall.PTRACE_EVENT_CLONE:
+			err := target.handleClone(pid)
+			if err != nil {
+				log.Printf("clone failed: %v", err)
+				return
+			}
+		default:
+			log.Printf("Unknown PTRACE_EVENT %d for pid %d", status.TrapCause(), pid)
+		}
+	} else if status.Exited() || status.Signaled() {
+		// "tracer can safely assume pid will exit"
+		t.threadExited(target)
+		return
+	} else if status.Stopped() {
+		// tracee recieved a non-trace related signal
+		signal = status.StopSignal()
+
+		if signal == syscall.SIGSTOP && target.process.detaching {
+			t.detachThread(target)
+			return
+		}
+	} else {
+		// unknown stop - shouldn't happen!
+		log.Printf("Pid %d random stop with status %x", pid, status)
+	}
+
+	// Restart stopped caller in syscall trap mode.
+	// log.Printf("Restarting pid %d with signal %d", pid, int(signal))
+	err = syscall.PtraceSyscall(pid, int(signal))
+	if err != nil {
+		log.Printf("PtraceSyscall failed, pid=%d, err=%v", pid, err)
+	}
+}
+
+func (t *PTracer) detachThread(thread *thread) {
+	syscall.PtraceDetach(thread.tid)
+	process := thread.process
+	delete(process.threads, thread.tid)
+	delete(t.threads, thread.tid)
+	if len(process.threads) == 0 {
+		delete(t.processes, process.pid)
+		close(process.detached)
+		log.Printf("Process %d detached", process.pid)
+	}
+}
+
+func pidForTid(tid int) (pid int, err error) {
+	var (
+		status  *os.File
+		scanner *bufio.Scanner
+		splits  []string
+	)
+
+	status, err = os.Open(fmt.Sprintf("/proc/%d/status", tid))
+	if err != nil {
+		return
+	}
+	defer status.Close()
+
+	scanner = bufio.NewScanner(status)
+	for scanner.Scan() {
+		splits = strings.Split(scanner.Text(), ":")
+		if splits[0] != "Tgid" {
+			continue
+		}
+
+		pid, err = strconv.Atoi(strings.TrimSpace(splits[1]))
+		return
+	}
+
+	if err = scanner.Err(); err != nil {
+		return
+	}
+
+	err = fmt.Errorf("Pid not found for proc %d", tid)
+	return
+}
+
+func (t *PTracer) thread(tid int) (*thread, error) {
+	// unfortunately we can't propage fd affinitiy as we
+	// can reliably determin who clone'd new threads, as
+	// the pids returned by the ptrace calls are in the process'
+	// namespace, not ours.
+
+	thread, ok := t.threads[tid]
+	if !ok {
+		pid, err := pidForTid(tid)
+		if err != nil {
+			return nil, err
+		}
+
+		proc, ok := t.processes[pid]
+		if !ok {
+			return nil, fmt.Errorf("Got new thread %d for unknown process", tid)
+		}
+
+		thread = newThread(tid, proc, t)
+		t.threads[tid] = thread
+		log.Printf("New thread reported, tid=%d, pid=%d", tid, pid)
+	}
+	return thread, nil
+}
+
+func (t *PTracer) threadExited(thread *thread) {
+	thread.handleExit()
+	delete(t.threads, thread.tid)
+	if thread.process != nil {
+		delete(thread.process.threads, thread.tid)
+	}
+}
--- a/experimental/tracer/ptrace/thread.go
+++ b/experimental/tracer/ptrace/thread.go
@@ -0,0 +1,291 @@
+package ptrace
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"syscall"
+	"unsafe"
+)
+
+// Syscall numbers
+const (
+	READ          = 0
+	WRITE         = 1
+	OPEN          = 2
+	CLOSE         = 3
+	STAT          = 4
+	MMAP          = 9
+	MPROTECT      = 10
+	MUNMAP        = 11
+	SELECT        = 23
+	MADVISE       = 28
+	SOCKET        = 41
+	CONNECT       = 42
+	ACCEPT        = 43
+	SENDTO        = 44
+	RECVFROM      = 45
+	SHUTDOWN      = 48
+	CLONE         = 56
+	GETTIMEOFDAY  = 96
+	GETID         = 186
+	FUTEX         = 202
+	SETROBUSTLIST = 273
+	ACCEPT4       = 288
+)
+
+// States for a given thread
+const (
+	NORMAL = iota
+	INSYSCALL
+)
+
+type thread struct {
+	tid      int
+	attached bool
+	process  *process // might be nil!
+	tracer   *PTracer
+
+	state      int
+	callRegs   syscall.PtraceRegs
+	resultRegs syscall.PtraceRegs
+
+	currentIncoming map[int]*Fd
+	currentOutgoing map[int]*Fd
+	closedOutgoing  []*Fd
+}
+
+func newThread(pid int, process *process, tracer *PTracer) *thread {
+	t := &thread{
+		tid:             pid,
+		process:         process,
+		tracer:          tracer,
+		currentIncoming: map[int]*Fd{},
+		currentOutgoing: map[int]*Fd{},
+	}
+	return t
+}
+
+// trace thread calls this
+func (t *thread) syscallStopped() {
+	var err error
+
+	if t.state == NORMAL {
+		if err = syscall.PtraceGetRegs(t.tid, &t.callRegs); err != nil {
+			t.logf("GetRegs failed, pid=%d, err=%v", t.tid, err)
+		}
+		t.state = INSYSCALL
+		return
+	}
+
+	t.state = NORMAL
+
+	if err = syscall.PtraceGetRegs(t.tid, &t.resultRegs); err != nil {
+		t.logf("GetRegs failed, pid=%d, err=%v", t.tid, err)
+		return
+	}
+
+	if t.process == nil {
+		t.logf("Got syscall, but don't know parent process!")
+		return
+	}
+
+	switch t.callRegs.Orig_rax {
+	case ACCEPT, ACCEPT4:
+		t.handleAccept(&t.callRegs, &t.resultRegs)
+
+	case CLOSE:
+		t.handleClose(&t.callRegs, &t.resultRegs)
+
+	case CONNECT:
+		t.handleConnect(&t.callRegs, &t.resultRegs)
+
+	case READ, WRITE, RECVFROM, SENDTO:
+		t.handleIO(&t.callRegs, &t.resultRegs)
+
+	// we can ignore these syscalls
+	case SETROBUSTLIST, GETID, MMAP, MPROTECT, MADVISE, SOCKET, CLONE, STAT, SELECT:
+		return
+
+	case OPEN, FUTEX, SHUTDOWN, GETTIMEOFDAY, MUNMAP:
+		return
+
+	default:
+		t.logf("syscall(%d)", t.callRegs.Orig_rax)
+	}
+}
+
+func ntohl(b uint16) uint16 {
+	return (b << 8) | ((b & 0xff00) >> 8)
+}
+
+func (t *thread) getSocketAddress(ptr uintptr) (addr net.IP, port uint16, err error) {
+	var (
+		buf  = make([]byte, syscall.SizeofSockaddrAny)
+		read int
+	)
+
+	if ptr == 0 {
+		err = fmt.Errorf("Null ptr")
+		return
+	}
+
+	read, err = syscall.PtracePeekData(t.tid, ptr, buf)
+	if read != syscall.SizeofSockaddrAny || err != nil {
+		return
+	}
+
+	var sockaddr4 = (*syscall.RawSockaddrInet4)(unsafe.Pointer(&buf[0]))
+	if sockaddr4.Family != syscall.AF_INET {
+		return
+	}
+
+	addr = net.IP(sockaddr4.Addr[0:])
+	port = ntohl(sockaddr4.Port)
+	return
+}
+
+func (t *thread) handleAccept(call, result *syscall.PtraceRegs) {
+	var (
+		err             error
+		ok              bool
+		listeningFdNum  int
+		connectionFdNum int
+		addrPtr         uintptr
+		addr            net.IP
+		port            uint16
+		listeningFd     *Fd
+		connection      *Fd
+	)
+
+	listeningFdNum = int(result.Rdi)
+	connectionFdNum = int(result.Rax)
+	addrPtr = uintptr(result.Rsi)
+	addr, port, err = t.getSocketAddress(addrPtr)
+	if err != nil {
+		t.logf("failed to read sockaddr: %v", err)
+		return
+	}
+
+	listeningFd, ok = t.process.fds[listeningFdNum]
+	if !ok {
+		listeningFd, err = newListeningFd(t.process.pid, listeningFdNum)
+		if err != nil {
+			t.logf("Failed to read listening port: %v", err)
+			return
+		}
+		t.process.fds[listeningFdNum] = listeningFd
+	}
+
+	connection, err = listeningFd.newConnection(addr, port, connectionFdNum)
+	if err != nil {
+		t.logf("Failed to create connection fd: %v", err)
+		return
+	}
+
+	t.process.newFd(connection)
+
+	t.logf("Accepted connection from %s:%d -> %s:%d on fd %d, new fd %d",
+		addr, port, connection.ToAddr, connection.ToPort, listeningFdNum, connectionFdNum)
+}
+
+func (t *thread) handleConnect(call, result *syscall.PtraceRegs) {
+	fd := int(result.Rdi)
+	ptr := result.Rsi
+	addr, port, err := t.getSocketAddress(uintptr(ptr))
+	if err != nil {
+		t.logf("failed to read sockaddr: %v", err)
+		return
+	}
+
+	connection, err := newConnectionFd(t.process.pid, fd, addr, port)
+	if err != nil {
+		t.logf("Failed to create connection fd: %v", err)
+		return
+	}
+
+	t.process.newFd(connection)
+
+	t.logf("Made connection from %s:%d -> %s:%d on fd %d",
+		connection.FromAddr, connection.FromPort,
+		connection.ToAddr, connection.ToPort, fd)
+}
+
+func (t *thread) handleClose(call, result *syscall.PtraceRegs) {
+	fdNum := int(call.Rdi)
+	fd, ok := t.process.fds[fdNum]
+	if !ok {
+		t.logf("Got close unknown fd %d", fdNum)
+		return
+	}
+
+	//t.logf("Closing fd %d", fdNum)
+	fd.close()
+
+	// if this connection was incoming, add it to 'the registry'
+	if fd.direction == incoming {
+		// collect all the outgoing connections this thread has made
+		// and treat them as caused by this incoming connections
+		for _, outgoing := range t.currentOutgoing {
+			//t.logf("Fd %d caused %d", fdNum, outgoing.fd)
+			fd.Children = append(fd.Children, outgoing)
+		}
+		t.currentOutgoing = map[int]*Fd{}
+
+		for _, outgoing := range t.closedOutgoing {
+			//t.logf("Fd %d caused %d", fdNum, outgoing.fd)
+			fd.Children = append(fd.Children, outgoing)
+		}
+		t.closedOutgoing = []*Fd{}
+
+		t.tracer.store.RecordConnection(t.process.pid, fd)
+	}
+
+	// now make sure we've remove it from everywhere
+	delete(t.process.fds, fdNum)
+	for _, thread := range t.process.threads {
+		delete(thread.currentIncoming, fdNum)
+
+		if _, ok := thread.currentOutgoing[fdNum]; ok {
+			thread.closedOutgoing = append(thread.closedOutgoing, fd)
+		}
+		delete(thread.currentOutgoing, fdNum)
+	}
+}
+
+func (t *thread) handleIO(call, result *syscall.PtraceRegs) {
+	fdNum := int(call.Rdi)
+	fd, ok := t.process.fds[fdNum]
+	if !ok {
+		//t.logf("IO on unknown fd %d", fdNum)
+		return
+	}
+
+	if fd.direction == incoming {
+		//t.logf("IO on incoming connection %d; setting affinity", fdNum)
+		t.currentIncoming[fdNum] = fd
+	} else {
+		//t.logf("IO on outgoing connection %d; setting affinity", fdNum)
+		t.currentOutgoing[fdNum] = fd
+	}
+}
+
+func (t *thread) handleClone(pid int) error {
+	// We can't use the pid in the process, as it may be in a namespace
+	newPid, err := syscall.PtraceGetEventMsg(pid)
+	if err != nil {
+		log.Printf("PtraceGetEventMsg failed: %v", err)
+		return err
+	}
+
+	t.logf("New thread clone'd, pid=%d", newPid)
+	return nil
+}
+
+func (t *thread) handleExit() {
+	t.logf("Exiting")
+}
+
+func (t *thread) logf(fmt string, args ...interface{}) {
+	log.Printf("[thread %d] "+fmt, append([]interface{}{t.tid}, args...)...)
+}
--- a/experimental/tracer/tracer
+++ b/experimental/tracer/tracer
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eu
+
+usage() {
+    echo "$0"
+}
+
+PORT=6060
+CONTAINER_NAME=weavetracer
+
+[ $# -gt 0 ] || usage
+COMMAND=$1
+shift 1
+
+case "$COMMAND" in
+
+    launch)
+        docker run --privileged --net=host --pid=host -ti -v /var/run/docker.sock:/var/run/docker.sock \
+            --name $CONTAINER_NAME weaveworks/tracer
+        ;;
+
+    stop)
+        docker stop $CONTAINER_NAME || true
+        docker rm $CONTAINER_NAME >/dev/null || true
+        ;;
+
+    attach)
+        PID=$1
+        if [ -z "${PID##*[!0-9]*}" ]; then
+            PID=$(pgrep $PID)
+        fi
+        curl -X POST http://localhost:$PORT/pid/$PID
+        ;;
+
+    detach)
+        PID=$1
+        if [ -z "${PID##*[!0-9]*}" ]; then
+            PID=$(pgrep $PID)
+        fi
+        curl -X DELETE http://localhost:$PORT/pid/$PID
+        ;;
+
+    traces)
+        curl http://localhost:$PORT/trace
+        ;;
+esac
--- a/experimental/tracer/ui/.gitignore
+++ b/experimental/tracer/ui/.gitignore
@@ -0,0 +1,4 @@
+container.json
+traces.json
+node_modules
+
--- a/experimental/tracer/ui/README.md
+++ b/experimental/tracer/ui/README.md
@@ -0,0 +1,10 @@
+Develop UI
+##########
+
+* npm install
+* npm start
+
+`server.js` serves a fake backend. You can add `container.json` and
+`traces.json` to this directory to serve them as /traces and /container
+respectively.
+
--- a/experimental/tracer/ui/favicon.ico
+++ b/experimental/tracer/ui/favicon.ico
--- a/experimental/tracer/ui/index.html
+++ b/experimental/tracer/ui/index.html
@@ -0,0 +1,393 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>Weave Tracer</title>
+    <!-- Latest compiled and minified CSS -->
+    <link rel="stylesheet" href="https://cask.scotch.io/bootstrap-4.0-flex.css">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.4.0/css/font-awesome.min.css">
+    <link href='https://fonts.googleapis.com/css?family=Roboto' rel='stylesheet' type='text/css'>
+
+    <!-- Latest compiled and minified JavaScript -->
+    <script src="http://code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/handlebars.js/3.0.3/handlebars.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.10.6/moment.min.js"></script>
+    <script src="sprintf.min.js"></script>
+
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+    <script>
+    $(function () {
+      var currentContainer = null;
+      var expandedTrace = null;
+      var containersByID = {};
+      var containersByPID = {};
+      var containers = [];
+      var traces = [];
+
+      Handlebars.registerHelper('isSelected', function(input) {
+        if (currentContainer && input === currentContainer.ID) {
+          return 'class=selected';
+        }
+      });
+
+      Handlebars.registerHelper('isExpanded', function(options) {
+        if (expandedTrace && this.Key === expandedTrace) {
+          return options.fn(this)
+        }
+      });
+
+      function containerName(trace) {
+        var container = containersByPID[trace.PID]
+        if (!container) {
+          return sprintf("%s:%d", trace.ToAddr, trace.ToPort)
+        }
+        return sprintf("%s (%d)", container.Name, trace.PID)
+      }
+      Handlebars.registerHelper('containerName', containerName);
+
+      Handlebars.registerHelper('spaces', function(input) {
+        return new Array(input + 1).join("> ");
+      });
+
+      Handlebars.registerHelper('ts', function(input) {
+        var ts = moment(input).format("LTS")
+        return new Handlebars.SafeString(ts);
+      });
+
+      Handlebars.registerHelper('duration', function(input) {
+        var durationText = formatDuration(input);
+        return new Handlebars.SafeString(durationText);
+      });
+
+      function numChildren(input) {
+        if (input.Children === null) {
+          return 0
+        }
+        var count = input.Children.length
+        $.each(input.Children, function(i, child) {
+          count += numChildren(child)
+        })
+        return count
+      }
+      Handlebars.registerHelper('count', function(input) {
+        return sprintf("%d", numChildren(input));
+      });
+
+      Handlebars.registerHelper('childTitle', function() {
+        var duration = formatDuration(this);
+        return '[' + duration + '] ' + containerName(this);
+      });
+
+      Handlebars.registerHelper('childWrapperStyle', function() {
+        var parentSpan = this.ParentStop - this.ParentStart; // = 100%
+        var span = (this.Stop - this.Start) / parentSpan * 100;
+        var offset = (this.Start - this.ParentStart) / parentSpan * 100;
+
+        return 'width:' + span + '%; left:' + offset + '%;';
+      });
+
+      Handlebars.registerHelper('childStyle', function() {
+        var color = shadeColor(weaveRed, this.Level / 5);
+        return 'width: 100%; background-color:' + color;
+      });
+
+      Handlebars.registerPartial('traces', $("#traces").html());
+      Handlebars.registerPartial('children', $("#children").html());
+      Handlebars.registerPartial('childrenDetails', $("#childrenDetails").html());
+
+      function render() {
+        var template = $('script#process-template').text();
+        template = Handlebars.compile(template);
+        var rendered = template({
+          containers: containers,
+          container: currentContainer,
+          traces: traces
+        });
+        $('body').html(rendered);
+      }
+
+      function updateContainers() {
+        $.get("/container").done(function (data) {
+          data.sort(function (a, b) {
+              if (a.Name > b.Name) {
+                  return 1;
+              }
+              if (a.Name < b.Name) {
+                  return -1;
+              }
+              // a must be equal to b
+              return 0;
+          });
+          containers = data;
+          containersByID = {};
+          containersByPID = {};
+          $.each(data, function(i, container) {
+            containersByID[container.ID] = container
+            $.each(container.PIDs, function(i, pid) {
+              containersByPID[pid] = container
+            });
+          });
+          // auto-select first container
+          if (containers.length && currentContainer === null) {
+            currentContainer = containersByID[containers[0].ID];
+          }
+          render();
+          window.setTimeout(updateContainers, 5 * 1000);
+        });
+      }
+      updateContainers()
+
+      var weaveRed = '#FF4B19';
+
+      function shadeColor(color, percent) {
+        var f=parseInt(color.slice(1),16),t=percent<0?0:255,p=percent<0?percent*-1:percent,R=f>>16,G=f>>8&0x00FF,B=f&0x0000FF;
+        return "#"+(0x1000000+(Math.round((t-R)*p)+R)*0x10000+(Math.round((t-G)*p)+G)*0x100+(Math.round((t-B)*p)+B)).toString(16).slice(1);
+      }
+
+      function formatDuration(input) {
+        var ms = input.Stop - input.Start
+        if (ms < 60000) {
+          return sprintf("%0.2fs", ms / 1000);
+        }
+        var ds = moment.duration(ms).humanize();
+        return ds;
+      }
+
+      function addParentTimeToTrace(trace) {
+        var details = trace.ServerDetails || trace.ClientDetails;
+        trace.Children && $.each(trace.Children, function(i, childTrace) {
+          childTrace.ParentStart = details.Start;
+          childTrace.ParentStop = details.Stop;
+          addParentTimeToTrace(childTrace);
+        });
+      }
+
+      function fetchTraces() {
+        $.get("/traces").done(function (data) {
+          traces = data;
+          traces && $.each(traces, function(i, trace) {
+            addParentTimeToTrace(trace);
+          });
+          render();
+          window.setTimeout(fetchTraces, 2 * 1000);
+        });
+      }
+      fetchTraces();
+
+      $("body").on("click", "ul.containers li", function() {
+        var container = containersByID[$(this).attr("id")]
+        currentContainer = container
+        render()
+      })
+
+      $("body").on("click", "div.mainview button.start", function() {
+        var id = $(this).parent().data("containerId")
+        var container = containersByID[id]
+        $.post(sprintf("/container/%s", container.ID))
+      })
+
+      $("body").on("click", "div.mainview button.stop", function() {
+        var id = $(this).parent().data("containerId")
+        var container = containersByID[id]
+        $.ajax({
+          url: sprintf("/container/%s", container.ID),
+          type: 'DELETE',
+        });
+      })
+
+      $("body").on("click", "table tr.trace", function() {
+        var key = $(this).data("key");
+        if (expandedTrace === key) {
+          expandedTrace = null;
+        } else {
+          expandedTrace = key;
+        }
+        render()
+      })
+    })
+    </script>
+    <style>
+      body {
+        height: 100%;
+        width: 100%;
+        color: #46466a;
+        font-family: "Roboto", sans-serif;
+      }
+      .logo {
+        margin-bottom: 30px;
+      }
+      .container-fluid {
+        background: linear-gradient(30deg, #e2e2ec 0%, #fafafc 100%);
+        padding: 30px 45px;
+        position: absolute;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        overflow-y: scroll;
+      }
+      ul.containers li {
+        cursor: pointer;
+        list-style: none;
+        font-size: 85%;
+        margin-bottom: 0.5rem;
+        opacity: 0.7;
+      }
+      ul.containers li.selected {
+        opacity: 1;
+      }
+      .heading {
+        margin: 10px 40px;
+        opacity: 0.4;
+        text-transform: uppercase;
+      }
+      .btn-default {
+        text-transform: uppercase;
+        margin-top: 3px;
+        opacity: 0.8;
+      }
+      h2 {
+        font-weight: normal;
+        margin-left: 1.5rem;
+        margin-right: 2rem;
+        margin-bottom: 1rem;
+      }
+      table {
+        width: 100%;
+      }
+      th {
+        text-transform: uppercase;
+        opacity: 0.4;
+        font-weight: normal;
+      }
+      tr.trace {
+        cursor: pointer;
+      }
+      .mainview {
+        margin-top: 2rem;
+      }
+      .table td {
+        font-size: 80%;
+        font-family: monospace;
+        padding-left: 0.5rem;
+      }
+      .table-striped tbody tr:nth-of-type(odd) {
+        background-color: #e2e2ec;
+      }
+      .childBox {
+        padding: 0.25rem;
+        white-space: nowrap;
+        overflow: hidden;
+        text-overflow: ellipsis;
+      }
+      .childBoxWrapper {
+        position: absolute;
+      }
+      .childRow {
+        position: relative;
+        height: 100px;
+      }
+    </style>
+    <script type="text/x-handlebars-template" id="traces">
+      {{#.}}
+      <tr class="trace" data-key="{{Key}}">
+      {{#if ClientDetails}}
+        {{#with ClientDetails}}
+        <td>{{spaces ../Level}}{{ts Start}}</td>
+        <td>{{containerName . PID=../PID}}</td>
+        <td>{{duration .}}</td>
+        <td>{{FromAddr}}:{{FromPort}}</td>
+        <td>{{ToAddr}}:{{ToPort}}</td>
+        <td>{{count ../.}}</td>
+        {{/with}}
+      {{else}}
+        {{#with ServerDetails}}
+        <td>{{spaces ../Level}}{{ts Start}}</td>
+        <td>{{containerName ../.}}</td>
+        <td>{{duration .}}</td>
+        <td>{{FromAddr}}:{{FromPort}}</td>
+        <td>{{ToAddr}}:{{ToPort}}</td>
+        <td>{{count ../.}}</td>
+        {{/with}}
+      {{/if}}
+      </tr>
+      {{#isExpanded}}
+        <tr>
+          <td colspan="6" style="padding: 0 0.5rem;">
+            <div style="width: 100%; background-color: #FF4B19; height: 4px;"></div>
+            {{>children Children}}
+        </td>
+      </tr>
+      {{/isExpanded}}
+      {{/.}}
+    </script>
+
+    <script type="text/x-handlebars-template" id="childrenDetails">
+      <div style="{{childWrapperStyle}}" class="childBoxWrapper">
+        <div title="{{childTitle}}" style="{{childStyle}}" class="childBox">
+          {{childTitle}}
+        </div>
+        <div class="childRow">
+          {{>children Children}}
+        </div>
+      </div>
+    </script>
+
+    <script type="text/x-handlebars-template" id="children">
+      <div class="childRow">
+      {{#.}}
+      {{#if ClientDetails}}
+        {{>childrenDetails ClientDetails Level=../Level PID=../PID Children=../Children ParentStart=../ParentStart ParentStop=../ParentStop}}
+      {{else}}
+        {{>childrenDetails ServerDetails Level=../Level PID=../PID Children=../Children ParentStart=../ParentStart ParentStop=../ParentStop}}
+      {{/if}}
+      {{/.}}
+      </div>
+    </script>
+
+    <script type="text/x-handlebars-template" id="process-template">
+      <div class="container-fluid">
+        <div class="row">
+        <div class="col-md-4">
+        <div class="logo"><img src="logo.svg" width="300"/></div>
+          <div class="heading">Containers</div>
+          <ul class="containers">
+            {{#containers}}
+            <li {{isSelected ID}} id={{ID}}>{{Name}}</li>
+            {{/containers}}
+          </ul>
+        </div>
+
+        <div class="col-md-8 mainview">
+          {{#if container}}
+          <h2 class="pull-left">{{container.Name}}</h2>
+          <div class="btn-group btn-group-sm" role="group" data-container-id="{{container.ID}}">
+            <button type="button" class="btn btn-default start">
+              <span class="fa fa-play" aria-hidden="true"></span> Start</button>
+            <button type="button" class="btn btn-default stop">
+              <span class="fa fa-stop" aria-hidden="true"></span> Stop</button>
+          </div>
+
+          <table class="table table-sm">
+            <thead><tr>
+              <th width="15%">Start time</th>
+              <th width="25%">Container</th>
+              <th width="15%">Duration</th>
+              <th width="15%">From</th>
+              <th width="15%">To</th>
+              <th width="15%">Sub-traces</th>
+            </tr></thead>
+            <tbody>
+          {{>traces traces}}
+            </tbody>
+          </table>
+
+          {{/if}}
+        </div>
+        </div>
+      </div>
+    </script>
+  </head>
+  <body>
+  </body>
+</html>
--- a/experimental/tracer/ui/logo.svg
+++ b/experimental/tracer/ui/logo.svg
@@ -0,0 +1,54 @@
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 width="960px" height="173px" viewBox="0 0 960 173" enable-background="new 0 0 960 173" xml:space="preserve">
+<path fill="#32324B" d="M51.937,95.165l75.419-67.366c-5.989-4.707-12.71-8.52-19.981-11.211l-55.438,49.52V95.165z"/>
+<path fill="#32324B" d="M30.265,85.465l-20.431,18.25c1.86,7.57,4.88,14.683,8.87,21.135l11.561-10.326V85.465z"/>
+<path fill="#00D2FF" d="M92.276,30.074V12.768C88.815,12.27,85.282,12,81.685,12c-3.766,0-7.465,0.286-11.079,0.828v36.604
+	L92.276,30.074z"/>
+<path fill="#00D2FF" d="M92.276,131.874V59.133L70.605,78.49v80.682c3.614,0.543,7.313,0.828,11.079,0.828
+	c4.41,0,8.723-0.407,12.921-1.147l58.033-51.838c1.971-6.664,3.046-13.712,3.046-21.015c0-3.439-0.254-6.817-0.708-10.132
+	L92.276,131.874z"/>
+<path fill="#FF4B19" d="M92.276,110.518l58.14-51.933c-2.77-6.938-6.551-13.358-11.175-19.076L92.276,81.46V110.518z"/>
+<path fill="#FF4B19" d="M70.605,100.817l-18.668,16.676V18.242c-8.086,3.555-15.409,8.513-21.672,14.567V139.19
+	c4.885,4.724,10.409,8.787,16.444,12.03l23.896-21.345V100.817z"/>
+<polygon fill="#32324B" points="262.563,101.099 276.389,49.22 294.955,49.22 274.414,121.377 252.556,121.377 240.311,72.79 
+	228.065,121.377 206.207,121.377 185.666,49.22 204.232,49.22 218.058,101.099 231.752,49.22 248.869,49.22 "/>
+<path fill="#32324B" d="M363.429,97.676c-2.106,14.352-13.167,24.623-32.128,24.623c-20.146,0-35.025-12.114-35.025-36.605
+	c0-24.622,15.406-37.395,35.025-37.395c21.726,0,33.182,15.933,33.182,37.263v3.819h-49.772c0,8.031,3.291,18.17,16.327,18.17
+	c7.242,0,12.904-3.555,14.353-10.27L363.429,97.676z M345.654,76.608c-0.659-10.008-7.11-13.694-14.484-13.694
+	c-8.427,0-14.879,5.135-15.801,13.694H345.654z"/>
+<path fill="#32324B" d="M417.628,74.634v-2.502c0-5.662-2.37-9.351-13.036-9.351c-13.298,0-13.694,7.375-13.694,9.877h-17.117
+	c0-10.666,4.477-24.359,31.338-24.359c25.676,0,30.285,12.771,30.285,23.174v39.766c0,2.897,0.131,5.267,0.395,7.11l0.527,3.028
+	h-18.172v-7.241c-5.134,5.134-12.245,8.163-22.384,8.163c-14.221,0-25.018-8.296-25.018-22.648c0-16.59,15.67-20.146,21.99-21.199
+	L417.628,74.634z M417.628,88.195l-6.979,1.054c-3.819,0.658-8.427,1.315-11.192,1.843c-3.029,0.527-5.662,1.186-7.637,2.765
+	c-1.844,1.449-2.765,3.425-2.765,5.926c0,2.107,0.79,8.69,10.666,8.69c5.793,0,10.928-2.105,13.693-4.872
+	c3.556-3.555,4.214-8.032,4.214-11.587V88.195z"/>
+<polygon fill="#32324B" points="486.495,121.377 462.399,121.377 438.698,49.221 458.186,49.221 474.775,104.392 491.499,49.221 
+	510.459,49.221 "/>
+<path fill="#32324B" d="M578.273,97.676c-2.106,14.352-13.167,24.623-32.128,24.623c-20.146,0-35.025-12.114-35.025-36.605
+	c0-24.622,15.406-37.395,35.025-37.395c21.726,0,33.182,15.933,33.182,37.263v3.819h-49.772c0,8.031,3.291,18.17,16.327,18.17
+	c7.242,0,12.904-3.555,14.354-10.27L578.273,97.676z M560.498,76.608c-0.659-10.008-7.109-13.694-14.483-13.694
+	c-8.428,0-14.88,5.135-15.802,13.694H560.498z"/>
+<g>
+	<path fill="#32324B" d="M615.355,120.575c-11.88,0-15.841-5.017-15.841-16.765V58.797h-11.616v-9.108h11.616V27.776h10.825v21.912
+		h18.216v9.108H610.34v42.241c0,8.977,1.979,10.164,9.107,10.164c1.98,0,7.524-0.396,8.185-0.396v8.976
+		C627.632,119.782,622.22,120.575,615.355,120.575z"/>
+	<path fill="#32324B" d="M674.095,60.777c-12.672,0-21.517,4.355-21.517,20.988v38.413h-10.824v-70.49h10.692v12.145
+		c4.62-8.977,13.992-12.145,21.385-12.145h5.016v11.089H674.095z"/>
+	<path fill="#32324B" d="M732.703,75.957v-3.828c0-9.768-4.224-14.388-16.633-14.388c-13.332,0-17.028,6.072-17.556,13.2H687.69
+		c0-9.108,5.147-22.177,28.116-22.177c24.157,0,27.721,12.54,27.721,23.761v36.301c0,3.301,0.132,8.58,0.924,11.353h-10.956
+		c0,0-0.264-3.828-0.264-9.504c-3.433,4.487-10.297,10.428-24.685,10.428c-13.597,0-23.761-7.92-23.761-21.12
+		c0-14.652,11.748-18.613,18.876-20.064C710.395,78.598,732.703,75.957,732.703,75.957z M732.703,84.67
+		c0,0-15.312,1.979-23.101,3.168c-7.524,1.056-13.597,3.564-13.597,11.88c0,6.996,4.753,12.276,14.389,12.276
+		c13.332,0,22.309-7.656,22.309-19.272V84.67z"/>
+	<path fill="#32324B" d="M790.649,111.994c10.429,0,17.953-5.939,19.009-16.632h10.957c-1.98,17.028-13.597,25.74-29.966,25.74
+		c-18.744,0-32.076-12.012-32.076-35.905c0-23.76,13.464-36.433,32.209-36.433c16.104,0,27.721,8.712,29.568,25.213h-10.956
+		c-1.452-11.353-9.24-16.104-18.877-16.104c-12.012,0-20.856,8.448-20.856,27.324C769.661,104.471,778.901,111.994,790.649,111.994z
+		"/>
+	<path fill="#32324B" d="M895.062,96.022c-1.452,12.408-10.032,25.08-30.229,25.08c-18.745,0-32.341-12.804-32.341-36.037
+		c0-21.912,13.464-36.301,32.209-36.301c19.8,0,30.757,14.784,30.757,38.018h-51.878c0.265,13.332,5.809,25.212,21.385,25.212
+		c11.484,0,18.217-7.128,19.141-16.104L895.062,96.022z M883.71,78.201c-1.056-14.916-9.636-20.328-19.272-20.328
+		c-10.824,0-19.141,7.26-20.46,20.328H883.71z"/>
+	<path fill="#32324B" d="M942.976,60.777c-12.672,0-21.517,4.355-21.517,20.988v38.413h-10.824v-70.49h10.692v12.145
+		c4.62-8.977,13.992-12.145,21.385-12.145h5.016v11.089H942.976z"/>
+</g>
+</svg>
--- a/experimental/tracer/ui/package.json
+++ b/experimental/tracer/ui/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "scope-tracer",
+  "version": "1.0.0",
+  "description": "",
+  "main": "server.js",
+  "scripts": {
+    "start": "node server.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "David Kaltschmidt <david.kaltschmidt@gmail.com>",
+  "license": "UNLICENSED",
+  "devDependencies": {
+    "express": "^4.13.3"
+  }
+}
--- a/experimental/tracer/ui/server.js
+++ b/experimental/tracer/ui/server.js
@@ -0,0 +1,25 @@
+var express = require('express')
+
+var app = express();
+
+app.get('/', function(req, res) {
+    res.sendFile(__dirname + '/index.html');
+});
+
+app.get('/container', function(req, res) {
+    res.sendFile(__dirname + '/container.json');
+});
+
+app.get('/traces', function(req, res) {
+    res.sendFile(__dirname + '/traces.json');
+});
+
+app.use(express.static('./'));
+
+var port = process.env.PORT || 4050;
+var server = app.listen(port, function () {
+    var host = server.address().address;
+    var port = server.address().port;
+
+    console.log('Scope Tracer UI listening at http://%s:%s', host, port);
+});
--- a/experimental/tracer/ui/sprintf.min.js
+++ b/experimental/tracer/ui/sprintf.min.js
@@ -0,0 +1,4 @@
+/*! sprintf-js | Alexandru Marasteanu <hello@alexei.ro> (http://alexei.ro/) | BSD-3-Clause */
+
+!function(a){function b(){var a=arguments[0],c=b.cache;return c[a]&&c.hasOwnProperty(a)||(c[a]=b.parse(a)),b.format.call(null,c[a],arguments)}function c(a){return Object.prototype.toString.call(a).slice(8,-1).toLowerCase()}function d(a,b){return Array(b+1).join(a)}var e={not_string:/[^s]/,number:/[dief]/,json:/[j]/,not_json:/[^j]/,text:/^[^\x25]+/,modulo:/^\x25{2}/,placeholder:/^\x25(?:([1-9]\d*)\$|\(([^\)]+)\))?(\+)?(0|'[^$])?(-)?(\d+)?(?:\.(\d+))?([b-fijosuxX])/,key:/^([a-z_][a-z_\d]*)/i,key_access:/^\.([a-z_][a-z_\d]*)/i,index_access:/^\[(\d+)\]/,sign:/^[\+\-]/};b.format=function(a,f){var g,h,i,j,k,l,m,n=1,o=a.length,p="",q=[],r=!0,s="";for(h=0;o>h;h++)if(p=c(a[h]),"string"===p)q[q.length]=a[h];else if("array"===p){if(j=a[h],j[2])for(g=f[n],i=0;i<j[2].length;i++){if(!g.hasOwnProperty(j[2][i]))throw new Error(b("[sprintf] property '%s' does not exist",j[2][i]));g=g[j[2][i]]}else g=j[1]?f[j[1]]:f[n++];if("function"==c(g)&&(g=g()),e.not_string.test(j[8])&&e.not_json.test(j[8])&&"number"!=c(g)&&isNaN(g))throw new TypeError(b("[sprintf] expecting number but found %s",c(g)));switch(e.number.test(j[8])&&(r=g>=0),j[8]){case"b":g=g.toString(2);break;case"c":g=String.fromCharCode(g);break;case"d":case"i":g=parseInt(g,10);break;case"j":g=JSON.stringify(g,null,j[6]?parseInt(j[6]):0);break;case"e":g=j[7]?g.toExponential(j[7]):g.toExponential();break;case"f":g=j[7]?parseFloat(g).toFixed(j[7]):parseFloat(g);break;case"o":g=g.toString(8);break;case"s":g=(g=String(g))&&j[7]?g.substring(0,j[7]):g;break;case"u":g>>>=0;break;case"x":g=g.toString(16);break;case"X":g=g.toString(16).toUpperCase()}e.json.test(j[8])?q[q.length]=g:(!e.number.test(j[8])||r&&!j[3]?s="":(s=r?"+":"-",g=g.toString().replace(e.sign,"")),l=j[4]?"0"===j[4]?"0":j[4].charAt(1):" ",m=j[6]-(s+g).length,k=j[6]&&m>0?d(l,m):"",q[q.length]=j[5]?s+g+k:"0"===l?s+k+g:k+s+g)}return q.join("")},b.cache={},b.parse=function(a){for(var b=a,c=[],d=[],f=0;b;){if(null!==(c=e.text.exec(b)))d[d.length]=c[0];else if(null!==(c=e.modulo.exec(b)))d[d.length]="%";else{if(null===(c=e.placeholder.exec(b)))throw new SyntaxError("[sprintf] unexpected placeholder");if(c[2]){f|=1;var g=[],h=c[2],i=[];if(null===(i=e.key.exec(h)))throw new SyntaxError("[sprintf] failed to parse named argument key");for(g[g.length]=i[1];""!==(h=h.substring(i[0].length));)if(null!==(i=e.key_access.exec(h)))g[g.length]=i[1];else{if(null===(i=e.index_access.exec(h)))throw new SyntaxError("[sprintf] failed to parse named argument key");g[g.length]=i[1]}c[2]=g}else f|=2;if(3===f)throw new Error("[sprintf] mixing positional and named placeholders is not (yet) supported");d[d.length]=c}b=b.substring(c[0].length)}return d};var f=function(a,c,d){return d=(c||[]).slice(0),d.splice(0,0,a),b.apply(null,d)};"undefined"!=typeof exports?(exports.sprintf=b,exports.vsprintf=f):(a.sprintf=b,a.vsprintf=f,"function"==typeof define&&define.amd&&define(function(){return{sprintf:b,vsprintf:f}}))}("undefined"==typeof window?this:window);
+//# sourceMappingURL=sprintf.min.js.map
--- a/probe/docker/container.go
+++ b/probe/docker/container.go
@@ -86,7 +86,7 @@ type Container interface {
 	GetNode(string, []net.IP) report.Node
 	State() string
 	HasTTY() bool
-
+	Container() *docker.Container
 	StartGatheringStats() error
 	StopGatheringStats()
 }
@@ -146,6 +146,10 @@ func (c *container) State() string {
 	return StateStopped
 }

+func (c *container) Container() *docker.Container {
+	return c.container
+}
+
 func (c *container) StartGatheringStats() error {
 	c.Lock()
 	defer c.Unlock()
--- a/probe/docker/registry_test.go
+++ b/probe/docker/registry_test.go
@@ -56,6 +56,10 @@ func (c *mockContainer) GetNode(_ string, _ []net.IP) report.Node {
 	})
 }

+func (c *mockContainer) Container() *client.Container {
+	return c.c
+}
+
 func (c *mockContainer) HasTTY() bool { return true }

 type mockDockerClient struct {
--- a/probe/docker/tagger_test.go
+++ b/probe/docker/tagger_test.go
@@ -23,6 +23,10 @@ func (m *mockProcessTree) GetParent(pid int) (int, error) {
 	return parent, nil
 }

+func (m *mockProcessTree) GetChildren(int) ([]int, error) {
+	panic("Not implemented")
+}
+
 func TestTagger(t *testing.T) {
 	oldProcessTree := docker.NewProcessTreeStub
 	defer func() { docker.NewProcessTreeStub = oldProcessTree }()
--- a/probe/process/tree.go
+++ b/probe/process/tree.go
@@ -7,6 +7,7 @@ import (
 // Tree represents all processes on the machine.
 type Tree interface {
 	GetParent(pid int) (int, error)
+	GetChildren(pid int) ([]int, error)
 }

 type tree struct {
@@ -32,3 +33,30 @@ func (pt *tree) GetParent(pid int) (int, error) {

 	return proc.PPID, nil
 }
+
+// GetChildren
+func (pt *tree) GetChildren(pid int) ([]int, error) {
+	_, ok := pt.processes[pid]
+	if !ok {
+		return []int{}, fmt.Errorf("PID %d not found", pid)
+	}
+
+	var isChild func(id int) bool
+	isChild = func(id int) bool {
+		p, ok := pt.processes[id]
+		if !ok || p.PPID == 0 {
+			return false
+		} else if p.PPID == pid {
+			return true
+		}
+		return isChild(p.PPID)
+	}
+
+	children := []int{pid}
+	for id := range pt.processes {
+		if isChild(id) {
+			children = append(children, id)
+		}
+	}
+	return children, nil
+}