Merge branch 'master' into remove-topology-type

.gitignore

@@ -31,6 +31,7 @@ coverage.html
 
 # Project specific
 scope.tar
+scope_ui_build.tar
 app/app
 probe/probe
 docker/app

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all build client static dist test clean
+.PHONY: all static test clean
 
 # If you can use Docker without being root, you can `make SUDO= <target>`
 SUDO=sudo
@@ -9,22 +9,29 @@ PROBE_EXE=probe/probe
 FIXPROBE_EXE=experimental/fixprobe/fixprobe
 SCOPE_IMAGE=$(DOCKERHUB_USER)/scope
 SCOPE_EXPORT=scope.tar
+SCOPE_UI_BUILD_EXPORT=scope_ui_build.tar
+SCOPE_UI_BUILD_IMAGE=weave/scope-ui-build
 
 all: $(SCOPE_EXPORT)
-dist: client static $(APP_EXE) $(PROBE_EXE)
 
-client:
-	cd client && make build && rm -f dist/.htaccess
+$(SCOPE_UI_BUILD_EXPORT): client/Dockerfile client/gulpfile.js client/package.json
+	docker build -t $(SCOPE_UI_BUILD_IMAGE) client
+	docker save $(SCOPE_UI_BUILD_IMAGE):latest > $@
 
-app/static.go:
-	go get github.com/mjibson/esc
+client/dist/scripts/bundle.js: client/app/scripts/*
+	mkdir -p client/dist
+	docker run -ti -v $(shell pwd)/client/app:/home/weave/app \
+		-v $(shell pwd)/client/dist:/home/weave/dist \
+		$(SCOPE_UI_BUILD_IMAGE)
+
+static: client/dist/scripts/bundle.js
 	esc -o app/static.go -prefix client/dist client/dist
 
 test: $(APP_EXE) $(FIXPROBE_EXE)
 	# app and fixprobe needed for integration tests
 	go test ./...
 
-$(APP_EXE): app/*.go app/static.go report/*.go xfer/*.go
+$(APP_EXE): app/*.go report/*.go xfer/*.go
 $(PROBE_EXE): probe/*.go report/*.go xfer/*.go
 
 $(APP_EXE) $(PROBE_EXE):
@@ -34,11 +41,11 @@ $(APP_EXE) $(PROBE_EXE):
 $(FIXPROBE_EXE):
 	cd experimental/fixprobe && go build
 
-$(SCOPE_EXPORT):  $(APP_EXE) $(PROBE_EXE) docker/Dockerfile docker/entrypoint.sh
+$(SCOPE_EXPORT):  $(APP_EXE) $(PROBE_EXE) docker/*
 	cp $(APP_EXE) $(PROBE_EXE) docker/
 	$(SUDO) docker build -t $(SCOPE_IMAGE) docker/
 	$(SUDO) docker save $(SCOPE_IMAGE):latest > $@
 
 clean:
 	go clean ./...
-	rm -f $(SCOPE_EXPORT) app/static.go
+	rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_EXPORT) client/dist

README.md

@@ -47,6 +47,15 @@ To build a Docker container,
 make docker
 ```
 
+### The UI
+
+This repository contains a copy of the compiled UI.  To build a fresh UI, run:
+
+```
+make scope_ui_build.tar
+make static
+```
+
 ## Running
 
 ### Manually

circle.yml

@@ -10,19 +10,42 @@ machine:
     GOPATH: /home/ubuntu:$GOPATH
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope
     PATH: $PATH:$HOME/.local/bin
+    SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar
 
 dependencies:
+  cache_directories:
+    - "~/docker"
+  override:
+    - if [[ -e "$SCOPE_UI_BUILD" ]]; then
+        docker load -i $SCOPE_UI_BUILD;
+      else
+        make scope_ui_build.tar;
+        mkdir -p $(dirname "$SCOPE_UI_BUILD");
+        mv scope_ui_build.tar $(dirname "$SCOPE_UI_BUILD");
+      fi
   post:
     - go get github.com/golang/lint/golint
     - go get github.com/fzipp/gocyclo
     - go get github.com/mattn/goveralls
+    - go get github.com/mjibson/esc
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
 
 test:
   override:
     - cd $SRCDIR; ./bin/lint .
+    - cd $SRCDIR; make static
     - cd $SRCDIR; make
     - cd $SRCDIR; ./bin/test
   post:
-    - goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci
+    - goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true
+    - cd $SRCDIR; cp coverage.html $CIRCLE_ARTIFACTS
+    - cd $SRCDIR; cp scope.tar $CIRCLE_ARTIFACTS
+
+deployment:
+  hub:
+    branch: master
+    owner: weaveworks
+    commands:
+      - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - docker push weaveworks/scope

client/.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+dist

client/Dockerfile

@@ -0,0 +1,19 @@
+FROM mhart/alpine-node
+
+WORKDIR /home/weave
+
+# build tool
+RUN npm install -g gulp
+
+# install app and build dependencies
+ADD package.json /home/weave/
+RUN npm install
+
+ADD gulpfile.js /home/weave/
+
+# run container via
+#
+# `docker run -v $GOPATH/src/github.com/weaveworks/scope/client:/app weaveworks/scope-build`
+#
+# after the container is run, bundled app should be in ./dist/ dir
+CMD gulp build

client/gulpfile.js

@@ -47,8 +47,6 @@ gulp.task('scripts', function() {
 gulp.task('html', ['styles', 'scripts'], function () {
     return gulp.src('app/*.html')
         .pipe($.preprocess())
-        //.pipe($.useref.assets({searchPath: 'dist'}))
-        //.pipe($.useref())
         .pipe(gulp.dest('dist'))
         .pipe($.size())
         .pipe(livereload());
@@ -107,9 +105,7 @@ gulp.task('connect', function () {
     });
 });
 
-gulp.task('serve', ['connect', 'styles', 'scripts', 'fonts'], function () {
-    //require('opn')('http://localhost:9000');
-});
+gulp.task('serve', ['connect', 'styles', 'scripts', 'fonts']);
 
 gulp.task('watch', ['serve'], function () {
     livereload.listen();

client/package.json

@@ -1,53 +1,52 @@
 {
-  "name": "scope-webapp",
-  "version": "0.2.0",
+  "name": "weave-scope",
+  "version": "1.2.0",
+  "description": "SPA JS app for Weave Scope visualising the application network.",
+  "repository": "weaveworks/scope",
+  "license": "Apache-2.0",
+  "private": true,
   "dependencies": {
-    "d3": "^3.5.3",
-    "dagre": "^0.7.1",
-    "flux": "^2.0.1",
+    "d3": "^3.5.5",
+    "dagre": "^0.7.2",
+    "flux": "^2.0.3",
     "font-awesome": "^4.3.0",
     "keymirror": "^0.1.1",
-    "lodash": "~3.0.1",
+    "lodash": "~3.8.0",
     "material-ui": "^0.7.5",
     "object-assign": "^2.0.0",
-    "page": "^1.6.0",
-    "react": "^0.13.2",
+    "page": "^1.6.3",
+    "react": "^0.13.3",
+    "react-tap-event-plugin": "^0.1.6",
     "react-tween-state": "0.0.5",
     "reqwest": "~1.1.5"
   },
   "devDependencies": {
-    "browserify": "^8.1.3",
-    "connect": "^3.3.4",
+    "browserify": "^10.2.0",
     "del": "^1.1.1",
-    "gulp": "^3.8.10",
-    "gulp-autoprefixer": "^2.1.0",
-    "gulp-cache": "^0.2.4",
-    "gulp-clean": "^0.3.1",
+    "gulp": "^3.8.11",
+    "gulp-autoprefixer": "^2.3.0",
     "gulp-connect": "^2.2.0",
     "gulp-csso": "^1.0.0",
-    "gulp-filter": "^2.0.0",
+    "gulp-filter": "^2.0.2",
     "gulp-flatten": "^0.0.4",
     "gulp-if": "^1.2.5",
-    "gulp-imagemin": "^2.1.0",
-    "gulp-jshint": "^1.9.2",
+    "gulp-jshint": "^1.10.0",
     "gulp-less": "^3.0.3",
     "gulp-livereload": "^3.8.0",
-    "gulp-load-plugins": "^0.8.0",
+    "gulp-load-plugins": "^0.10.0",
     "gulp-preprocess": "^1.2.0",
-    "gulp-size": "^1.2.0",
-    "gulp-sourcemaps": "^1.3.0",
-    "gulp-uglify": "^1.1.0",
-    "gulp-useref": "^1.1.1",
-    "gulp-util": "^3.0.3",
-    "jshint-stylish": "^1.0.0",
-    "opn": "^1.0.1",
-    "proxy-middleware": "^0.11.0",
-    "reactify": "^1.1.0",
+    "gulp-size": "^1.2.1",
+    "gulp-sourcemaps": "^1.5.2",
+    "gulp-uglify": "^1.2.0",
+    "gulp-util": "^3.0.4",
+    "jshint-stylish": "^1.0.2",
+    "proxy-middleware": "^0.11.1",
+    "reactify": "^1.1.1",
     "vinyl-buffer": "^1.0.0",
-    "vinyl-source-stream": "^1.0.0"
+    "vinyl-source-stream": "^1.1.0"
   },
   "scripts": {
-    "start": "npm install && gulp serve"
+    "start": "gulp"
   },
   "engines": {
     "node": ">=0.10.0"

docker/run-probe

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec /home/weave/probe
+exec /home/weave/probe -proc.root=/hostproc

probe/docker_process_mapper.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	docker "github.com/fsouza/go-dockerclient"
+)
+
+type dockerMapper struct {
+	sync.RWMutex
+	d        map[int]*docker.Container
+	procRoot string
+}
+
+func newDockerMapper(procRoot string, interval time.Duration) *dockerMapper {
+	m := dockerMapper{
+		procRoot: procRoot,
+		d:        map[int]*docker.Container{},
+	}
+	m.update()
+	go m.loop(interval)
+	return &m
+}
+
+func (m *dockerMapper) loop(d time.Duration) {
+	for range time.Tick(d) {
+		m.update()
+	}
+}
+
+// for mocking
+type dockerClient interface {
+	ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error)
+	InspectContainer(string) (*docker.Container, error)
+}
+
+func newRealDockerClient(endpoint string) (dockerClient, error) {
+	return docker.NewClient(endpoint)
+}
+
+var (
+	newDockerClient = newRealDockerClient
+	newPIDTreeStub  = newPIDTree
+)
+
+func (m *dockerMapper) update() {
+	pidTree, err := newPIDTreeStub(m.procRoot)
+	if err != nil {
+		log.Printf("docker mapper: %s", err)
+		return
+	}
+
+	endpoint := "unix:///var/run/docker.sock"
+	client, err := newDockerClient(endpoint)
+	if err != nil {
+		log.Printf("docker mapper: %s", err)
+		return
+	}
+
+	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
+	if err != nil {
+		log.Printf("docker mapper: %s", err)
+		return
+	}
+
+	pmap := map[int]*docker.Container{}
+	for _, container := range containers {
+		info, err := client.InspectContainer(container.ID)
+		if err != nil {
+			log.Printf("docker mapper: %s", err)
+			continue
+		}
+
+		if !info.State.Running {
+			continue
+		}
+
+		pids, err := pidTree.allChildren(info.State.Pid)
+		if err != nil {
+			log.Printf("docker mapper: %s", err)
+			continue
+		}
+		for _, pid := range pids {
+			pmap[pid] = info
+		}
+	}
+
+	m.Lock()
+	m.d = pmap
+	m.Unlock()
+}
+
+type dockerIDMapper struct {
+	*dockerMapper
+}
+
+func (m dockerIDMapper) Key() string { return "docker_id" }
+func (m dockerIDMapper) Map(pid uint) (string, error) {
+	m.RLock()
+	container, ok := m.d[int(pid)]
+	m.RUnlock()
+
+	if !ok {
+		return "", fmt.Errorf("no container found for PID %d", pid)
+	}
+
+	return container.ID, nil
+}
+
+type dockerNameMapper struct {
+	*dockerMapper
+}
+
+func (m dockerNameMapper) Key() string { return "docker_name" }
+func (m dockerNameMapper) Map(pid uint) (string, error) {
+	m.RLock()
+	container, ok := m.d[int(pid)]
+	m.RUnlock()
+
+	if !ok {
+		return "", fmt.Errorf("no container found for PID %d", pid)
+	}
+
+	return container.Name, nil
+}

probe/docker_process_mapper_test.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	docker "github.com/fsouza/go-dockerclient"
+)
+
+type mockDockerClient struct {
+	containers    []docker.APIContainers
+	containerInfo map[string]*docker.Container
+}
+
+func (m mockDockerClient) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) {
+	return m.containers, nil
+}
+
+func (m mockDockerClient) InspectContainer(id string) (*docker.Container, error) {
+	return m.containerInfo[id], nil
+}
+
+func TestDockerProcessMapper(t *testing.T) {
+	oldPIDTreeStub, oldDockerClientStub := newPIDTreeStub, newDockerClient
+	defer func() {
+		newPIDTreeStub = oldPIDTreeStub
+		newDockerClient = oldDockerClientStub
+	}()
+
+	newPIDTreeStub = func(procRoot string) (*pidTree, error) {
+		pid1 := &process{pid: 1}
+		pid2 := &process{pid: 2, ppid: 1, parent: pid1}
+		pid1.children = []*process{pid2}
+
+		return &pidTree{
+			processes: map[int]*process{
+				1: pid1, 2: pid2,
+			},
+		}, nil
+	}
+
+	newDockerClient = func(endpoint string) (dockerClient, error) {
+		return mockDockerClient{
+			containers: []docker.APIContainers{{ID: "foo"}},
+			containerInfo: map[string]*docker.Container{
+				"foo": {
+					ID:    "foo",
+					Name:  "bar",
+					State: docker.State{Pid: 1, Running: true},
+				},
+			},
+		}, nil
+	}
+
+	dockerMapper := newDockerMapper("/proc", 10*time.Second)
+	dockerIDMapper := dockerIDMapper{dockerMapper}
+	dockerNameMapper := dockerNameMapper{dockerMapper}
+
+	for pid, want := range map[uint]struct{ id, name string }{
+		1: {"foo", "bar"},
+		2: {"foo", "bar"},
+	} {
+		haveID, err := dockerIDMapper.Map(pid)
+		if err != nil || want.id != haveID {
+			t.Errorf("%d: want %q, have %q (%v)", pid, want.id, haveID, err)
+		}
+		haveName, err := dockerNameMapper.Map(pid)
+		if err != nil || want.name != haveName {
+			t.Errorf("%d: want %q, have %q (%v)", pid, want.name, haveName, err)
+		}
+	}
+}

probe/main.go

@@ -27,7 +27,9 @@ func main() {
 		prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)")
 		spyProcs           = flag.Bool("processes", true, "report processes (needs root)")
 		cgroupsRoot        = flag.String("cgroups.root", "", "if provided, enrich -processes with cgroup names from this root (e.g. /mnt/cgroups)")
-		cgroupsUpdate      = flag.Duration("cgroups.update", 10*time.Second, "how often to update cgroup names")
+		cgroupsInterval    = flag.Duration("cgroups.interval", 10*time.Second, "how often to update cgroup names")
+		dockerMapper       = flag.Bool("docker", true, "collect docker-related attributes for processes.")
+		dockerInterval     = flag.Duration("docker.interval", 10*time.Second, "how often to update docker container info")
 		procRoot           = flag.String("proc.root", "/proc", "location of the proc filesystem")
 	)
 	flag.Parse()
@@ -64,12 +66,17 @@ func main() {
 	if *cgroupsRoot != "" {
 		if fi, err := os.Stat(*cgroupsRoot); err == nil && fi.IsDir() {
 			log.Printf("enriching -processes with cgroup names from %s", *cgroupsRoot)
-			pms = append(pms, newCgroupMapper(*cgroupsRoot, *cgroupsUpdate))
+			pms = append(pms, newCgroupMapper(*cgroupsRoot, *cgroupsInterval))
 		} else {
 			log.Printf("-cgroups.root=%s: %v", *cgroupsRoot, err)
 		}
 	}
 
+	if *dockerMapper {
+		docker := newDockerMapper(*procRoot, *dockerInterval)
+		pms = append(pms, &dockerIDMapper{docker}, &dockerNameMapper{docker})
+	}
+
 	log.Printf("listening on %s", *listen)
 
 	go func() {

probe/pidtree.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strconv"
+	"strings"
+)
+
+type pidTree struct {
+	processes map[int]*process
+}
+
+type process struct {
+	pid, ppid int
+	parent    *process
+	children  []*process
+}
+
+// Hooks for mocking
+var (
+	readDir  = ioutil.ReadDir
+	readFile = ioutil.ReadFile
+)
+
+func newPIDTree(procRoot string) (*pidTree, error) {
+	dirEntries, err := readDir(procRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	pt := pidTree{processes: map[int]*process{}}
+	for _, dirEntry := range dirEntries {
+		pid, err := strconv.Atoi(dirEntry.Name())
+		if err != nil {
+			continue
+		}
+
+		stat, err := readFile(path.Join(procRoot, dirEntry.Name(), "stat"))
+		if err != nil {
+			continue
+		}
+
+		splits := strings.Split(string(stat), " ")
+		ppid, err := strconv.Atoi(splits[3])
+		if err != nil {
+			return nil, err
+		}
+
+		pt.processes[pid] = &process{pid: pid, ppid: ppid}
+	}
+
+	for _, child := range pt.processes {
+		parent, ok := pt.processes[child.ppid]
+		if !ok {
+			// This can happen as listing proc is not a consistent snapshot
+			continue
+		}
+		child.parent = parent
+		parent.children = append(parent.children, child)
+	}
+
+	return &pt, nil
+}
+
+// allChildren returns a flattened list of child pids including the given pid
+func (pt *pidTree) allChildren(pid int) ([]int, error) {
+	proc, ok := pt.processes[pid]
+	if !ok {
+		return []int{}, fmt.Errorf("PID %d not found", pid)
+	}
+
+	var result []int
+
+	var f func(*process)
+	f = func(p *process) {
+		result = append(result, p.pid)
+		for _, child := range p.children {
+			f(child)
+		}
+	}
+
+	f(proc)
+	return result, nil
+}

probe/pidtree_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+type fileinfo struct {
+	name string
+}
+
+func (f fileinfo) Name() string       { return f.name }
+func (f fileinfo) Size() int64        { return 0 }
+func (f fileinfo) Mode() os.FileMode  { return 0 }
+func (f fileinfo) ModTime() time.Time { return time.Now() }
+func (f fileinfo) IsDir() bool        { return true }
+func (f fileinfo) Sys() interface{}   { return nil }
+
+func TestPIDTree(t *testing.T) {
+	oldReadDir, oldReadFile := readDir, readFile
+	defer func() {
+		readDir = oldReadDir
+		readFile = oldReadFile
+	}()
+
+	readDir = func(path string) ([]os.FileInfo, error) {
+		return []os.FileInfo{
+			fileinfo{"3"}, fileinfo{"2"}, fileinfo{"4"},
+			fileinfo{"notapid"}, fileinfo{"1"},
+		}, nil
+	}
+
+	readFile = func(path string) ([]byte, error) {
+		splits := strings.Split(path, "/")
+		if splits[len(splits)-1] != "stat" {
+			return nil, fmt.Errorf("not stat")
+		}
+		pid, err := strconv.Atoi(splits[len(splits)-2])
+		if err != nil {
+			return nil, err
+		}
+		parent := pid - 1
+		return []byte(fmt.Sprintf("%d na R %d", pid, parent)), nil
+	}
+
+	pidtree, err := newPIDTree("/proc")
+	if err != nil {
+		t.Fatalf("newPIDTree error: %v", err)
+	}
+
+	for pid, want := range map[int][]int{
+		1: {1, 2, 3, 4},
+		2: {2, 3, 4},
+	} {
+		have, err := pidtree.allChildren(pid)
+		if err != nil || !reflect.DeepEqual(want, have) {
+			t.Errorf("%d: want %#v, have %#v (%v)", pid, want, have, err)
+		}
+	}
+}

probe/spy.go

@@ -87,7 +87,6 @@ func addConnection(
 			for _, pm := range pms {
 				v, err := pm.Map(c.PID)
 				if err != nil {
-					log.Printf("spy processes: %s", err)
 					continue
 				}
 				md[pm.Key()] = v