diff --git a/.gitignore b/.gitignore index cb71a89a6..156a2334c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ coverage.html # Project specific scope.tar +scope_ui_build.tar app/app probe/probe docker/app diff --git a/Makefile b/Makefile index bc8341c4b..23e3d89f9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build client static dist test clean +.PHONY: all static test clean # If you can use Docker without being root, you can `make SUDO= ` SUDO=sudo @@ -9,22 +9,29 @@ PROBE_EXE=probe/probe FIXPROBE_EXE=experimental/fixprobe/fixprobe SCOPE_IMAGE=$(DOCKERHUB_USER)/scope SCOPE_EXPORT=scope.tar +SCOPE_UI_BUILD_EXPORT=scope_ui_build.tar +SCOPE_UI_BUILD_IMAGE=weave/scope-ui-build all: $(SCOPE_EXPORT) -dist: client static $(APP_EXE) $(PROBE_EXE) -client: - cd client && make build && rm -f dist/.htaccess +$(SCOPE_UI_BUILD_EXPORT): client/Dockerfile client/gulpfile.js client/package.json + docker build -t $(SCOPE_UI_BUILD_IMAGE) client + docker save $(SCOPE_UI_BUILD_IMAGE):latest > $@ -app/static.go: - go get github.com/mjibson/esc +client/dist/scripts/bundle.js: client/app/scripts/* + mkdir -p client/dist + docker run -ti -v $(shell pwd)/client/app:/home/weave/app \ + -v $(shell pwd)/client/dist:/home/weave/dist \ + $(SCOPE_UI_BUILD_IMAGE) + +static: client/dist/scripts/bundle.js esc -o app/static.go -prefix client/dist client/dist test: $(APP_EXE) $(FIXPROBE_EXE) # app and fixprobe needed for integration tests go test ./... -$(APP_EXE): app/*.go app/static.go report/*.go xfer/*.go +$(APP_EXE): app/*.go report/*.go xfer/*.go $(PROBE_EXE): probe/*.go report/*.go xfer/*.go $(APP_EXE) $(PROBE_EXE): @@ -34,11 +41,11 @@ $(APP_EXE) $(PROBE_EXE): $(FIXPROBE_EXE): cd experimental/fixprobe && go build -$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) docker/Dockerfile docker/entrypoint.sh +$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) docker/* cp $(APP_EXE) $(PROBE_EXE) docker/ $(SUDO) docker build -t $(SCOPE_IMAGE) docker/ $(SUDO) docker save $(SCOPE_IMAGE):latest > $@ clean: go clean ./... - rm -f $(SCOPE_EXPORT) app/static.go + rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_EXPORT) client/dist diff --git a/README.md b/README.md index bf689203e..07b12fc16 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,15 @@ To build a Docker container, make docker ``` +### The UI + +This repository contains a copy of the compiled UI. To build a fresh UI, run: + +``` +make scope_ui_build.tar +make static +``` + ## Running ### Manually diff --git a/circle.yml b/circle.yml index 069e0d9ed..9fdc6b662 100644 --- a/circle.yml +++ b/circle.yml @@ -10,19 +10,42 @@ machine: GOPATH: /home/ubuntu:$GOPATH SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope PATH: $PATH:$HOME/.local/bin + SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar dependencies: + cache_directories: + - "~/docker" + override: + - if [[ -e "$SCOPE_UI_BUILD" ]]; then + docker load -i $SCOPE_UI_BUILD; + else + make scope_ui_build.tar; + mkdir -p $(dirname "$SCOPE_UI_BUILD"); + mv scope_ui_build.tar $(dirname "$SCOPE_UI_BUILD"); + fi post: - go get github.com/golang/lint/golint - go get github.com/fzipp/gocyclo - go get github.com/mattn/goveralls + - go get github.com/mjibson/esc - mkdir -p $(dirname $SRCDIR) - cp -r $(pwd)/ $SRCDIR test: override: - cd $SRCDIR; ./bin/lint . + - cd $SRCDIR; make static - cd $SRCDIR; make - cd $SRCDIR; ./bin/test post: - - goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci + - goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true + - cd $SRCDIR; cp coverage.html $CIRCLE_ARTIFACTS + - cd $SRCDIR; cp scope.tar $CIRCLE_ARTIFACTS + +deployment: + hub: + branch: master + owner: weaveworks + commands: + - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + - docker push weaveworks/scope diff --git a/client/.dockerignore b/client/.dockerignore new file mode 100644 index 000000000..f06235c46 --- /dev/null +++ b/client/.dockerignore @@ -0,0 +1,2 @@ +node_modules +dist diff --git a/client/Dockerfile b/client/Dockerfile new file mode 100644 index 000000000..9ec183c6f --- /dev/null +++ b/client/Dockerfile @@ -0,0 +1,19 @@ +FROM mhart/alpine-node + +WORKDIR /home/weave + +# build tool +RUN npm install -g gulp + +# install app and build dependencies +ADD package.json /home/weave/ +RUN npm install + +ADD gulpfile.js /home/weave/ + +# run container via +# +# `docker run -v $GOPATH/src/github.com/weaveworks/scope/client:/app weaveworks/scope-build` +# +# after the container is run, bundled app should be in ./dist/ dir +CMD gulp build diff --git a/client/gulpfile.js b/client/gulpfile.js index 747cd0dcb..c443f086e 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -47,8 +47,6 @@ gulp.task('scripts', function() { gulp.task('html', ['styles', 'scripts'], function () { return gulp.src('app/*.html') .pipe($.preprocess()) - //.pipe($.useref.assets({searchPath: 'dist'})) - //.pipe($.useref()) .pipe(gulp.dest('dist')) .pipe($.size()) .pipe(livereload()); @@ -107,9 +105,7 @@ gulp.task('connect', function () { }); }); -gulp.task('serve', ['connect', 'styles', 'scripts', 'fonts'], function () { - //require('opn')('http://localhost:9000'); -}); +gulp.task('serve', ['connect', 'styles', 'scripts', 'fonts']); gulp.task('watch', ['serve'], function () { livereload.listen(); diff --git a/client/package.json b/client/package.json index a7943cf45..cb3b71d28 100644 --- a/client/package.json +++ b/client/package.json @@ -1,53 +1,52 @@ { - "name": "scope-webapp", - "version": "0.2.0", + "name": "weave-scope", + "version": "1.2.0", + "description": "SPA JS app for Weave Scope visualising the application network.", + "repository": "weaveworks/scope", + "license": "Apache-2.0", + "private": true, "dependencies": { - "d3": "^3.5.3", - "dagre": "^0.7.1", - "flux": "^2.0.1", + "d3": "^3.5.5", + "dagre": "^0.7.2", + "flux": "^2.0.3", "font-awesome": "^4.3.0", "keymirror": "^0.1.1", - "lodash": "~3.0.1", + "lodash": "~3.8.0", "material-ui": "^0.7.5", "object-assign": "^2.0.0", - "page": "^1.6.0", - "react": "^0.13.2", + "page": "^1.6.3", + "react": "^0.13.3", + "react-tap-event-plugin": "^0.1.6", "react-tween-state": "0.0.5", "reqwest": "~1.1.5" }, "devDependencies": { - "browserify": "^8.1.3", - "connect": "^3.3.4", + "browserify": "^10.2.0", "del": "^1.1.1", - "gulp": "^3.8.10", - "gulp-autoprefixer": "^2.1.0", - "gulp-cache": "^0.2.4", - "gulp-clean": "^0.3.1", + "gulp": "^3.8.11", + "gulp-autoprefixer": "^2.3.0", "gulp-connect": "^2.2.0", "gulp-csso": "^1.0.0", - "gulp-filter": "^2.0.0", + "gulp-filter": "^2.0.2", "gulp-flatten": "^0.0.4", "gulp-if": "^1.2.5", - "gulp-imagemin": "^2.1.0", - "gulp-jshint": "^1.9.2", + "gulp-jshint": "^1.10.0", "gulp-less": "^3.0.3", "gulp-livereload": "^3.8.0", - "gulp-load-plugins": "^0.8.0", + "gulp-load-plugins": "^0.10.0", "gulp-preprocess": "^1.2.0", - "gulp-size": "^1.2.0", - "gulp-sourcemaps": "^1.3.0", - "gulp-uglify": "^1.1.0", - "gulp-useref": "^1.1.1", - "gulp-util": "^3.0.3", - "jshint-stylish": "^1.0.0", - "opn": "^1.0.1", - "proxy-middleware": "^0.11.0", - "reactify": "^1.1.0", + "gulp-size": "^1.2.1", + "gulp-sourcemaps": "^1.5.2", + "gulp-uglify": "^1.2.0", + "gulp-util": "^3.0.4", + "jshint-stylish": "^1.0.2", + "proxy-middleware": "^0.11.1", + "reactify": "^1.1.1", "vinyl-buffer": "^1.0.0", - "vinyl-source-stream": "^1.0.0" + "vinyl-source-stream": "^1.1.0" }, "scripts": { - "start": "npm install && gulp serve" + "start": "gulp" }, "engines": { "node": ">=0.10.0" diff --git a/docker/run-probe b/docker/run-probe index fc9ec1bbb..23251387a 100755 --- a/docker/run-probe +++ b/docker/run-probe @@ -1,3 +1,3 @@ #!/bin/sh -exec /home/weave/probe +exec /home/weave/probe -proc.root=/hostproc diff --git a/probe/docker_process_mapper.go b/probe/docker_process_mapper.go new file mode 100644 index 000000000..93ef81e15 --- /dev/null +++ b/probe/docker_process_mapper.go @@ -0,0 +1,128 @@ +package main + +import ( + "fmt" + "log" + "sync" + "time" + + docker "github.com/fsouza/go-dockerclient" +) + +type dockerMapper struct { + sync.RWMutex + d map[int]*docker.Container + procRoot string +} + +func newDockerMapper(procRoot string, interval time.Duration) *dockerMapper { + m := dockerMapper{ + procRoot: procRoot, + d: map[int]*docker.Container{}, + } + m.update() + go m.loop(interval) + return &m +} + +func (m *dockerMapper) loop(d time.Duration) { + for range time.Tick(d) { + m.update() + } +} + +// for mocking +type dockerClient interface { + ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error) + InspectContainer(string) (*docker.Container, error) +} + +func newRealDockerClient(endpoint string) (dockerClient, error) { + return docker.NewClient(endpoint) +} + +var ( + newDockerClient = newRealDockerClient + newPIDTreeStub = newPIDTree +) + +func (m *dockerMapper) update() { + pidTree, err := newPIDTreeStub(m.procRoot) + if err != nil { + log.Printf("docker mapper: %s", err) + return + } + + endpoint := "unix:///var/run/docker.sock" + client, err := newDockerClient(endpoint) + if err != nil { + log.Printf("docker mapper: %s", err) + return + } + + containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) + if err != nil { + log.Printf("docker mapper: %s", err) + return + } + + pmap := map[int]*docker.Container{} + for _, container := range containers { + info, err := client.InspectContainer(container.ID) + if err != nil { + log.Printf("docker mapper: %s", err) + continue + } + + if !info.State.Running { + continue + } + + pids, err := pidTree.allChildren(info.State.Pid) + if err != nil { + log.Printf("docker mapper: %s", err) + continue + } + for _, pid := range pids { + pmap[pid] = info + } + } + + m.Lock() + m.d = pmap + m.Unlock() +} + +type dockerIDMapper struct { + *dockerMapper +} + +func (m dockerIDMapper) Key() string { return "docker_id" } +func (m dockerIDMapper) Map(pid uint) (string, error) { + m.RLock() + container, ok := m.d[int(pid)] + m.RUnlock() + + if !ok { + return "", fmt.Errorf("no container found for PID %d", pid) + } + + return container.ID, nil +} + +type dockerNameMapper struct { + *dockerMapper +} + +func (m dockerNameMapper) Key() string { return "docker_name" } +func (m dockerNameMapper) Map(pid uint) (string, error) { + m.RLock() + container, ok := m.d[int(pid)] + m.RUnlock() + + if !ok { + return "", fmt.Errorf("no container found for PID %d", pid) + } + + return container.Name, nil +} diff --git a/probe/docker_process_mapper_test.go b/probe/docker_process_mapper_test.go new file mode 100644 index 000000000..0059d7804 --- /dev/null +++ b/probe/docker_process_mapper_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "testing" + "time" + + docker "github.com/fsouza/go-dockerclient" +) + +type mockDockerClient struct { + containers []docker.APIContainers + containerInfo map[string]*docker.Container +} + +func (m mockDockerClient) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) { + return m.containers, nil +} + +func (m mockDockerClient) InspectContainer(id string) (*docker.Container, error) { + return m.containerInfo[id], nil +} + +func TestDockerProcessMapper(t *testing.T) { + oldPIDTreeStub, oldDockerClientStub := newPIDTreeStub, newDockerClient + defer func() { + newPIDTreeStub = oldPIDTreeStub + newDockerClient = oldDockerClientStub + }() + + newPIDTreeStub = func(procRoot string) (*pidTree, error) { + pid1 := &process{pid: 1} + pid2 := &process{pid: 2, ppid: 1, parent: pid1} + pid1.children = []*process{pid2} + + return &pidTree{ + processes: map[int]*process{ + 1: pid1, 2: pid2, + }, + }, nil + } + + newDockerClient = func(endpoint string) (dockerClient, error) { + return mockDockerClient{ + containers: []docker.APIContainers{{ID: "foo"}}, + containerInfo: map[string]*docker.Container{ + "foo": { + ID: "foo", + Name: "bar", + State: docker.State{Pid: 1, Running: true}, + }, + }, + }, nil + } + + dockerMapper := newDockerMapper("/proc", 10*time.Second) + dockerIDMapper := dockerIDMapper{dockerMapper} + dockerNameMapper := dockerNameMapper{dockerMapper} + + for pid, want := range map[uint]struct{ id, name string }{ + 1: {"foo", "bar"}, + 2: {"foo", "bar"}, + } { + haveID, err := dockerIDMapper.Map(pid) + if err != nil || want.id != haveID { + t.Errorf("%d: want %q, have %q (%v)", pid, want.id, haveID, err) + } + haveName, err := dockerNameMapper.Map(pid) + if err != nil || want.name != haveName { + t.Errorf("%d: want %q, have %q (%v)", pid, want.name, haveName, err) + } + } +} diff --git a/probe/main.go b/probe/main.go index ba45114d3..db10698e6 100644 --- a/probe/main.go +++ b/probe/main.go @@ -27,7 +27,9 @@ func main() { prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)") spyProcs = flag.Bool("processes", true, "report processes (needs root)") cgroupsRoot = flag.String("cgroups.root", "", "if provided, enrich -processes with cgroup names from this root (e.g. /mnt/cgroups)") - cgroupsUpdate = flag.Duration("cgroups.update", 10*time.Second, "how often to update cgroup names") + cgroupsInterval = flag.Duration("cgroups.interval", 10*time.Second, "how often to update cgroup names") + dockerMapper = flag.Bool("docker", true, "collect docker-related attributes for processes.") + dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update docker container info") procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem") ) flag.Parse() @@ -64,12 +66,17 @@ func main() { if *cgroupsRoot != "" { if fi, err := os.Stat(*cgroupsRoot); err == nil && fi.IsDir() { log.Printf("enriching -processes with cgroup names from %s", *cgroupsRoot) - pms = append(pms, newCgroupMapper(*cgroupsRoot, *cgroupsUpdate)) + pms = append(pms, newCgroupMapper(*cgroupsRoot, *cgroupsInterval)) } else { log.Printf("-cgroups.root=%s: %v", *cgroupsRoot, err) } } + if *dockerMapper { + docker := newDockerMapper(*procRoot, *dockerInterval) + pms = append(pms, &dockerIDMapper{docker}, &dockerNameMapper{docker}) + } + log.Printf("listening on %s", *listen) go func() { diff --git a/probe/pidtree.go b/probe/pidtree.go new file mode 100644 index 000000000..c9a62b6c2 --- /dev/null +++ b/probe/pidtree.go @@ -0,0 +1,86 @@ +package main + +import ( + "fmt" + "io/ioutil" + "path" + "strconv" + "strings" +) + +type pidTree struct { + processes map[int]*process +} + +type process struct { + pid, ppid int + parent *process + children []*process +} + +// Hooks for mocking +var ( + readDir = ioutil.ReadDir + readFile = ioutil.ReadFile +) + +func newPIDTree(procRoot string) (*pidTree, error) { + dirEntries, err := readDir(procRoot) + if err != nil { + return nil, err + } + + pt := pidTree{processes: map[int]*process{}} + for _, dirEntry := range dirEntries { + pid, err := strconv.Atoi(dirEntry.Name()) + if err != nil { + continue + } + + stat, err := readFile(path.Join(procRoot, dirEntry.Name(), "stat")) + if err != nil { + continue + } + + splits := strings.Split(string(stat), " ") + ppid, err := strconv.Atoi(splits[3]) + if err != nil { + return nil, err + } + + pt.processes[pid] = &process{pid: pid, ppid: ppid} + } + + for _, child := range pt.processes { + parent, ok := pt.processes[child.ppid] + if !ok { + // This can happen as listing proc is not a consistent snapshot + continue + } + child.parent = parent + parent.children = append(parent.children, child) + } + + return &pt, nil +} + +// allChildren returns a flattened list of child pids including the given pid +func (pt *pidTree) allChildren(pid int) ([]int, error) { + proc, ok := pt.processes[pid] + if !ok { + return []int{}, fmt.Errorf("PID %d not found", pid) + } + + var result []int + + var f func(*process) + f = func(p *process) { + result = append(result, p.pid) + for _, child := range p.children { + f(child) + } + } + + f(proc) + return result, nil +} diff --git a/probe/pidtree_test.go b/probe/pidtree_test.go new file mode 100644 index 000000000..f83fbd497 --- /dev/null +++ b/probe/pidtree_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + "os" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +type fileinfo struct { + name string +} + +func (f fileinfo) Name() string { return f.name } +func (f fileinfo) Size() int64 { return 0 } +func (f fileinfo) Mode() os.FileMode { return 0 } +func (f fileinfo) ModTime() time.Time { return time.Now() } +func (f fileinfo) IsDir() bool { return true } +func (f fileinfo) Sys() interface{} { return nil } + +func TestPIDTree(t *testing.T) { + oldReadDir, oldReadFile := readDir, readFile + defer func() { + readDir = oldReadDir + readFile = oldReadFile + }() + + readDir = func(path string) ([]os.FileInfo, error) { + return []os.FileInfo{ + fileinfo{"3"}, fileinfo{"2"}, fileinfo{"4"}, + fileinfo{"notapid"}, fileinfo{"1"}, + }, nil + } + + readFile = func(path string) ([]byte, error) { + splits := strings.Split(path, "/") + if splits[len(splits)-1] != "stat" { + return nil, fmt.Errorf("not stat") + } + pid, err := strconv.Atoi(splits[len(splits)-2]) + if err != nil { + return nil, err + } + parent := pid - 1 + return []byte(fmt.Sprintf("%d na R %d", pid, parent)), nil + } + + pidtree, err := newPIDTree("/proc") + if err != nil { + t.Fatalf("newPIDTree error: %v", err) + } + + for pid, want := range map[int][]int{ + 1: {1, 2, 3, 4}, + 2: {2, 3, 4}, + } { + have, err := pidtree.allChildren(pid) + if err != nil || !reflect.DeepEqual(want, have) { + t.Errorf("%d: want %#v, have %#v (%v)", pid, want, have, err) + } + } +} diff --git a/probe/spy.go b/probe/spy.go index 38c80a32e..dfac75e74 100644 --- a/probe/spy.go +++ b/probe/spy.go @@ -87,7 +87,6 @@ func addConnection( for _, pm := range pms { v, err := pm.Map(c.PID) if err != nil { - log.Printf("spy processes: %s", err) continue } md[pm.Key()] = v