Merge pull request #503 from weaveworks/shutdown

Shutdown behavior

.gitignore

@@ -42,6 +42,7 @@ docker/scope-app
 docker/scope-probe
 docker/docker*
 docker/weave
+docker/runsvinit
 experimental/bridge/bridge
 experimental/demoprobe/demoprobe
 experimental/fixprobe/fixprobe

Makefile

@@ -17,6 +17,7 @@ SCOPE_VERSION=$(shell git rev-parse --short HEAD)
 DOCKER_VERSION=1.3.1
 DOCKER_DISTRIB=docker/docker-$(DOCKER_VERSION).tgz
 DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(DOCKER_VERSION).tgz
+RUNSVINIT=docker/runsvinit
 RM=--rm
 
 all: $(SCOPE_EXPORT)
@@ -28,13 +29,16 @@ docker/weave:
 	curl -L git.io/weave -o docker/weave
 	chmod u+x docker/weave
 
-$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) $(DOCKER_DISTRIB) docker/weave docker/entrypoint.sh docker/Dockerfile docker/run-app docker/run-probe
+$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) $(DOCKER_DISTRIB) docker/weave $(RUNSVINIT) docker/Dockerfile docker/run-app docker/run-probe
 	@if [ -z '$(DOCKER_SQUASH)' ] ; then echo "Please install docker-squash by running 'make deps' (and make sure GOPATH/bin is in your PATH)." && exit 1 ; fi
 	cp $(APP_EXE) $(PROBE_EXE) docker/
 	cp $(DOCKER_DISTRIB) docker/docker.tgz
 	$(SUDO) docker build -t $(SCOPE_IMAGE) docker/
 	$(SUDO) docker save $(SCOPE_IMAGE):latest | sudo $(DOCKER_SQUASH) -t $(SCOPE_IMAGE) | tee $@ | $(SUDO) docker load
 
+$(RUNSVINIT): vendor/runsvinit/*.go
+	go build -o $@ github.com/weaveworks/scope/vendor/runsvinit
+
 $(APP_EXE): app/*.go render/*.go report/*.go xfer/*.go
 
 $(PROBE_EXE): probe/*.go probe/docker/*.go probe/endpoint/*.go probe/host/*.go probe/process/*.go probe/overlay/*.go report/*.go xfer/*.go

app/main.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
+	"strings"
 	"syscall"
 	"time"
 
@@ -28,6 +29,7 @@ func main() {
 	var (
 		window       = flag.Duration("window", 15*time.Second, "window")
 		listen       = flag.String("http.address", ":"+strconv.Itoa(xfer.AppPort), "webserver listen address")
+		logPrefix    = flag.String("log.prefix", "<app>", "prefix for each log line")
 		printVersion = flag.Bool("version", false, "print version number and exit")
 	)
 	flag.Parse()
@@ -37,20 +39,24 @@ func main() {
 		return
 	}
 
+	if !strings.HasSuffix(*logPrefix, " ") {
+		*logPrefix += " "
+	}
+	log.SetPrefix(*logPrefix)
+
+	defer log.Print("app exiting")
+
 	rand.Seed(time.Now().UnixNano())
 	uniqueID = strconv.FormatInt(rand.Int63(), 16)
 	log.Printf("app starting, version %s, ID %s", version, uniqueID)
 
 	c := xfer.NewCollector(*window)
 	http.Handle("/", Router(c))
-	irq := interrupt()
 	go func() {
 		log.Printf("listening on %s", *listen)
 		log.Print(http.ListenAndServe(*listen, nil))
-		irq <- syscall.SIGINT
 	}()
-	<-irq
-	log.Printf("shutting down")
+	log.Printf("%s", <-interrupt())
 }
 
 func interrupt() chan os.Signal {

docker/Dockerfile

@@ -7,7 +7,7 @@ RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositori
 	rm -rf /var/cache/apk/*
 ADD ./docker.tgz /
 ADD ./weave /usr/bin/
-COPY ./scope-app ./scope-probe ./entrypoint.sh /home/weave/
+COPY ./scope-app ./scope-probe ./runsvinit ./entrypoint.sh /home/weave/
 COPY ./run-app /etc/service/app/run
 COPY ./run-probe /etc/service/probe/run
 EXPOSE 4040

docker/entrypoint.sh

@@ -89,4 +89,5 @@ fi
 MANUAL_APPS=$@
 echo "$MANUAL_APPS" >>/etc/weave/apps
 
-exec /sbin/runsvdir /etc/service
+exec /home/weave/runsvinit
+

integration/110_shutdown_test.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Check scope exits cleanly within 10 seconds"
+
+scope_on $HOST1 launch
+scope_on $HOST1 stop
+
+sleep 10
+
+assert_raises "docker_on $HOST1 logs weavescope 2>&1 | grep 'app exiting'"
+assert_raises "docker_on $HOST1 logs weavescope 2>&1 | grep 'probe exiting'"
+assert_raises "docker_on $HOST1 inspect --format='{{.State.Running}}' weavescope" "false"
+
+end_suite

probe/docker/reporter.go

@@ -15,14 +15,14 @@ const (
 // Reporter generate Reports containing Container and ContainerImage topologies
 type Reporter struct {
 	registry Registry
-	scope    string
+	hostID   string
 }
 
 // NewReporter makes a new Reporter
-func NewReporter(registry Registry, scope string) *Reporter {
+func NewReporter(registry Registry, hostID string) *Reporter {
 	return &Reporter{
 		registry: registry,
-		scope:    scope,
+		hostID:   hostID,
 	}
 }
 
@@ -38,7 +38,7 @@ func (r *Reporter) containerTopology() report.Topology {
 	result := report.MakeTopology()
 
 	r.registry.WalkContainers(func(c Container) {
-		nodeID := report.MakeContainerNodeID(r.scope, c.ID())
+		nodeID := report.MakeContainerNodeID(r.hostID, c.ID())
 		result.AddNode(nodeID, c.GetNode())
 	})
 
@@ -58,7 +58,7 @@ func (r *Reporter) containerImageTopology() report.Topology {
 			nmd.Metadata[ImageName] = image.RepoTags[0]
 		}
 
-		nodeID := report.MakeContainerNodeID(r.scope, image.ID)
+		nodeID := report.MakeContainerNodeID(r.hostID, image.ID)
 		result.AddNode(nodeID, nmd)
 	})
 

probe/main.go

@@ -13,7 +13,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/weaveworks/procspy"
 	"github.com/weaveworks/scope/probe/docker"
 	"github.com/weaveworks/scope/probe/endpoint"
 	"github.com/weaveworks/scope/probe/host"
@@ -41,6 +40,7 @@ func main() {
 		procRoot           = flag.String("proc.root", "/proc", "location of the proc filesystem")
 		printVersion       = flag.Bool("version", false, "print version number and exit")
 		useConntrack       = flag.Bool("conntrack", true, "also use conntrack to track connections")
+		logPrefix          = flag.String("log.prefix", "<probe>", "prefix for each log line")
 	)
 	flag.Parse()
 
@@ -49,48 +49,23 @@ func main() {
 		return
 	}
 
-	var (
-		hostName = hostname()
-		hostID   = hostName // TODO: we should sanitize the hostname
-		probeID  = hostName // TODO: does this need to be a random string instead?
-	)
-	log.Printf("probe starting, version %s, ID %s", version, probeID)
-
-	if len(flag.Args()) > 0 {
-		targets = flag.Args()
+	if !strings.HasSuffix(*logPrefix, " ") {
+		*logPrefix += " "
 	}
-	log.Printf("publishing to: %s", strings.Join(targets, ", "))
+	log.SetPrefix(*logPrefix)
 
-	procspy.SetProcRoot(*procRoot)
-
-	if *httpListen != "" {
-		log.Printf("profiling data being exported to %s", *httpListen)
-		log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
-		if *prometheusEndpoint != "" {
-			log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
-			http.Handle(*prometheusEndpoint, makePrometheusHandler())
-		}
-		go func() {
-			err := http.ListenAndServe(*httpListen, nil)
-			log.Print(err)
-		}()
-	}
+	defer log.Print("probe exiting")
 
 	if *spyProcs && os.Getegid() != 0 {
-		log.Printf("warning: process reporting enabled, but that requires root to find everything")
+		log.Printf("warning: -process=true, but that requires root to find everything")
 	}
 
-	factory := func(endpoint string) (string, xfer.Publisher, error) {
-		id, publisher, err := xfer.NewHTTPPublisher(endpoint, *token, probeID)
-		if err != nil {
-			return "", nil, err
-		}
-		return id, xfer.NewBackgroundPublisher(publisher), nil
-	}
-	publishers := xfer.NewMultiPublisher(factory)
-	defer publishers.Stop()
-	resolver := newStaticResolver(targets, publishers.Set)
-	defer resolver.Stop()
+	var (
+		hostName = hostname()
+		hostID   = hostName // TODO(pb): we should sanitize the hostname
+		probeID  = hostName // TODO(pb): does this need to be a random string instead?
+	)
+	log.Printf("probe starting, version %s, ID %s", version, probeID)
 
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
@@ -104,32 +79,59 @@ func main() {
 		}
 	}
 
-	var (
-		endpointReporter = endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
-		processCache     = process.NewCachingWalker(process.NewWalker(*procRoot))
-		tickers          = []Ticker{processCache}
-		reporters        = []Reporter{
-			endpointReporter,
-			host.NewReporter(hostID, hostName, localNets),
-			process.NewReporter(processCache, hostID),
+	if len(flag.Args()) > 0 {
+		targets = flag.Args()
+	}
+	log.Printf("publishing to: %s", strings.Join(targets, ", "))
+
+	factory := func(endpoint string) (string, xfer.Publisher, error) {
+		id, publisher, err := xfer.NewHTTPPublisher(endpoint, *token, probeID)
+		if err != nil {
+			return "", nil, err
 		}
-		taggers = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
-	)
+		return id, xfer.NewBackgroundPublisher(publisher), nil
+	}
+
+	publishers := xfer.NewMultiPublisher(factory)
+	defer publishers.Stop()
+
+	resolver := newStaticResolver(targets, publishers.Set)
+	defer resolver.Stop()
+
+	endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack)
 	defer endpointReporter.Stop()
 
-	if *dockerEnabled {
+	processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
+
+	var (
+		tickers   = []Ticker{processCache}
+		reporters = []Reporter{endpointReporter, host.NewReporter(hostID, hostName, localNets), process.NewReporter(processCache, hostID)}
+		taggers   = []Tagger{newTopologyTagger(), host.NewTagger(hostID)}
+	)
+
+	dockerTagger, dockerReporter, dockerRegistry := func() (*docker.Tagger, *docker.Reporter, docker.Registry) {
+		if !*dockerEnabled {
+			return nil, nil, nil
+		}
 		if err := report.AddLocalBridge(*dockerBridge); err != nil {
-			log.Fatalf("failed to get docker bridge address: %v", err)
+			log.Printf("Docker: problem with bridge %s: %v", *dockerBridge, err)
+			return nil, nil, nil
 		}
-
-		dockerRegistry, err := docker.NewRegistry(*dockerInterval)
+		registry, err := docker.NewRegistry(*dockerInterval)
 		if err != nil {
-			log.Fatalf("failed to start docker registry: %v", err)
+			log.Printf("Docker: failed to start registry: %v", err)
+			return nil, nil, nil
 		}
+		return docker.NewTagger(registry, processCache), docker.NewReporter(registry, hostID), registry
+	}()
+	if dockerTagger != nil {
+		taggers = append(taggers, dockerTagger)
+	}
+	if dockerReporter != nil {
+		reporters = append(reporters, dockerReporter)
+	}
+	if dockerRegistry != nil {
 		defer dockerRegistry.Stop()
-
-		taggers = append(taggers, docker.NewTagger(dockerRegistry, processCache))
-		reporters = append(reporters, docker.NewReporter(dockerRegistry, hostID))
 	}
 
 	if *weaveRouterAddr != "" {
@@ -139,16 +141,29 @@ func main() {
 		reporters = append(reporters, weave)
 	}
 
-	quit := make(chan struct{})
-	defer close(quit)
+	if *httpListen != "" {
+		go func() {
+			log.Printf("Profiling data being exported to %s", *httpListen)
+			log.Printf("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
+			if *prometheusEndpoint != "" {
+				log.Printf("exposing Prometheus endpoint at %s%s", *httpListen, *prometheusEndpoint)
+				http.Handle(*prometheusEndpoint, makePrometheusHandler())
+			}
+			log.Printf("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
+		}()
+	}
+
+	quit, done := make(chan struct{}), make(chan struct{})
+	defer func() { <-done }() // second, wait for the main loop to be killed
+	defer close(quit)         // first, kill the main loop
 	go func() {
+		defer close(done)
 		var (
 			pubTick = time.Tick(*publishInterval)
 			spyTick = time.Tick(*spyInterval)
 			r       = report.MakeReport()
 			p       = xfer.NewReportPublisher(publishers)
 		)
-
 		for {
 			select {
 			case <-pubTick:
@@ -161,16 +176,13 @@ func main() {
 
 			case <-spyTick:
 				start := time.Now()
-
 				for _, ticker := range tickers {
 					if err := ticker.Tick(); err != nil {
 						log.Printf("error doing ticker: %v", err)
 					}
 				}
-
 				r = r.Merge(doReport(reporters))
 				r = Apply(r, taggers)
-
 				if took := time.Since(start); took > *spyInterval {
 					log.Printf("report generation took too long (%s)", took)
 				}
@@ -180,6 +192,7 @@ func main() {
 			}
 		}
 	}()
+
 	log.Printf("%s", <-interrupt())
 }
 
@@ -203,7 +216,7 @@ func doReport(reporters []Reporter) report.Report {
 	return result
 }
 
-func interrupt() chan os.Signal {
+func interrupt() <-chan os.Signal {
 	c := make(chan os.Signal)
 	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
 	return c

scope

@@ -164,6 +164,8 @@ case "$COMMAND" in
             WEAVESCOPE_DNS_ARGS="$WEAVESCOPE_DNS_ARGS --dns $DOCKER_BRIDGE_IP --searchpath $DOMAINNAME"
         fi
 
+        docker rm -f $SCOPE_CONTAINER_NAME >/dev/null 2>&1 || true
+
         CONTAINER=$(docker run --privileged -d --name=$SCOPE_CONTAINER_NAME --net=host --pid=host \
             -v /var/run/docker.sock:/var/run/docker.sock \
             $WEAVESCOPE_DOCKER_ARGS $SCOPE_IMAGE $WEAVESCOPE_DNS_ARGS $SCOPE_ARGS --probe.docker true "$@")
@@ -192,7 +194,6 @@ case "$COMMAND" in
         if ! docker stop $SCOPE_CONTAINER_NAME >/dev/null 2>&1 ; then
             echo "Weave Scope is not running." >&2
         fi
-        docker rm -f $SCOPE_CONTAINER_NAME >/dev/null 2>&1 || true
         ;;
 
     *)

vendor/runsvinit/.gitignore

@@ -0,0 +1,31 @@
+runsvinit
+runsvinit-*-*
+examples/runsvinit-linux-amd64*
+zombietest/zombie
+zombietest/runsvinit
+*.uptodate
+
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

vendor/runsvinit/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Peter Bourgon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

vendor/runsvinit/README.md

@@ -0,0 +1,43 @@
+# runsvinit
+
+If you have a Docker container that's a collection of runit-supervised daemons,
+this process is suitable for use as the ENTRYPOINT.
+See [the example](https://github.com/peterbourgon/runsvinit/tree/master/example).
+
+**Why not just exec runsvdir?**
+
+`docker stop` issues SIGTERM (or, in a future version of Docker, perhaps another custom signal)
+but if runsvdir receives a signal,
+it doesn't wait for its supervised processes to exit before returning.
+If you don't care about graceful shutdown of your daemons, no problem, you don't need this tool.
+
+**Why not wrap runsvdir in a simple shell script?**
+
+This works great:
+
+```sh
+#!/bin/sh
+
+sv_stop() {
+	for s in $(ls -d /etc/service/*)
+	do
+		/sbin/sv stop $s
+	done
+}
+
+trap "sv_stop; exit" SIGTERM
+/sbin/runsvdir /etc/service &
+wait
+```
+
+...except it doesn't [reap orphaned child processes](https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/)
+and is therefore unsuitable for being PID 1.
+
+**Why not use my_init from [phusion/baseimage-docker](https://github.com/phusion/baseimage-docker)?**
+
+That works great — if you're willing to add python3 to your Docker images :)
+
+**So this is just a stripped-down my_init in Go?**
+
+Basically, yes.
+

vendor/runsvinit/circle.yml

@@ -0,0 +1,8 @@
+machine:
+  services:
+    - docker
+
+test:
+  override:
+    - go test -v
+    - make CIRCLECI=true SUDO=sudo RM= -C zombietest

vendor/runsvinit/example/Dockerfile

@@ -0,0 +1,15 @@
+FROM alpine:latest
+RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
+    apk add --update runit && \
+    rm -rf /var/cache/apk/*
+
+ADD foo /
+RUN mkdir -p /etc/service/foo
+ADD run-foo /etc/service/foo/run
+
+ADD bar /
+RUN mkdir -p /etc/service/bar
+ADD run-bar /etc/service/bar/run
+
+ADD /runsvinit /
+ENTRYPOINT ["/runsvinit"]

vendor/runsvinit/example/Makefile

@@ -0,0 +1,10 @@
+.PHONY: docker
+docker: runsvinit
+	docker build -t runsvinit-example .
+
+runsvinit: runsvinit-linux-amd64.tgz
+	tar zxf $<
+
+runsvinit-linux-amd64.tgz:
+	wget --quiet https://github.com/peterbourgon/runsvinit/releases/download/v2.0.0/runsvinit-linux-amd64.tgz
+

vendor/runsvinit/example/README.md

@@ -0,0 +1,14 @@
+# runsvinit example
+
+## Build the Docker container
+
+`make`
+
+## Run the Docker container
+
+`docker run --name example runsvinit-example:latest`
+
+## Test it works
+
+CTRL-C, or `docker stop example`.
+

vendor/runsvinit/example/bar

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+handle() {
+	echo "got signal"
+	exit
+}
+
+trap handle SIGINT
+
+while true
+do
+	echo bar: `date`
+	sleep 1
+done
+

vendor/runsvinit/example/foo

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+handle() {
+	echo "got signal"
+	exit
+}
+
+trap "handle" SIGINT
+
+for i in `seq 1 5`
+do
+	echo foo: $i
+	sleep 1
+done
+
+echo foo: terminating!
+

vendor/runsvinit/example/run-bar

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec /bar

vendor/runsvinit/example/run-foo

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec /foo

vendor/runsvinit/main.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+const etcService = "/etc/service"
+
+func main() {
+	reap := flag.Bool("reap", true, "reap orphan children")
+	flag.Parse()
+
+	log.SetFlags(0)
+
+	runsvdir, err := exec.LookPath("runsvdir")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	sv, err := exec.LookPath("sv")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if fi, err := os.Stat(etcService); err != nil {
+		log.Fatal(err)
+	} else if !fi.IsDir() {
+		log.Fatalf("%s is not a directory", etcService)
+	}
+
+	if pid := os.Getpid(); pid != 1 {
+		log.Printf("warning: I'm not PID 1, I'm PID %d", pid)
+	}
+
+	if *reap {
+		log.Print("reaping zombies")
+		go reapLoop()
+	} else {
+		log.Print("NOT reaping zombies")
+	}
+
+	supervisor := cmd(runsvdir, etcService)
+	if err := supervisor.Start(); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("%s started", runsvdir)
+
+	go shutdown(sv, supervisor.Process)
+
+	if err := supervisor.Wait(); err != nil {
+		log.Printf("%s exited with error: %v", runsvdir, err)
+	} else {
+		log.Printf("%s exited cleanly", runsvdir)
+	}
+}
+
+// From https://github.com/ramr/go-reaper/blob/master/reaper.go
+func reapLoop() {
+	c := make(chan os.Signal)
+	signal.Notify(c, syscall.SIGCHLD)
+	for range c {
+		reapChildren()
+	}
+}
+
+func reapChildren() {
+	for {
+		var (
+			ws  syscall.WaitStatus
+			pid int
+			err error
+		)
+		for {
+			pid, err = syscall.Wait4(-1, &ws, 0, nil)
+			if err != syscall.EINTR {
+				break
+			}
+		}
+		if err == syscall.ECHILD {
+			return // done
+		}
+		log.Printf("reaped child process %d (%+v)", pid, ws)
+	}
+}
+
+type signaler interface {
+	Signal(os.Signal) error
+}
+
+func shutdown(sv string, s signaler) {
+	c := make(chan os.Signal)
+	signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
+	sig := <-c
+	log.Printf("received %s", sig)
+
+	matches, err := filepath.Glob(filepath.Join(etcService, "*"))
+	if err != nil {
+		log.Printf("when shutting down services: %v", err)
+		return
+	}
+
+	var stopped []string
+	for _, match := range matches {
+		fi, err := os.Stat(match)
+		if err != nil {
+			log.Printf("%s: %v", match, err)
+			continue
+		}
+		if !fi.IsDir() {
+			log.Printf("%s: not a directory", match)
+			continue
+		}
+		service := filepath.Base(match)
+		stop := cmd(sv, "stop", service)
+		if err := stop.Run(); err != nil {
+			log.Printf("%s: %v", strings.Join(stop.Args, " "), err)
+			continue
+		}
+		stopped = append(stopped, service)
+	}
+
+	log.Printf("stopped %d: %s", len(stopped), strings.Join(stopped, ", "))
+	log.Printf("stopping supervisor with signal %s...", sig)
+	if err := s.Signal(sig); err != nil {
+		log.Print(err)
+	}
+	log.Printf("shutdown handler exiting")
+}
+
+func cmd(path string, args ...string) *exec.Cmd {
+	return &exec.Cmd{
+		Path:   path,
+		Args:   append([]string{path}, args...),
+		Env:    os.Environ(),
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}

vendor/runsvinit/zombietest/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:latest
+RUN echo "http://dl-4.alpinelinux.org/alpine/edge/testing" >>/etc/apk/repositories && \
+    apk add --update runit && \
+    rm -rf /var/cache/apk/*
+
+COPY zombie /
+RUN mkdir -p /etc/service/zombie
+COPY run-zombie /etc/service/zombie/run
+
+COPY /runsvinit /
+

vendor/runsvinit/zombietest/Makefile

@@ -0,0 +1,27 @@
+GO?=go
+SUDO?=
+RM?=--rm
+
+.PHONY: test clean
+
+test: .test.uptodate
+	./test.bash
+
+.test.uptodate: runsvinit zombie run-zombie Dockerfile
+	$(SUDO) docker build -t zombietest .
+	touch $@
+
+runsvinit: ../*.go
+	env GOOS=linux GOARCH=amd64 $(GO) build -o $@ github.com/peterbourgon/runsvinit
+
+zombie: .build.uptodate
+	$(SUDO) docker run $(RM) -v $(shell pwd):/mount zombietest-build cc -Wall -Werror -o /mount/zombie /zombie.c
+
+.build.uptodate: build/zombie.c build/Dockerfile
+	$(SUDO) docker build -t zombietest-build build/
+	touch $@
+
+clean:
+	rm -rf .test.uptodate .build.uptodate runsvinit zombie
+	$(SUDO) docker stop zombietest zombietest-build >/dev/null 2>&1 || true
+	$(SUDO) docker rm zombietest zombietest-build >/dev/null 2>&1 || true

vendor/runsvinit/zombietest/README.md

@@ -0,0 +1,24 @@
+# zombietest
+
+This directory contains an integration test to prove runsvinit is actually
+reaping zombies. `make` builds and executes the test as follows:
+
+1. We produce a linux/amd64 runsvinit binary by setting GOOS/GOARCH and
+   invoking the Go compiler. Requires Go 1.5, or Go 1.4 built with the
+   appropriate cross-compile options.
+
+2. The build/zombie.c program spawns five zombies and exits. We compile it for
+   linux/amd64 via a zombietest-build container. We do this so `make` works
+   from a Mac. This requires a working Docker installation.
+
+3. Once we have linux/amd64 runsvinit and zombie binaries, we produce a
+   zombietest container via the Dockerfile. That container contains a single
+   runit service, /etc/service/zombie, which supervises the zombie binary. We
+   provide no default ENTRYPOINT, so we can supply it at runtime.
+
+4. Once the zombietest container is built, we invoke the test.bash script.
+   That launches a version of the container with runsvinit set to NOT reap
+   zombies, and after 1 second, verifies that zombies exist. Then, it launches
+   a version of the container with runsvinit set to reap zombies, and after 1
+   second, verifies that no zombies exist.
+

vendor/runsvinit/zombietest/build/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest
+RUN apk add --update gcc musl-dev && rm -rf /var/cache/apk/*
+COPY zombie.c /

vendor/runsvinit/zombietest/build/zombie.c

@@ -0,0 +1,18 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+int main() {
+        pid_t pid;
+        int i; 
+        for (i = 0; i<5; i++)  {  
+                pid = fork();
+                if (pid > 0) {
+                        printf("Zombie #%d born\n", i + 1);
+                } else {
+                        printf("Brains...\n");
+                        exit(0);
+                }  
+        }  
+        return 0; 
+}

vendor/runsvinit/zombietest/run-zombie

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec /zombie

vendor/runsvinit/zombietest/test.bash

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+function zombies() {
+	if [ -z "$CIRCLECI" ]
+	then
+		docker exec $C ps -o pid,stat | grep Z | wc -l
+	else
+		# https://circleci.com/docs/docker#docker-exec
+		sudo lxc-attach -n "$(docker inspect --format '{{.Id}}' $C)" -- sh -c "ps -o pid,stat | grep Z | wc -l"
+	fi
+}
+
+function stop_rm() {
+	docker stop $1
+	docker rm $1
+}
+
+SLEEP=1
+RC=0
+
+C=$(docker run -d zombietest /runsvinit -reap=false)
+sleep $SLEEP
+NOREAP=$(zombies)
+echo -n without reaping, we have $NOREAP zombies...
+if [ "$NOREAP" -le "0" ]
+then
+	echo " FAIL"
+	RC=1
+else
+	echo " good"
+fi
+stop_rm $C
+
+C=$(docker run -d zombietest /runsvinit)
+sleep $SLEEP
+YESREAP=$(zombies)
+echo -n with reaping, we have $YESREAP zombies...
+if [ "$YESREAP" -gt "0" ]
+then
+	echo " FAIL"
+	RC=1
+else
+	echo " good"
+fi
+stop_rm $C
+
+exit $RC