Time out commands after three minutes

This means we get to see their output up to the point they timed out,
and also means that we avoid CircleCI killing the entire job after
five minutes of inactivity.

runner/runner.go

@@ -33,6 +33,7 @@ var (
 	useScheduler  = false
 	runParallel   = false
 	verbose       = false
+	timeout       = 180 // In seconds. Three minutes ought to be enough for any test
 
 	consoleLock = sync.Mutex{}
 )
@@ -96,7 +97,16 @@ func (t test) run(hosts []string) bool {
 	}
 
 	start := time.Now()
-	err := cmd.Run()
+	var err error
+
+	c := make(chan error, 1)
+	go func() { c <- cmd.Run() }()
+	select {
+	case err = <-c:
+	case <-time.After(time.Duration(timeout) * time.Second):
+		err = fmt.Errorf("timed out")
+	}
+
 	duration := float64(time.Now().Sub(start)) / float64(time.Second)
 
 	consoleLock.Lock()
@@ -245,6 +255,7 @@ func main() {
 	mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible")
 	mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)")
 	mflag.StringVar(&schedulerHost, []string{"scheduler-host"}, defaultSchedulerHost, "Hostname of scheduler.")
+	mflag.IntVar(&timeout, []string{"timeout"}, 180, "Max time to run one test for, in seconds")
 	mflag.Parse()
 
 	if len(os.Getenv("DEBUG")) > 0 {