app: Add graceful shutdown for http server

This will let it finish up existing connections before exiting,
which makes things like rolling restarts behave much better.

prog/app.go

@@ -17,6 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/tylerb/graceful"
 	"github.com/weaveworks/go-checkpoint"
 	"github.com/weaveworks/weave/common"
 
@@ -270,12 +271,27 @@ func appMain(flags appFlags) {
 	if flags.logHTTP {
 		handler = middleware.Logging.Wrap(handler)
 	}
+
+	server := &graceful.Server{
+		// we want to manage the stop condition ourselves below
+		NoSignalHandling: true,
+		Server: &http.Server{
+			Addr:    flags.listen,
+			Handler: handler,
+		},
+	}
 	go func() {
 		log.Infof("listening on %s", flags.listen)
-		log.Info(http.ListenAndServe(flags.listen, handler))
+		if err := server.ListenAndServe(); err != nil {
+			log.Error(err)
+		}
 	}()
 
+	// block until INT/TERM
 	common.SignalHandlerLoop()
+	// stop listening, wait for any active connections to finish
+	server.Stop(flags.stopTimeout)
+	<-server.StopChan()
 }
 
 func newWeavePublisher(dockerEndpoint, weaveAddr, weaveHostname, containerName string) (*app.WeavePublisher, error) {

prog/main.go

@@ -103,11 +103,12 @@ type probeFlags struct {
 }
 
 type appFlags struct {
-	window    time.Duration
-	listen    string
-	logLevel  string
-	logPrefix string
-	logHTTP   bool
+	window      time.Duration
+	listen      string
+	stopTimeout time.Duration
+	logLevel    string
+	logPrefix   string
+	logHTTP     bool
 
 	weaveEnabled   bool
 	weaveAddr      string
@@ -229,6 +230,7 @@ func main() {
 	// App flags
 	flag.DurationVar(&flags.app.window, "app.window", 15*time.Second, "window")
 	flag.StringVar(&flags.app.listen, "app.http.address", ":"+strconv.Itoa(xfer.AppPort), "webserver listen address")
+	flag.DurationVar(&flags.app.stopTimeout, "app.stopTimeout", 5*time.Second, "How long to wait for http requests to finish when shutting down")
 	flag.StringVar(&flags.app.logLevel, "app.log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")
 	flag.StringVar(&flags.app.logPrefix, "app.log.prefix", "<app>", "prefix for each log line")
 	flag.BoolVar(&flags.app.logHTTP, "app.log.http", false, "Log individual HTTP requests")