Merge pull request #1234 from weaveworks/middleware

Add various middleware to app.

app/api_report_test.go

@@ -12,8 +12,9 @@ import (
 )
 
 func topologyServer() *httptest.Server {
-	handler := app.TopologyHandler(StaticReport{}, mux.NewRouter(), nil)
-	return httptest.NewServer(handler)
+	router := mux.NewRouter().SkipClean(true)
+	app.RegisterTopologyRoutes(router, StaticReport{})
+	return httptest.NewServer(router)
 }
 
 func TestAPIReport(t *testing.T) {

app/api_topologies.go

@@ -307,9 +307,9 @@ func (r *registry) captureRenderer(rep Reporter, f reportRenderHandler) CtxHandl
 	}
 }
 
-func (r *registry) captureRendererWithoutFilters(rep Reporter, topologyID string, f reportRenderHandler) CtxHandlerFunc {
+func (r *registry) captureRendererWithoutFilters(rep Reporter, f reportRenderHandler) CtxHandlerFunc {
 	return func(ctx context.Context, w http.ResponseWriter, req *http.Request) {
-		topology, ok := r.get(topologyID)
+		topology, ok := r.get(mux.Vars(req)["topology"])
 		if !ok {
 			http.NotFound(w, req)
 			return

app/api_topologies_test.go

@@ -54,8 +54,8 @@ func TestAPITopologyAddsKubernetes(t *testing.T) {
 	router := mux.NewRouter()
 	c := app.NewCollector(1 * time.Minute)
 	app.RegisterReportPostHandler(c, router)
-	handler := app.TopologyHandler(c, router, nil)
-	ts := httptest.NewServer(handler)
+	app.RegisterTopologyRoutes(router, c)
+	ts := httptest.NewServer(router)
 	defer ts.Close()
 
 	body := getRawJSON(t, ts, "/api/topology")

app/api_topology.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/gorilla/mux"
 	"golang.org/x/net/context"
 
 	"github.com/weaveworks/scope/common/xfer"
@@ -56,23 +57,24 @@ func handleWs(ctx context.Context, rep Reporter, renderer render.Renderer, w htt
 }
 
 // Individual nodes.
-func handleNode(topologyID, nodeID string) func(context.Context, Reporter, render.Renderer, http.ResponseWriter, *http.Request) {
-	return func(ctx context.Context, rep Reporter, renderer render.Renderer, w http.ResponseWriter, r *http.Request) {
-		var (
-			report, err = rep.Report(ctx)
-			rendered    = renderer.Render(report)
-			node, ok    = rendered[nodeID]
-		)
-		if err != nil {
-			respondWith(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-		if !ok {
-			http.NotFound(w, r)
-			return
-		}
-		respondWith(w, http.StatusOK, APINode{Node: detailed.MakeNode(topologyID, report, rendered, node)})
+func handleNode(ctx context.Context, rep Reporter, renderer render.Renderer, w http.ResponseWriter, r *http.Request) {
+	var (
+		vars        = mux.Vars(r)
+		topologyID  = vars["topology"]
+		nodeID      = vars["id"]
+		report, err = rep.Report(ctx)
+		rendered    = renderer.Render(report)
+		node, ok    = rendered[nodeID]
+	)
+	if err != nil {
+		respondWith(w, http.StatusInternalServerError, err.Error())
+		return
 	}
+	if !ok {
+		http.NotFound(w, r)
+		return
+	}
+	respondWith(w, http.StatusOK, APINode{Node: detailed.MakeNode(topologyID, report, rendered, node)})
 }
 
 func handleWebsocket(

app/router.go

@@ -80,52 +80,21 @@ func gzipHandler(h http.HandlerFunc) http.HandlerFunc {
 	return handlers.GZIPHandlerFunc(h, nil)
 }
 
-// TopologyHandler registers the various topology routes with a http mux.
-//
-// The returned http.Handler has to be passed directly to http.ListenAndServe,
-// and cannot be nested inside another gorrilla.mux.
-//
-// Routes which should be matched before the topology routes should be added
-// to a router and passed in preRoutes.  Routes to be matches after topology
-// routes should be added to a router and passed to postRoutes.
-func TopologyHandler(c Reporter, preRoutes *mux.Router, postRoutes http.Handler) http.Handler {
-	get := preRoutes.Methods("GET").Subrouter()
-	get.HandleFunc("/api", gzipHandler(requestContextDecorator(apiHandler)))
-	get.HandleFunc("/api/topology", gzipHandler(requestContextDecorator(topologyRegistry.makeTopologyList(c))))
+// RegisterTopologyRoutes registers the various topology routes with a http mux.
+func RegisterTopologyRoutes(router *mux.Router, r Reporter) {
+	get := router.Methods("GET").Subrouter()
+	get.HandleFunc("/api",
+		gzipHandler(requestContextDecorator(apiHandler)))
+	get.HandleFunc("/api/topology",
+		gzipHandler(requestContextDecorator(topologyRegistry.makeTopologyList(r))))
 	get.HandleFunc("/api/topology/{topology}",
-		gzipHandler(requestContextDecorator(topologyRegistry.captureRenderer(c, handleTopology))))
+		gzipHandler(requestContextDecorator(topologyRegistry.captureRenderer(r, handleTopology))))
 	get.HandleFunc("/api/topology/{topology}/ws",
-		requestContextDecorator(topologyRegistry.captureRenderer(c, handleWs))) // NB not gzip!
-	get.HandleFunc("/api/report", gzipHandler(requestContextDecorator(makeRawReportHandler(c))))
-
-	// We pull in the http.DefaultServeMux to get the pprof routes
-	preRoutes.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
-
-	if postRoutes != nil {
-		preRoutes.PathPrefix("/").Handler(postRoutes)
-	}
-
-	// We have to handle the node details path manually due to
-	// various bugs in gorilla.mux and Go URL parsing.  See #804.
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		vars, match := matchURL(r, "/api/topology/{topology}/{id}")
-		if !match {
-			preRoutes.ServeHTTP(w, r)
-			return
-		}
-
-		topologyID := vars["topology"]
-		nodeID := vars["id"]
-		if nodeID == "ws" {
-			preRoutes.ServeHTTP(w, r)
-			return
-		}
-
-		handler := gzipHandler(requestContextDecorator(topologyRegistry.captureRendererWithoutFilters(
-			c, topologyID, handleNode(topologyID, nodeID),
-		)))
-		handler.ServeHTTP(w, r)
-	})
+		requestContextDecorator(topologyRegistry.captureRenderer(r, handleWs))) // NB not gzip!
+	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{id}")).HandlerFunc(
+		gzipHandler(requestContextDecorator(topologyRegistry.captureRendererWithoutFilters(r, handleNode))))
+	get.HandleFunc("/api/report",
+		gzipHandler(requestContextDecorator(makeRawReportHandler(r))))
 }
 
 // RegisterReportPostHandler registers the handler for report submission

common/middleware/instrument.go

@@ -0,0 +1,45 @@
+package middleware
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Instrument is a Middleware which records timings for every HTTP request
+type Instrument struct {
+	RouteMatcher interface {
+		Match(*http.Request, *mux.RouteMatch) bool
+	}
+	Duration *prometheus.SummaryVec
+}
+
+// Wrap implements middleware.Interface
+func (i Instrument) Wrap(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		begin := time.Now()
+		interceptor := &interceptor{ResponseWriter: w, statusCode: http.StatusOK}
+		next.ServeHTTP(interceptor, r)
+		var (
+			route  = i.getRouteName(r)
+			status = strconv.Itoa(interceptor.statusCode)
+			took   = time.Since(begin)
+		)
+		i.Duration.WithLabelValues(r.Method, route, status).Observe(float64(took.Nanoseconds()))
+	})
+}
+
+func (i Instrument) getRouteName(r *http.Request) string {
+	var routeMatch mux.RouteMatch
+	if !i.RouteMatcher.Match(r, &routeMatch) {
+		return "unmatched_path"
+	}
+	name := routeMatch.Route.GetName()
+	if name == "" {
+		return "unnamed_path"
+	}
+	return name
+}

common/middleware/logging.go

@@ -0,0 +1,45 @@
+package middleware
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+// Logging middleware logs each HTTP request method, path, response code and duration.
+var Logging = Func(func(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		begin := time.Now()
+		i := &interceptor{ResponseWriter: w, statusCode: http.StatusOK}
+		next.ServeHTTP(i, r)
+		log.Infof("%s %s (%d) %s", r.Method, r.RequestURI, i.statusCode, time.Since(begin))
+	})
+})
+
+// interceptor implements WriteHeader to intercept status codes. WriteHeader
+// may not be called on success, so initialize statusCode with the status you
+// want to report on success, i.e. http.StatusOK.
+//
+// interceptor also implements net.Hijacker, to let the downstream Handler
+// hijack the connection. This is needed by the app-mapper's proxy.
+type interceptor struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func (i *interceptor) WriteHeader(code int) {
+	i.statusCode = code
+	i.ResponseWriter.WriteHeader(code)
+}
+
+func (i *interceptor) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	hj, ok := i.ResponseWriter.(http.Hijacker)
+	if !ok {
+		return nil, nil, fmt.Errorf("interceptor: can't cast parent ResponseWriter to Hijacker")
+	}
+	return hj.Hijack()
+}

common/middleware/middleware.go

@@ -0,0 +1,30 @@
+package middleware
+
+import (
+	"net/http"
+)
+
+// Interface is the shared contract for all middlesware, and allows middlesware
+// to wrap handlers.
+type Interface interface {
+	Wrap(http.Handler) http.Handler
+}
+
+// Func is to Interface as http.HandlerFunc is to http.Handler
+type Func func(http.Handler) http.Handler
+
+// Wrap implements Interface
+func (m Func) Wrap(next http.Handler) http.Handler {
+	return m(next)
+}
+
+// Merge produces a middleware that applies multiple middlesware in turn;
+// ie Merge(f,g,h).Wrap(handler) == f.Wrap(g.Wrap(h.Wrap(handler)))
+func Merge(middlesware ...Interface) Interface {
+	return Func(func(next http.Handler) http.Handler {
+		for i := len(middlesware) - 1; i >= 0; i-- {
+			next = middlesware[i].Wrap(next)
+		}
+		return next
+	})
+}

prog/app.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -15,23 +16,50 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/weaveworks/go-checkpoint"
 	"github.com/weaveworks/weave/common"
 
 	"github.com/weaveworks/scope/app"
 	"github.com/weaveworks/scope/app/multitenant"
+	"github.com/weaveworks/scope/common/middleware"
 	"github.com/weaveworks/scope/common/weave"
 	"github.com/weaveworks/scope/common/xfer"
 	"github.com/weaveworks/scope/probe/docker"
 )
 
+var (
+	requestDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Namespace: "scope",
+		Name:      "request_duration_nanoseconds",
+		Help:      "Time spent serving HTTP requests.",
+	}, []string{"method", "route", "status_code"})
+)
+
+func init() {
+	prometheus.MustRegister(requestDuration)
+}
+
 // Router creates the mux for all the various app components.
 func router(collector app.Collector, controlRouter app.ControlRouter, pipeRouter app.PipeRouter) http.Handler {
-	router := mux.NewRouter()
+	router := mux.NewRouter().SkipClean(true)
+
+	// We pull in the http.DefaultServeMux to get the pprof routes
+	router.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
+	router.Path("/metrics").Handler(prometheus.Handler())
+
 	app.RegisterReportPostHandler(collector, router)
 	app.RegisterControlRoutes(router, controlRouter)
 	app.RegisterPipeRoutes(router, pipeRouter)
-	return app.TopologyHandler(collector, router, http.FileServer(FS(false)))
+	app.RegisterTopologyRoutes(router, collector)
+
+	router.PathPrefix("/").Handler(http.FileServer(FS(false)))
+
+	instrument := middleware.Instrument{
+		RouteMatcher: router,
+		Duration:     requestDuration,
+	}
+	return instrument.Wrap(router)
 }
 
 func awsConfigFromURL(url *url.URL) *aws.Config {
@@ -114,6 +142,7 @@ func appMain() {
 		listen    = flag.String("http.address", ":"+strconv.Itoa(xfer.AppPort), "webserver listen address")
 		logLevel  = flag.String("log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")
 		logPrefix = flag.String("log.prefix", "<app>", "prefix for each log line")
+		logHTTP   = flag.Bool("log.http", false, "Log individual HTTP requests")
 
 		weaveAddr      = flag.String("weave.addr", app.DefaultWeaveURL, "Address on which to contact WeaveDNS")
 		weaveHostname  = flag.String("weave.hostname", app.DefaultHostname, "Hostname to advertise in WeaveDNS")
@@ -161,6 +190,7 @@ func appMain() {
 	app.UniqueID = strconv.FormatInt(rand.Int63(), 16)
 	app.Version = version
 	log.Infof("app starting, version %s, ID %s", app.Version, app.UniqueID)
+	log.Infof("command line: %v", os.Args)
 
 	// Start background version checking
 	checkpoint.CheckInterval(&checkpoint.CheckParams{
@@ -189,6 +219,9 @@ func appMain() {
 	}
 
 	handler := router(collector, controlRouter, pipeRouter)
+	if *logHTTP {
+		handler = middleware.Logging.Wrap(handler)
+	}
 	go func() {
 		log.Infof("listening on %s", *listen)
 		log.Info(http.ListenAndServe(*listen, handler))

vendor/github.com/gorilla/mux/mux.go

@@ -48,6 +48,8 @@ type Router struct {
 	namedRoutes map[string]*Route
 	// See Router.StrictSlash(). This defines the flag for new routes.
 	strictSlash bool
+	// See Router.SkipClean(). This defines the flag for new routes.
+	skipClean bool
 	// If true, do not clear the request context after handling the request
 	KeepContext bool
 }
@@ -59,6 +61,12 @@ func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
 			return true
 		}
 	}
+
+	// Closest match for a router (includes sub-routers)
+	if r.NotFoundHandler != nil {
+		match.Handler = r.NotFoundHandler
+		return true
+	}
 	return false
 }
 
@@ -67,19 +75,21 @@ func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
 // When there is a match, the route variables can be retrieved calling
 // mux.Vars(request).
 func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	// Clean path to canonical form and redirect.
-	if p := cleanPath(req.URL.Path); p != req.URL.Path {
+	if !r.skipClean {
+		// Clean path to canonical form and redirect.
+		if p := cleanPath(req.URL.Path); p != req.URL.Path {
 
-		// Added 3 lines (Philip Schlump) - It was dropping the query string and #whatever from query.
-		// This matches with fix in go 1.2 r.c. 4 for same problem.  Go Issue:
-		// http://code.google.com/p/go/issues/detail?id=5252
-		url := *req.URL
-		url.Path = p
-		p = url.String()
+			// Added 3 lines (Philip Schlump) - It was dropping the query string and #whatever from query.
+			// This matches with fix in go 1.2 r.c. 4 for same problem.  Go Issue:
+			// http://code.google.com/p/go/issues/detail?id=5252
+			url := *req.URL
+			url.Path = p
+			p = url.String()
 
-		w.Header().Set("Location", p)
-		w.WriteHeader(http.StatusMovedPermanently)
-		return
+			w.Header().Set("Location", p)
+			w.WriteHeader(http.StatusMovedPermanently)
+			return
+		}
 	}
 	var match RouteMatch
 	var handler http.Handler
@@ -89,10 +99,7 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		setCurrentRoute(req, match.Route)
 	}
 	if handler == nil {
-		handler = r.NotFoundHandler
-		if handler == nil {
-			handler = http.NotFoundHandler()
-		}
+		handler = http.NotFoundHandler()
 	}
 	if !r.KeepContext {
 		defer context.Clear(req)
@@ -130,6 +137,19 @@ func (r *Router) StrictSlash(value bool) *Router {
 	return r
 }
 
+// SkipClean defines the path cleaning behaviour for new routes. The initial
+// value is false.
+//
+// When true, if the route path is "/path//to", it will remain with the double
+// slash. This is helpful if you have a route like: /fetch/http://xkcd.com/534/
+//
+// When false, the path will be cleaned, so /fetch/http://xkcd.com/534/ will
+// become /fetch/http/xkcd.com/534
+func (r *Router) SkipClean(value bool) *Router {
+	r.skipClean = value
+	return r
+}
+
 // ----------------------------------------------------------------------------
 // parentRoute
 // ----------------------------------------------------------------------------
@@ -167,7 +187,7 @@ func (r *Router) buildVars(m map[string]string) map[string]string {
 
 // NewRoute registers an empty route.
 func (r *Router) NewRoute() *Route {
-	route := &Route{parent: r, strictSlash: r.strictSlash}
+	route := &Route{parent: r, strictSlash: r.strictSlash, skipClean: r.skipClean}
 	r.routes = append(r.routes, route)
 	return route
 }
@@ -233,7 +253,7 @@ func (r *Router) Schemes(schemes ...string) *Route {
 	return r.NewRoute().Schemes(schemes...)
 }
 
-// BuildVars registers a new route with a custom function for modifying
+// BuildVarsFunc registers a new route with a custom function for modifying
 // route variables before building a URL.
 func (r *Router) BuildVarsFunc(f BuildVarsFunc) *Route {
 	return r.NewRoute().BuildVarsFunc(f)
@@ -324,11 +344,15 @@ func CurrentRoute(r *http.Request) *Route {
 }
 
 func setVars(r *http.Request, val interface{}) {
-	context.Set(r, varsKey, val)
+	if val != nil {
+		context.Set(r, varsKey, val)
+	}
 }
 
 func setCurrentRoute(r *http.Request, val interface{}) {
-	context.Set(r, routeKey, val)
+	if val != nil {
+		context.Set(r, routeKey, val)
+	}
 }
 
 // ----------------------------------------------------------------------------
@@ -350,6 +374,7 @@ func cleanPath(p string) string {
 	if p[len(p)-1] == '/' && np != "/" {
 		np += "/"
 	}
+
 	return np
 }
 

vendor/github.com/gorilla/mux/route.go

@@ -26,6 +26,9 @@ type Route struct {
 	// If true, when the path pattern is "/path/", accessing "/path" will
 	// redirect to the former and vice versa.
 	strictSlash bool
+	// If true, when the path pattern is "/path//to", accessing "/path//to"
+	// will not redirect
+	skipClean bool
 	// If true, this route never matches: it is only used to build URLs.
 	buildOnly bool
 	// The name used to build URLs.
@@ -217,8 +220,9 @@ func (m headerRegexMatcher) Match(r *http.Request, match *RouteMatch) bool {
 	return matchMapWithRegex(m, r.Header, true)
 }
 
-// Regular expressions can be used with headers as well.
-// It accepts a sequence of key/value pairs, where the value has regex support. For example
+// HeadersRegexp accepts a sequence of key/value pairs, where the value has regex
+// support. For example:
+//
 //     r := mux.NewRouter()
 //     r.HeadersRegexp("Content-Type", "application/(text|json)",
 //               "X-Requested-With", "XMLHttpRequest")
@@ -263,6 +267,7 @@ func (r *Route) Host(tpl string) *Route {
 // MatcherFunc is the function signature used by custom matchers.
 type MatcherFunc func(*http.Request, *RouteMatch) bool
 
+// Match returns the match for a given request.
 func (m MatcherFunc) Match(r *http.Request, match *RouteMatch) bool {
 	return m(r, match)
 }
@@ -532,6 +537,36 @@ func (r *Route) URLPath(pairs ...string) (*url.URL, error) {
 	}, nil
 }
 
+// GetPathTemplate returns the template used to build the
+// route match.
+// This is useful for building simple REST API documentation and for instrumentation
+// against third-party services.
+// An error will be returned if the route does not define a path.
+func (r *Route) GetPathTemplate() (string, error) {
+	if r.err != nil {
+		return "", r.err
+	}
+	if r.regexp == nil || r.regexp.path == nil {
+		return "", errors.New("mux: route doesn't have a path")
+	}
+	return r.regexp.path.template, nil
+}
+
+// GetHostTemplate returns the template used to build the
+// route match.
+// This is useful for building simple REST API documentation and for instrumentation
+// against third-party services.
+// An error will be returned if the route does not define a host.
+func (r *Route) GetHostTemplate() (string, error) {
+	if r.err != nil {
+		return "", r.err
+	}
+	if r.regexp == nil || r.regexp.host == nil {
+		return "", errors.New("mux: route doesn't have a host")
+	}
+	return r.regexp.host.template, nil
+}
+
 // prepareVars converts the route variable pairs into a map. If the route has a
 // BuildVarsFunc, it is invoked.
 func (r *Route) prepareVars(pairs ...string) (map[string]string, error) {

vendor/manifest

@@ -621,8 +621,8 @@
 		},
 		{
 			"importpath": "github.com/gorilla/mux",
-			"repository": "https://github.com/gorilla/mux",
-			"revision": "ad4d7a5882b961e07e2626045eb995c022ac6664",
+			"repository": "https://github.com/jingweno/mux",
+			"revision": "786d36e5ab042d67efe94022439cf6c91ee711dc",
 			"branch": "master"
 		},
 		{
@@ -1211,4 +1211,4 @@
 			"branch": "master"
 		}
 	]
-}
+}