GZip compression for app HTTP endpoints and probe -> app communication.

app/router.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"compress/gzip"
 	"encoding/gob"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
 
+	"github.com/PuerkitoBio/ghost/handlers"
 	"github.com/gorilla/mux"
 
 	"github.com/weaveworks/scope/render"
@@ -49,29 +52,49 @@ type collector interface {
 	xfer.Adder
 }
 
+func gzipHandler(h http.HandlerFunc) http.HandlerFunc {
+	return handlers.GZIPHandlerFunc(h, nil)
+}
+
 // Router returns the HTTP dispatcher, managing API and UI requests, and
 // accepting reports from probes.. It will always use the embedded HTML
 // resources for the UI.
 func Router(c collector) *mux.Router {
 	router := mux.NewRouter()
 	router.HandleFunc("/api/report", makeReportPostHandler(c)).Methods("POST")
+
 	get := router.Methods("GET").Subrouter()
-	get.HandleFunc("/api", apiHandler)
-	get.HandleFunc("/api/topology", makeTopologyList(c))
-	get.HandleFunc("/api/topology/{topology}", captureTopology(c, handleTopology))
-	get.HandleFunc("/api/topology/{topology}/ws", captureTopology(c, handleWs))
-	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{id}")).HandlerFunc(captureTopology(c, handleNode))
-	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{local}/{remote}")).HandlerFunc(captureTopology(c, handleEdge))
-	get.MatcherFunc(URLMatcher("/api/origin/host/{id}")).HandlerFunc(makeOriginHostHandler(c))
-	get.HandleFunc("/api/report", makeRawReportHandler(c))
+	get.HandleFunc("/api", gzipHandler(apiHandler))
+	get.HandleFunc("/api/topology", gzipHandler(makeTopologyList(c)))
+	get.HandleFunc("/api/topology/{topology}", gzipHandler(captureTopology(c, handleTopology)))
+	get.HandleFunc("/api/topology/{topology}/ws", captureTopology(c, handleWs)) // NB not gzip!
+	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{id}")).HandlerFunc(gzipHandler(captureTopology(c, handleNode)))
+	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{local}/{remote}")).HandlerFunc(gzipHandler(captureTopology(c, handleEdge)))
+	get.MatcherFunc(URLMatcher("/api/origin/host/{id}")).HandlerFunc(gzipHandler(makeOriginHostHandler(c)))
+	get.HandleFunc("/api/report", gzipHandler(makeRawReportHandler(c)))
 	get.PathPrefix("/").Handler(http.FileServer(FS(false))) // everything else is static
+
 	return router
 }
 
 func makeReportPostHandler(a xfer.Adder) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var rpt report.Report
-		if err := gob.NewDecoder(r.Body).Decode(&rpt); err != nil {
+
+		var reader io.ReadCloser
+		defer func() { reader.Close() }()
+
+		reader = r.Body
+		var err error
+		if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
+			reader, err = gzip.NewReader(r.Body)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+		}
+
+		if err := gob.NewDecoder(reader).Decode(&rpt); err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}

xfer/publisher.go

@@ -2,6 +2,7 @@ package xfer
 
 import (
 	"bytes"
+	"compress/gzip"
 	"encoding/gob"
 	"fmt"
 	"log"
@@ -52,21 +53,30 @@ func NewHTTPPublisher(target, token, id string) (*HTTPPublisher, error) {
 
 // Publish publishes the report to the URL.
 func (p HTTPPublisher) Publish(rpt report.Report) error {
-	var buf bytes.Buffer
-	if err := gob.NewEncoder(&buf).Encode(rpt); err != nil {
+	gzbuf := bytes.Buffer{}
+	gzwriter := gzip.NewWriter(&gzbuf)
+
+	if err := gob.NewEncoder(gzwriter).Encode(rpt); err != nil {
 		return err
 	}
-	req, err := http.NewRequest("POST", p.url, &buf)
+	gzwriter.Close() // otherwise the content won't get flushed to the output stream
+
+	req, err := http.NewRequest("POST", p.url, &gzbuf)
 	if err != nil {
 		return err
 	}
+
 	req.Header.Set("Authorization", AuthorizationHeader(p.token))
 	req.Header.Set(ScopeProbeIDHeader, p.id)
+	req.Header.Set("Content-Encoding", "gzip")
+	// req.Header.Set("Content-Type", "application/binary") // TODO: we should use http.DetectContentType(..) on the gob'ed
+
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return err
 	}
 	defer resp.Body.Close()
+
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf(resp.Status)
 	}

xfer/publisher_test.go

@@ -1,13 +1,17 @@
 package xfer_test
 
 import (
+	"compress/gzip"
 	"encoding/gob"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/gorilla/handlers"
+
 	"github.com/weaveworks/scope/report"
 	"github.com/weaveworks/scope/test"
 	"github.com/weaveworks/scope/xfer"
@@ -21,7 +25,7 @@ func TestHTTPPublisher(t *testing.T) {
 		done  = make(chan struct{})
 	)
 
-	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if want, have := xfer.AuthorizationHeader(token), r.Header.Get("Authorization"); want != have {
 			t.Errorf("want %q, have %q", want, have)
 		}
@@ -29,7 +33,19 @@ func TestHTTPPublisher(t *testing.T) {
 			t.Errorf("want %q, have %q", want, have)
 		}
 		var have report.Report
-		if err := gob.NewDecoder(r.Body).Decode(&have); err != nil {
+
+		reader := r.Body
+		var err error
+		if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
+			reader, err = gzip.NewReader(r.Body)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+			defer reader.Close()
+		}
+
+		if err := gob.NewDecoder(reader).Decode(&have); err != nil {
 			t.Error(err)
 			return
 		}
@@ -39,7 +55,9 @@ func TestHTTPPublisher(t *testing.T) {
 		}
 		w.WriteHeader(http.StatusOK)
 		close(done)
-	}))
+	})
+
+	s := httptest.NewServer(handlers.CompressHandler(handler))
 	defer s.Close()
 
 	p, err := xfer.NewHTTPPublisher(s.URL, token, id)