From b085c80ef34e865acee9ef9602dd161fd89a7c72 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Tue, 14 Mar 2017 10:03:52 +0000
Subject: [PATCH] Decode via byte slice for memcache and file read

This is more efficient, since the decoder can read field names in-place.
It also appears to be absolutely faster.
---
 app/collector.go                   | 17 +++++++++++++++-
 app/multitenant/memcache_client.go |  3 +--
 report/marshal.go                  | 32 ++++++++++++++++++++++++++++++
 report/marshal_test.go             |  8 ++++++++
 4 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/app/collector.go b/app/collector.go
index b17bacbab..51cce7092 100644
--- a/app/collector.go
+++ b/app/collector.go
@@ -1,7 +1,9 @@
 package app
 
 import (
+	"compress/gzip"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -290,7 +292,20 @@ func readReport(path string) (rpt report.Report, _ error) {
 		return rpt, fmt.Errorf("Unsupported file extension: %v", fileType)
 	}
 
-	err = rpt.ReadBinary(f, gzipped, handle)
+	var buf []byte
+	if gzipped {
+		r, err := gzip.NewReader(f)
+		if err != nil {
+			return rpt, err
+		}
+		buf, err = ioutil.ReadAll(r)
+	} else {
+		buf, err = ioutil.ReadAll(f)
+	}
+	if err != nil {
+		return rpt, err
+	}
+	err = rpt.ReadBytes(buf, handle)
 
 	return rpt, err
 }
diff --git a/app/multitenant/memcache_client.go b/app/multitenant/memcache_client.go
index 44a34b4bd..c4fd3e297 100644
--- a/app/multitenant/memcache_client.go
+++ b/app/multitenant/memcache_client.go
@@ -1,7 +1,6 @@
 package multitenant
 
 import (
-	"bytes"
 	"fmt"
 	"net"
 	"sort"
@@ -176,7 +175,7 @@ func (c *MemcacheClient) FetchReports(ctx context.Context, keys []string) (map[s
 			continue
 		}
 		go func(key string) {
-			rep, err := report.MakeFromBinary(bytes.NewReader(item.Value))
+			rep, err := report.MakeFromBytes(item.Value)
 			if err != nil {
 				log.Warningf("Corrupt report in memcache %v: %v", key, err)
 				ch <- result{key: key}
diff --git a/report/marshal.go b/report/marshal.go
index e362102f4..1e013761a 100644
--- a/report/marshal.go
+++ b/report/marshal.go
@@ -1,8 +1,10 @@
 package report
 
 import (
+	"bytes"
 	"compress/gzip"
 	"io"
+	"io/ioutil"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/ugorji/go/codec"
@@ -75,3 +77,33 @@ func MakeFromBinary(r io.Reader) (*Report, error) {
 	}
 	return &rep, nil
 }
+
+// ReadBytes reads bytes into a Report, using a codecHandle.
+func (rep *Report) ReadBytes(buf []byte, codecHandle codec.Handle) error {
+	return codec.NewDecoderBytes(buf, codecHandle).Decode(&rep)
+}
+
+// MakeFromBytes constructs a Report from a gzipped msgpack.
+func MakeFromBytes(buf []byte) (*Report, error) {
+	compressedSize := len(buf)
+	r, err := gzip.NewReader(bytes.NewBuffer(buf))
+	if err != nil {
+		return nil, err
+	}
+	buf, err = ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	uncompressedSize := len(buf)
+	log.Debugf(
+		"Received report sizes: compressed %d bytes, uncompressed %d bytes (%.2f%%)",
+		compressedSize,
+		uncompressedSize,
+		float32(compressedSize)/float32(uncompressedSize)*100,
+	)
+	rep := MakeReport()
+	if err := rep.ReadBytes(buf, &codec.MsgpackHandle{}); err != nil {
+		return nil, err
+	}
+	return &rep, nil
+}
diff --git a/report/marshal_test.go b/report/marshal_test.go
index f779c4743..3a4c9b76d 100644
--- a/report/marshal_test.go
+++ b/report/marshal_test.go
@@ -13,6 +13,7 @@ func TestRoundtrip(t *testing.T) {
 	var buf bytes.Buffer
 	r1 := report.MakeReport()
 	r1.WriteBinary(&buf, gzip.BestCompression)
+	bytes := append([]byte{}, buf.Bytes()...) // copy the contents for later
 	r2, err := report.MakeFromBinary(&buf)
 	if err != nil {
 		t.Error(err)
@@ -20,6 +21,13 @@ func TestRoundtrip(t *testing.T) {
 	if !reflect.DeepEqual(r1, *r2) {
 		t.Errorf("%v != %v", r1, *r2)
 	}
+	r3, err := report.MakeFromBytes(bytes)
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(r1, *r3) {
+		t.Errorf("%v != %v", r1, *r3)
+	}
 }
 
 func TestRoundtripNoCompression(t *testing.T) {