diff --git a/app/multitenant/aws_collector.go b/app/multitenant/aws_collector.go index 571522c0f..346b7ebb4 100644 --- a/app/multitenant/aws_collector.go +++ b/app/multitenant/aws_collector.go @@ -1,8 +1,6 @@ package multitenant import ( - "bytes" - "compress/gzip" "crypto/md5" "fmt" "io" @@ -361,26 +359,22 @@ func (c *awsCollector) Add(ctx context.Context, rep report.Report) error { return err } - // first, encode the report into a buffer and record its size - var buf bytes.Buffer - rep.WriteBinary(&buf, gzip.BestCompression) - reportSizeHistogram.Observe(float64(buf.Len())) - - // second, put the report on s3 + // first, put the report on s3 rowKey, colKey := calculateDynamoKeys(userid, time.Now()) reportKey, err := calculateReportKey(rowKey, colKey) if err != nil { return err } - err = c.s3.StoreBytes(reportKey, buf.Bytes()) + reportSize, err := c.s3.StoreReport(reportKey, &rep) if err != nil { return err } + reportSizeHistogram.Observe(float64(reportSize)) // third, put it in memcache if c.memcache != nil { - err = c.memcache.StoreBytes(reportKey, buf.Bytes()) + _, err = c.memcache.StoreReport(reportKey, &rep) if err != nil { // NOTE: We don't abort here because failing to store in memcache // doesn't actually break anything else -- it's just an diff --git a/app/multitenant/memcache_client.go b/app/multitenant/memcache_client.go index ee8e0e07e..3539b39fb 100644 --- a/app/multitenant/memcache_client.go +++ b/app/multitenant/memcache_client.go @@ -204,10 +204,11 @@ func (c *MemcacheClient) FetchReports(keys []string) (map[string]report.Report, } // StoreReport serializes and stores a report. -func (c *MemcacheClient) StoreReport(key string, report *report.Report) error { +func (c *MemcacheClient) StoreReport(key string, report *report.Report) (int, error) { var buf bytes.Buffer report.WriteBinary(&buf, gzip.BestCompression) - return c.StoreBytes(key, buf.Bytes()) + err := c.StoreBytes(key, buf.Bytes()) + return buf.Len(), err } // StoreBytes stores a report, expecting the report to be serialized already. diff --git a/app/multitenant/s3_client.go b/app/multitenant/s3_client.go index 6d90bda74..48fff4f19 100644 --- a/app/multitenant/s3_client.go +++ b/app/multitenant/s3_client.go @@ -86,10 +86,13 @@ func (store *S3Store) fetchReport(key string) (*report.Report, error) { } // StoreReport serializes and stores a report. -func (store *S3Store) StoreReport(key string, report *report.Report) error { +// +// Returns the size of the report. This only equals bytes written if err is nil. +func (store *S3Store) StoreReport(key string, report *report.Report) (int, error) { var buf bytes.Buffer report.WriteBinary(&buf, gzip.BestCompression) - return store.StoreBytes(key, buf.Bytes()) + err := store.StoreBytes(key, buf.Bytes()) + return buf.Len(), err } // StoreBytes stores a report in S3, expecting the report to be serialized