From 3a122aeed513258877003378a866c7b598bb2ecb Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 13 Dec 2017 23:40:04 +0000 Subject: [PATCH 1/5] refactor: rename awsCollector.getReportKeys to reportKeysInRange ...so we can recycle the former name. --- app/multitenant/aws_collector.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/multitenant/aws_collector.go b/app/multitenant/aws_collector.go index 18aff2769..8ebed467a 100644 --- a/app/multitenant/aws_collector.go +++ b/app/multitenant/aws_collector.go @@ -215,8 +215,8 @@ func (c *awsCollector) CreateTables() error { return err } -// getReportKeys gets the s3 keys for reports in this range -func (c *awsCollector) getReportKeys(ctx context.Context, userid string, row int64, start, end time.Time) ([]string, error) { +// reportKeysInRange returns the s3 keys for reports in the specified range +func (c *awsCollector) reportKeysInRange(ctx context.Context, userid string, row int64, start, end time.Time) ([]string, error) { rowKey := fmt.Sprintf("%s-%s", userid, strconv.FormatInt(row, 10)) var resp *dynamodb.QueryOutput err := instrument.TimeRequestHistogram(ctx, "DynamoDB.Query", dynamoRequestDuration, func(_ context.Context) error { @@ -314,12 +314,12 @@ func (c *awsCollector) Report(ctx context.Context, timestamp time.Time) (report. // Queries will only every span 2 rows max. var reportKeys []string if rowStart != rowEnd { - reportKeys1, err := c.getReportKeys(ctx, userid, rowStart, start, end) + reportKeys1, err := c.reportKeysInRange(ctx, userid, rowStart, start, end) if err != nil { return report.MakeReport(), err } - reportKeys2, err := c.getReportKeys(ctx, userid, rowEnd, start, end) + reportKeys2, err := c.reportKeysInRange(ctx, userid, rowEnd, start, end) if err != nil { return report.MakeReport(), err } @@ -327,7 +327,7 @@ func (c *awsCollector) Report(ctx context.Context, timestamp time.Time) (report. reportKeys = append(reportKeys, reportKeys1...) reportKeys = append(reportKeys, reportKeys2...) } else { - if reportKeys, err = c.getReportKeys(ctx, userid, rowEnd, start, end); err != nil { + if reportKeys, err = c.reportKeysInRange(ctx, userid, rowEnd, start, end); err != nil { return report.MakeReport(), err } } From 6c4bf58fe295fe0f6d4ba045a8c2b013ecd1abd9 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 13 Dec 2017 23:50:05 +0000 Subject: [PATCH 2/5] refactor: extract awsCollector.getReportKeys helper This makes Report() more readable. We also intend to use this function elsewhere. --- app/multitenant/aws_collector.go | 69 ++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/app/multitenant/aws_collector.go b/app/multitenant/aws_collector.go index 8ebed467a..dc4b6fbad 100644 --- a/app/multitenant/aws_collector.go +++ b/app/multitenant/aws_collector.go @@ -264,6 +264,43 @@ func (c *awsCollector) reportKeysInRange(ctx context.Context, userid string, row return result, nil } +// getReportKeys returns the S3 for reports in the reporting window ending at timestamp. +func (c *awsCollector) getReportKeys(ctx context.Context, timestamp time.Time) ([]string, error) { + var ( + end = timestamp + start = end.Add(-c.window) + rowStart = start.UnixNano() / time.Hour.Nanoseconds() + rowEnd = end.UnixNano() / time.Hour.Nanoseconds() + userid, err = c.userIDer(ctx) + ) + if err != nil { + return nil, err + } + + // Queries will only every span 2 rows max. + var reportKeys []string + if rowStart != rowEnd { + reportKeys1, err := c.reportKeysInRange(ctx, userid, rowStart, start, end) + if err != nil { + return nil, err + } + + reportKeys2, err := c.reportKeysInRange(ctx, userid, rowEnd, start, end) + if err != nil { + return nil, err + } + + reportKeys = append(reportKeys, reportKeys1...) + reportKeys = append(reportKeys, reportKeys2...) + } else { + if reportKeys, err = c.reportKeysInRange(ctx, userid, rowEnd, start, end); err != nil { + return nil, err + } + } + + return reportKeys, nil +} + func (c *awsCollector) getReports(ctx context.Context, reportKeys []string) ([]report.Report, error) { missing := reportKeys @@ -300,39 +337,11 @@ func (c *awsCollector) getReports(ctx context.Context, reportKeys []string) ([]r } func (c *awsCollector) Report(ctx context.Context, timestamp time.Time) (report.Report, error) { - var ( - end = timestamp - start = end.Add(-c.window) - rowStart = start.UnixNano() / time.Hour.Nanoseconds() - rowEnd = end.UnixNano() / time.Hour.Nanoseconds() - userid, err = c.userIDer(ctx) - ) + reportKeys, err := c.getReportKeys(ctx, timestamp) if err != nil { return report.MakeReport(), err } - - // Queries will only every span 2 rows max. - var reportKeys []string - if rowStart != rowEnd { - reportKeys1, err := c.reportKeysInRange(ctx, userid, rowStart, start, end) - if err != nil { - return report.MakeReport(), err - } - - reportKeys2, err := c.reportKeysInRange(ctx, userid, rowEnd, start, end) - if err != nil { - return report.MakeReport(), err - } - - reportKeys = append(reportKeys, reportKeys1...) - reportKeys = append(reportKeys, reportKeys2...) - } else { - if reportKeys, err = c.reportKeysInRange(ctx, userid, rowEnd, start, end); err != nil { - return report.MakeReport(), err - } - } - - log.Debugf("Fetching %d reports from %v to %v", len(reportKeys), start, end) + log.Debugf("Fetching %d reports to %v", len(reportKeys), timestamp) reports, err := c.getReports(ctx, reportKeys) if err != nil { return report.MakeReport(), err From 54fe1e37da8c9e47cbf40a25806094a34899cbda Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 13 Dec 2017 23:52:17 +0000 Subject: [PATCH 3/5] cosmetic --- app/multitenant/aws_collector.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/app/multitenant/aws_collector.go b/app/multitenant/aws_collector.go index dc4b6fbad..318925bc6 100644 --- a/app/multitenant/aws_collector.go +++ b/app/multitenant/aws_collector.go @@ -267,12 +267,13 @@ func (c *awsCollector) reportKeysInRange(ctx context.Context, userid string, row // getReportKeys returns the S3 for reports in the reporting window ending at timestamp. func (c *awsCollector) getReportKeys(ctx context.Context, timestamp time.Time) ([]string, error) { var ( - end = timestamp - start = end.Add(-c.window) - rowStart = start.UnixNano() / time.Hour.Nanoseconds() - rowEnd = end.UnixNano() / time.Hour.Nanoseconds() - userid, err = c.userIDer(ctx) + end = timestamp + start = end.Add(-c.window) + rowStart = start.UnixNano() / time.Hour.Nanoseconds() + rowEnd = end.UnixNano() / time.Hour.Nanoseconds() ) + + userid, err := c.userIDer(ctx) if err != nil { return nil, err } @@ -284,12 +285,10 @@ func (c *awsCollector) getReportKeys(ctx context.Context, timestamp time.Time) ( if err != nil { return nil, err } - reportKeys2, err := c.reportKeysInRange(ctx, userid, rowEnd, start, end) if err != nil { return nil, err } - reportKeys = append(reportKeys, reportKeys1...) reportKeys = append(reportKeys, reportKeys2...) } else { From 72b9e9c6b94f6be903a7977e88d53cba39d62ffd Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 14 Dec 2017 00:13:45 +0000 Subject: [PATCH 4/5] add Reporter.HasReports() for cheap report availability checking This requires no report reading / merging. We plan to expose this in the HTTP API, so the UI gets a cheap way of checking whether the app is currently receiving data from probes. --- app/collector.go | 20 ++++++++++++++++++++ app/multitenant/aws_collector.go | 5 +++++ 2 files changed, 25 insertions(+) diff --git a/app/collector.go b/app/collector.go index d71ff2087..8477be4da 100644 --- a/app/collector.go +++ b/app/collector.go @@ -29,6 +29,7 @@ const reportQuantisationInterval = 3 * time.Second // interface for parts of the app, and several experimental components. type Reporter interface { Report(context.Context, time.Time) (report.Report, error) + HasReports(context.Context, time.Time) (bool, error) HasHistoricReports() bool WaitOn(context.Context, chan struct{}) UnWait(context.Context, chan struct{}) @@ -161,6 +162,19 @@ func (c *collector) Report(_ context.Context, timestamp time.Time) (report.Repor return rpt, nil } +// HasReports indicates whether the collector contains reports between +// timestamp-app.window and timestamp. +func (c *collector) HasReports(ctx context.Context, timestamp time.Time) (bool, error) { + c.mtx.Lock() + defer c.mtx.Unlock() + + if len(c.timestamps) < 1 { + return false, nil + } + + return !c.timestamps[0].After(timestamp) && !c.timestamps[len(c.reports)-1].Before(timestamp.Add(-c.window)), nil +} + // HasHistoricReports indicates whether the collector contains reports // older than now-app.window. func (c *collector) HasHistoricReports() bool { @@ -223,6 +237,12 @@ func (c StaticCollector) Report(context.Context, time.Time) (report.Report, erro return report.Report(c), nil } +// HasReports indicates whether the collector contains reports between +// timestamp-app.window and timestamp. +func (c StaticCollector) HasReports(context.Context, time.Time) (bool, error) { + return true, nil +} + // HasHistoricReports indicates whether the collector contains reports // older than now-app.window. func (c StaticCollector) HasHistoricReports() bool { diff --git a/app/multitenant/aws_collector.go b/app/multitenant/aws_collector.go index 318925bc6..d0a75dbf3 100644 --- a/app/multitenant/aws_collector.go +++ b/app/multitenant/aws_collector.go @@ -349,6 +349,11 @@ func (c *awsCollector) Report(ctx context.Context, timestamp time.Time) (report. return c.merger.Merge(reports), nil } +func (c *awsCollector) HasReports(ctx context.Context, timestamp time.Time) (bool, error) { + reportKeys, err := c.getReportKeys(ctx, timestamp) + return len(reportKeys) > 0, err +} + func (c *awsCollector) HasHistoricReports() bool { return true } From 70d84170d6ca122db19e0a66fc57a4b1eb2c9bd8 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 14 Dec 2017 01:07:40 +0000 Subject: [PATCH 5/5] introduce a cheap /api/probes?sparse variant In many cases we only need to know whether there are _any_ connected probes, and not the probe details. Obtaining that info is cheaper since it requires no reading or merging or reports. --- app/api_report.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/api_report.go b/app/api_report.go index 6dfbd153d..d930a5373 100644 --- a/app/api_report.go +++ b/app/api_report.go @@ -32,6 +32,16 @@ type probeDesc struct { // Probe handler func makeProbeHandler(rep Reporter) CtxHandlerFunc { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + if _, sparse := r.Form["sparse"]; sparse { + // if we have reports, we must have connected probes + hasProbes, err := rep.HasReports(ctx, time.Now()) + if err != nil { + respondWith(w, http.StatusInternalServerError, err) + } + respondWith(w, http.StatusOK, hasProbes) + return + } rpt, err := rep.Report(ctx, time.Now()) if err != nil { respondWith(w, http.StatusInternalServerError, err)