From ab7a7dcb166c0bbfe9d341a67ca3d410fcf94e3a Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Fri, 10 May 2019 17:04:46 +0000
Subject: [PATCH] Expose probe metrics to Prometheus

We are already timing all report, tag and tick operations.
If Prometheus is in use, expose those metrics that way.

Adjust metrics naming to fit with Prometheus norms.

The previous way these metrics were exposed was via SIGUSR1, and we
can only have one "sink", so make it either-or.

Signed-off-by: Bryan Boreham <bryan@weave.works>
---
 probe/probe.go | 17 ++++++++++++-----
 prog/probe.go  | 24 +++++++++++++++++++-----
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/probe/probe.go b/probe/probe.go
index 4e9bac59e..60d6559ff 100644
--- a/probe/probe.go
+++ b/probe/probe.go
@@ -135,12 +135,10 @@ func (p *Probe) spyLoop() {
 	for {
 		select {
 		case <-spyTick:
-			t := time.Now()
 			p.tick()
 			rpt := p.report()
 			rpt = p.tag(rpt)
 			p.spiedReports <- rpt
-			metrics.MeasureSince([]string{"Report Generaton"}, t)
 		case <-p.quit:
 			return
 		}
@@ -151,7 +149,10 @@ func (p *Probe) tick() {
 	for _, ticker := range p.tickers {
 		t := time.Now()
 		err := ticker.Tick()
-		metrics.MeasureSince([]string{ticker.Name(), "ticker"}, t)
+		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
+			{Name: "operation", Value: "ticker"},
+			{Name: "module", Value: ticker.Name()},
+		})
 		if err != nil {
 			log.Errorf("error doing ticker: %v", err)
 		}
@@ -168,7 +169,10 @@ func (p *Probe) report() report.Report {
 			if !timer.Stop() {
 				log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
 			}
-			metrics.MeasureSince([]string{rep.Name(), "reporter"}, t)
+			metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
+				{Name: "operation", Value: "reporter"},
+				{Name: "module", Value: rep.Name()},
+			})
 			if err != nil {
 				log.Errorf("error generating report: %v", err)
 				newReport = report.MakeReport() // empty is OK to merge
@@ -193,7 +197,10 @@ func (p *Probe) tag(r report.Report) report.Report {
 		if !timer.Stop() {
 			log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
 		}
-		metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t)
+		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
+			{Name: "operation", Value: "tagger"},
+			{Name: "module", Value: tagger.Name()},
+		})
 		if err != nil {
 			log.Errorf("error applying tagger: %v", err)
 		}
diff --git a/prog/probe.go b/prog/probe.go
index 3e14d2f37..9df88d22e 100644
--- a/prog/probe.go
+++ b/prog/probe.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/armon/go-metrics"
+	metrics_prom "github.com/armon/go-metrics/prometheus"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 
@@ -108,11 +109,24 @@ func probeMain(flags probeFlags, targets []appclient.Target) {
 	traceCloser := tracing.NewFromEnv("scope-probe")
 	defer traceCloser.Close()
 
-	// Setup in memory metrics sink
-	inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
-	sig := metrics.DefaultInmemSignal(inm)
-	defer sig.Stop()
-	metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
+	cfg := &metrics.Config{
+		ServiceName:      "scope-probe",
+		TimerGranularity: time.Second,
+		FilterDefault:    true, // Don't filter metrics by default
+	}
+	if flags.httpListen == "" {
+		// Setup in memory metrics sink
+		inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
+		sig := metrics.DefaultInmemSignal(inm)
+		defer sig.Stop()
+		metrics.NewGlobal(cfg, inm)
+	} else {
+		sink, err := metrics_prom.NewPrometheusSink()
+		if err != nil {
+			log.Fatalf("Failed to create Prometheus metrics sink: %v", err)
+		}
+		metrics.NewGlobal(cfg, sink)
+	}
 	logCensoredArgs()
 	defer log.Info("probe exiting")