diff --git a/probe/probe.go b/probe/probe.go index 4e9bac59e..60d6559ff 100644 --- a/probe/probe.go +++ b/probe/probe.go @@ -135,12 +135,10 @@ func (p *Probe) spyLoop() { for { select { case <-spyTick: - t := time.Now() p.tick() rpt := p.report() rpt = p.tag(rpt) p.spiedReports <- rpt - metrics.MeasureSince([]string{"Report Generaton"}, t) case <-p.quit: return } @@ -151,7 +149,10 @@ func (p *Probe) tick() { for _, ticker := range p.tickers { t := time.Now() err := ticker.Tick() - metrics.MeasureSince([]string{ticker.Name(), "ticker"}, t) + metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{ + {Name: "operation", Value: "ticker"}, + {Name: "module", Value: ticker.Name()}, + }) if err != nil { log.Errorf("error doing ticker: %v", err) } @@ -168,7 +169,10 @@ func (p *Probe) report() report.Report { if !timer.Stop() { log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval) } - metrics.MeasureSince([]string{rep.Name(), "reporter"}, t) + metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{ + {Name: "operation", Value: "reporter"}, + {Name: "module", Value: rep.Name()}, + }) if err != nil { log.Errorf("error generating report: %v", err) newReport = report.MakeReport() // empty is OK to merge @@ -193,7 +197,10 @@ func (p *Probe) tag(r report.Report) report.Report { if !timer.Stop() { log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval) } - metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t) + metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{ + {Name: "operation", Value: "tagger"}, + {Name: "module", Value: tagger.Name()}, + }) if err != nil { log.Errorf("error applying tagger: %v", err) } diff --git a/prog/probe.go b/prog/probe.go index 3e14d2f37..9df88d22e 100644 --- a/prog/probe.go +++ b/prog/probe.go @@ -13,6 +13,7 @@ import ( "time" "github.com/armon/go-metrics" + metrics_prom "github.com/armon/go-metrics/prometheus" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -108,11 +109,24 @@ func probeMain(flags probeFlags, targets []appclient.Target) { traceCloser := tracing.NewFromEnv("scope-probe") defer traceCloser.Close() - // Setup in memory metrics sink - inm := metrics.NewInmemSink(time.Minute, 2*time.Minute) - sig := metrics.DefaultInmemSignal(inm) - defer sig.Stop() - metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm) + cfg := &metrics.Config{ + ServiceName: "scope-probe", + TimerGranularity: time.Second, + FilterDefault: true, // Don't filter metrics by default + } + if flags.httpListen == "" { + // Setup in memory metrics sink + inm := metrics.NewInmemSink(time.Minute, 2*time.Minute) + sig := metrics.DefaultInmemSignal(inm) + defer sig.Stop() + metrics.NewGlobal(cfg, inm) + } else { + sink, err := metrics_prom.NewPrometheusSink() + if err != nil { + log.Fatalf("Failed to create Prometheus metrics sink: %v", err) + } + metrics.NewGlobal(cfg, sink) + } logCensoredArgs() defer log.Info("probe exiting")