github/weave-scope
1
0
Fork 0
mirror of https://github.com/weaveworks/scope.git synced 2026-07-16 03:49:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expose probe metrics to Prometheus

Browse Source 
We are already timing all report, tag and tick operations.
If Prometheus is in use, expose those metrics that way.

Adjust metrics naming to fit with Prometheus norms.

The previous way these metrics were exposed was via SIGUSR1, and we
can only have one "sink", so make it either-or.

Signed-off-by: Bryan Boreham <bryan@weave.works>
This commit is contained in:
Bryan Boreham
2019-05-10 17:04:46 +00:00
parent ba84edefd2
commit ab7a7dcb16
2 changed files with 31 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
probe/probe.go
View File

@@ -135,12 +135,10 @@ func (p *Probe) spyLoop() {
for {
select {
case <-spyTick:
t := time.Now()
p.tick()
rpt := p.report()
rpt = p.tag(rpt)
p.spiedReports <- rpt
metrics.MeasureSince([]string{"Report Generaton"}, t)
case <-p.quit:
return
}
@@ -151,7 +149,10 @@ func (p *Probe) tick() {
for _, ticker := range p.tickers {
t := time.Now()
err := ticker.Tick()
metrics.MeasureSince([]string{ticker.Name(), "ticker"}, t)
metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
{Name: "operation", Value: "ticker"},
{Name: "module", Value: ticker.Name()},
})
if err != nil {
log.Errorf("error doing ticker: %v", err)
}
@@ -168,7 +169,10 @@ func (p *Probe) report() report.Report {
if !timer.Stop() {
log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
}
metrics.MeasureSince([]string{rep.Name(), "reporter"}, t)
metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
{Name: "operation", Value: "reporter"},
{Name: "module", Value: rep.Name()},
})
if err != nil {
log.Errorf("error generating report: %v", err)
newReport = report.MakeReport() // empty is OK to merge
@@ -193,7 +197,10 @@ func (p *Probe) tag(r report.Report) report.Report {
if !timer.Stop() {
log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
}
metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t)
metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
{Name: "operation", Value: "tagger"},
{Name: "module", Value: tagger.Name()},
})
if err != nil {
log.Errorf("error applying tagger: %v", err)
}

24
prog/probe.go
View File

@@ -13,6 +13,7 @@ import (
"time"
"github.com/armon/go-metrics"
metrics_prom "github.com/armon/go-metrics/prometheus"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
@@ -108,11 +109,24 @@ func probeMain(flags probeFlags, targets []appclient.Target) {
traceCloser := tracing.NewFromEnv("scope-probe")
defer traceCloser.Close()
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
cfg := &metrics.Config{
ServiceName: "scope-probe",
TimerGranularity: time.Second,
FilterDefault: true, // Don't filter metrics by default
}
if flags.httpListen == "" {
// Setup in memory metrics sink
inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
sig := metrics.DefaultInmemSignal(inm)
defer sig.Stop()
metrics.NewGlobal(cfg, inm)
} else {
sink, err := metrics_prom.NewPrometheusSink()
if err != nil {
log.Fatalf("Failed to create Prometheus metrics sink: %v", err)
}
metrics.NewGlobal(cfg, sink)
}
logCensoredArgs()
defer log.Info("probe exiting")