From 3d2d43bd4fc4c183fc8b2ac392abd870c5a8bf60 Mon Sep 17 00:00:00 2001 From: Eric Zhou Date: Wed, 24 Jun 2026 18:52:54 +0800 Subject: [PATCH] fix(metrics): reduce step metric cardinality (#6777) --- .../10-configuration/10-server.md | 19 +++++++++++++++---- server/metric/metrics_server.go | 13 +++++++------ server/rpc/rpc.go | 8 ++++---- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/docs/docs/30-administration/10-configuration/10-server.md b/docs/docs/30-administration/10-configuration/10-server.md index 1876f697f..d6bb942ba 100644 --- a/docs/docs/30-administration/10-configuration/10-server.md +++ b/docs/docs/30-administration/10-configuration/10-server.md @@ -402,10 +402,21 @@ woodpecker_waiting_steps 0 woodpecker_worker_count 4 # HELP woodpecker_step_failures_total Total number of pipeline step failures. # TYPE woodpecker_step_failures_total counter -woodpecker_step_failures_total{pipeline="1",repo="woodpecker-ci/woodpecker",step="deploy"} 1 +woodpecker_step_failures_total{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker"} 1 # HELP woodpecker_step_duration_seconds Step duration in seconds. -# TYPE woodpecker_step_duration_seconds gauge -woodpecker_step_duration_seconds{pipeline="1",repo="woodpecker-ci/woodpecker",step="deploy"} 12 +# TYPE woodpecker_step_duration_seconds histogram +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="1"} 0 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="5"} 0 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="10"} 0 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="30"} 1 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="60"} 1 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="300"} 1 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="600"} 1 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="1800"} 1 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="3600"} 1 +woodpecker_step_duration_seconds_bucket{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker",le="+Inf"} 1 +woodpecker_step_duration_seconds_sum{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker"} 12 +woodpecker_step_duration_seconds_count{repo="woodpecker-ci/woodpecker",step="deploy",workflow="woodpecker"} 1 ``` Step-level metrics are exported as long as `WOODPECKER_STEP_LEVEL_METRICS` is not disabled. @@ -671,7 +682,7 @@ Example: `:9001` - Name: `WOODPECKER_STEP_LEVEL_METRICS` - Default: `true` -Enable step-level metrics, including failed step counters and step duration gauges. +Enable step-level metrics, including failed step counters and step duration histograms. --- diff --git a/server/metric/metrics_server.go b/server/metric/metrics_server.go index 5acfa2944..fe295f6ad 100644 --- a/server/metric/metrics_server.go +++ b/server/metric/metrics_server.go @@ -34,8 +34,8 @@ const ( ) var ( - FailurePipelineStepInfoCount *prometheus.CounterVec = nil - StepDurationRecord *prometheus.GaugeVec = nil + FailurePipelineStepInfoCount *prometheus.CounterVec = nil + StepDurationRecord *prometheus.HistogramVec = nil ) func StartMetricsCollector(ctx context.Context, c *cli.Command, _store store.Store) { @@ -83,16 +83,17 @@ func StartMetricsCollector(ctx context.Context, c *cli.Command, _store store.Sto Name: "step_failures_total", Help: "Total number of pipeline step failures.", }, - []string{"pipeline", "repo", "step"}, + []string{"workflow", "repo", "step"}, ) - StepDurationRecord = promauto.NewGaugeVec( - prometheus.GaugeOpts{ + StepDurationRecord = promauto.NewHistogramVec( + prometheus.HistogramOpts{ Namespace: "woodpecker", Name: "step_duration_seconds", Help: "Step duration in seconds.", + Buckets: []float64{1, 5, 10, 30, 60, 300, 600, 1800, 3600}, }, - []string{"pipeline", "repo", "step"}, + []string{"workflow", "repo", "step"}, ) } go func() { diff --git a/server/rpc/rpc.go b/server/rpc/rpc.go index bea83c565..d80a79932 100644 --- a/server/rpc/rpc.go +++ b/server/rpc/rpc.go @@ -214,16 +214,16 @@ func (s *RPC) Update(c context.Context, strWorkflowID string, state rpc.StepStat (step.State == model.StatusFailure || step.State == model.StatusKilled || step.State == model.StatusError) { - metric.FailurePipelineStepInfoCount.WithLabelValues(strconv.FormatInt(workflow.PipelineID, 10), repo.FullName, step.Name).Inc() + metric.FailurePipelineStepInfoCount.WithLabelValues(workflow.Name, repo.FullName, step.Name).Inc() } - if metric.StepDurationRecord != nil && state.Exited && step.Started > 0 && step.Finished > step.Started { + if metric.StepDurationRecord != nil && state.Exited && step.Started > 0 && step.Finished >= step.Started { duration := step.Finished - step.Started metric.StepDurationRecord.WithLabelValues( - strconv.FormatInt(workflow.PipelineID, 10), + workflow.Name, repo.FullName, step.Name, - ).Set(float64(duration)) + ).Observe(float64(duration)) } if state.Exited { server.Config.Services.LogStore.StepFinished(step)