Add Prometheus metrics for pipeline step duration and failures (#6738)

- Add `WOODPECKER_STEP_LEVEL_METRICS` server option.
- Move server metrics collection into the `server/metric`.
- Export step-level Prometheus metrics when enabled:
  - `woodpecker_step_failures_total{pipeline,repo,step}`
  - `woodpecker_step_duration_seconds{pipeline,repo,step}`

cmd/server/flags.go

@@ -129,6 +129,12 @@ var flags = append([]cli.Flag{
 		Usage:   "metrics server address",
 		Value:   "",
 	},
+	&cli.BoolFlag{
+		Sources: cli.EnvVars("WOODPECKER_STEP_LEVEL_METRICS"),
+		Name:    "step-level-metrics",
+		Usage:   "enable step-level metrics",
+		Value:   true,
+	},
 	&cli.StringSliceFlag{
 		Sources: cli.EnvVars("WOODPECKER_ADMIN"),
 		Name:    "admin",

cmd/server/server.go

@@ -38,6 +38,7 @@ import (
 
 	"go.woodpecker-ci.org/woodpecker/v3/server"
 	cron_scheduler "go.woodpecker-ci.org/woodpecker/v3/server/cron"
+	"go.woodpecker-ci.org/woodpecker/v3/server/metric"
 	"go.woodpecker-ci.org/woodpecker/v3/server/router"
 	"go.woodpecker-ci.org/woodpecker/v3/server/router/middleware"
 	"go.woodpecker-ci.org/woodpecker/v3/server/store"
@@ -125,7 +126,7 @@ func run(ctx context.Context, c *cli.Command) error {
 
 	log.Info().Msgf("starting Woodpecker server with version '%s'", version.String())
 
-	startMetricsCollector(ctx, _store)
+	metric.StartMetricsCollector(ctx, c, _store)
 
 	serviceWaitingGroup.Go(func() error {
 		log.Info().Msg("starting cron service ...")

cmd/server/setup.go

@@ -23,7 +23,6 @@ import (
 	"net/url"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/rs/zerolog/log"
 	"github.com/tink-crypto/tink-go/v2/subtle/random"
@@ -47,11 +46,6 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v3/server/store/types"
 )
 
-const (
-	queueInfoRefreshInterval = 500 * time.Millisecond
-	storeInfoRefreshInterval = 10 * time.Second
-)
-
 func setupStore(ctx context.Context, c *cli.Command) (store.Store, error) {
 	datasource := c.String("db-datasource")
 	driver := c.String("db-driver")

docs/docs/30-administration/10-configuration/10-server.md

@@ -400,8 +400,16 @@ woodpecker_waiting_steps 0
 # HELP woodpecker_worker_count Total number of workers.
 # TYPE woodpecker_worker_count gauge
 woodpecker_worker_count 4
+# HELP woodpecker_step_failures_total Total number of pipeline step failures.
+# TYPE woodpecker_step_failures_total counter
+woodpecker_step_failures_total{pipeline="1",repo="woodpecker-ci/woodpecker",step="deploy"} 1
+# HELP woodpecker_step_duration_seconds Step duration in seconds.
+# TYPE woodpecker_step_duration_seconds gauge
+woodpecker_step_duration_seconds{pipeline="1",repo="woodpecker-ci/woodpecker",step="deploy"} 12
 ```
 
+Step-level metrics are exported as long as `WOODPECKER_STEP_LEVEL_METRICS` is not disabled.
+
 #### Example response structure
 
 ```json
@@ -658,6 +666,15 @@ Example: `:9001`
 
 ---
 
+### STEP_LEVEL_METRICS
+
+- Name: `WOODPECKER_STEP_LEVEL_METRICS`
+- Default: `true`
+
+Enable step-level metrics, including failed step counters and step duration gauges.
+
+---
+
 ### ADMIN
 
 - Name: `WOODPECKER_ADMIN`

server/metric/metrics_server.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package metric
 
 import (
 	"context"
@@ -22,12 +22,24 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v3"
 
 	"go.woodpecker-ci.org/woodpecker/v3/server"
 	"go.woodpecker-ci.org/woodpecker/v3/server/store"
 )
 
-func startMetricsCollector(ctx context.Context, _store store.Store) {
+const (
+	queueInfoRefreshInterval = 500 * time.Millisecond
+	storeInfoRefreshInterval = 10 * time.Second
+)
+
+var (
+	FailurePipelineStepInfoCount *prometheus.CounterVec = nil
+	StepDurationRecord           *prometheus.GaugeVec   = nil
+)
+
+func StartMetricsCollector(ctx context.Context, c *cli.Command, _store store.Store) {
+	detailedMetricsEnabled := c.Bool("step-level-metrics")
 	pendingSteps := promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: "woodpecker",
 		Name:      "pending_steps",
@@ -64,6 +76,24 @@ func startMetricsCollector(ctx context.Context, _store store.Store) {
 		Help:      "Total number of repos.",
 	})
 
+	if detailedMetricsEnabled {
+		FailurePipelineStepInfoCount = promauto.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: "woodpecker",
+				Name:      "step_failures_total",
+				Help:      "Total number of pipeline step failures.",
+			},
+			[]string{"pipeline", "repo", "step"})
+
+		StepDurationRecord = promauto.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "woodpecker",
+				Name:      "step_duration_seconds",
+				Help:      "Step duration in seconds.",
+			},
+			[]string{"pipeline", "repo", "step"},
+		)
+	}
 	go func() {
 		log.Info().Msg("queue metric collector started")
 

server/rpc/rpc.go

@@ -34,6 +34,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v3/server"
 	"go.woodpecker-ci.org/woodpecker/v3/server/forge"
 	"go.woodpecker-ci.org/woodpecker/v3/server/logging"
+	"go.woodpecker-ci.org/woodpecker/v3/server/metric"
 	"go.woodpecker-ci.org/woodpecker/v3/server/model"
 	"go.woodpecker-ci.org/woodpecker/v3/server/pipeline"
 	"go.woodpecker-ci.org/woodpecker/v3/server/pubsub"
@@ -213,6 +214,22 @@ func (s *RPC) Update(c context.Context, strWorkflowID string, state rpc.StepStat
 		log.Error().Err(err).Msg("rpc.update: cannot update step")
 	}
 
+	if metric.FailurePipelineStepInfoCount != nil &&
+		state.Exited &&
+		(step.State == model.StatusFailure ||
+			step.State == model.StatusKilled ||
+			step.State == model.StatusError) {
+		metric.FailurePipelineStepInfoCount.WithLabelValues(strconv.FormatInt(workflow.PipelineID, 10), repo.FullName, step.Name).Inc()
+	}
+
+	if metric.StepDurationRecord != nil && state.Exited && step.Started > 0 && step.Finished > step.Started {
+		duration := step.Finished - step.Started
+		metric.StepDurationRecord.WithLabelValues(
+			strconv.FormatInt(workflow.PipelineID, 10),
+			repo.FullName,
+			step.Name,
+		).Set(float64(duration))
+	}
 	if state.Exited {
 		server.Config.Services.LogStore.StepFinished(step)
 	}