diff --git a/cmd/server/flags.go b/cmd/server/flags.go index efad75d47..b1e796147 100644 --- a/cmd/server/flags.go +++ b/cmd/server/flags.go @@ -129,6 +129,12 @@ var flags = append([]cli.Flag{ Usage: "metrics server address", Value: "", }, + &cli.BoolFlag{ + Sources: cli.EnvVars("WOODPECKER_STEP_LEVEL_METRICS"), + Name: "step-level-metrics", + Usage: "enable step-level metrics", + Value: true, + }, &cli.StringSliceFlag{ Sources: cli.EnvVars("WOODPECKER_ADMIN"), Name: "admin", diff --git a/cmd/server/server.go b/cmd/server/server.go index bbd5588ff..feb8b11a3 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -38,6 +38,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/server" cron_scheduler "go.woodpecker-ci.org/woodpecker/v3/server/cron" + "go.woodpecker-ci.org/woodpecker/v3/server/metric" "go.woodpecker-ci.org/woodpecker/v3/server/router" "go.woodpecker-ci.org/woodpecker/v3/server/router/middleware" "go.woodpecker-ci.org/woodpecker/v3/server/store" @@ -125,7 +126,7 @@ func run(ctx context.Context, c *cli.Command) error { log.Info().Msgf("starting Woodpecker server with version '%s'", version.String()) - startMetricsCollector(ctx, _store) + metric.StartMetricsCollector(ctx, c, _store) serviceWaitingGroup.Go(func() error { log.Info().Msg("starting cron service ...") diff --git a/cmd/server/setup.go b/cmd/server/setup.go index 71da301e4..967fbd2c4 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -23,7 +23,6 @@ import ( "net/url" "os" "strings" - "time" "github.com/rs/zerolog/log" "github.com/tink-crypto/tink-go/v2/subtle/random" @@ -47,11 +46,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/server/store/types" ) -const ( - queueInfoRefreshInterval = 500 * time.Millisecond - storeInfoRefreshInterval = 10 * time.Second -) - func setupStore(ctx context.Context, c *cli.Command) (store.Store, error) { datasource := c.String("db-datasource") driver := c.String("db-driver") diff --git a/docs/docs/30-administration/10-configuration/10-server.md b/docs/docs/30-administration/10-configuration/10-server.md index 4cd8ce347..1876f697f 100644 --- a/docs/docs/30-administration/10-configuration/10-server.md +++ b/docs/docs/30-administration/10-configuration/10-server.md @@ -400,8 +400,16 @@ woodpecker_waiting_steps 0 # HELP woodpecker_worker_count Total number of workers. # TYPE woodpecker_worker_count gauge woodpecker_worker_count 4 +# HELP woodpecker_step_failures_total Total number of pipeline step failures. +# TYPE woodpecker_step_failures_total counter +woodpecker_step_failures_total{pipeline="1",repo="woodpecker-ci/woodpecker",step="deploy"} 1 +# HELP woodpecker_step_duration_seconds Step duration in seconds. +# TYPE woodpecker_step_duration_seconds gauge +woodpecker_step_duration_seconds{pipeline="1",repo="woodpecker-ci/woodpecker",step="deploy"} 12 ``` +Step-level metrics are exported as long as `WOODPECKER_STEP_LEVEL_METRICS` is not disabled. + #### Example response structure ```json @@ -658,6 +666,15 @@ Example: `:9001` --- +### STEP_LEVEL_METRICS + +- Name: `WOODPECKER_STEP_LEVEL_METRICS` +- Default: `true` + +Enable step-level metrics, including failed step counters and step duration gauges. + +--- + ### ADMIN - Name: `WOODPECKER_ADMIN` diff --git a/cmd/server/metrics_server.go b/server/metric/metrics_server.go similarity index 76% rename from cmd/server/metrics_server.go rename to server/metric/metrics_server.go index c746f1372..ddcfdcc2d 100644 --- a/cmd/server/metrics_server.go +++ b/server/metric/metrics_server.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package metric import ( "context" @@ -22,12 +22,24 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/rs/zerolog/log" + "github.com/urfave/cli/v3" "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) -func startMetricsCollector(ctx context.Context, _store store.Store) { +const ( + queueInfoRefreshInterval = 500 * time.Millisecond + storeInfoRefreshInterval = 10 * time.Second +) + +var ( + FailurePipelineStepInfoCount *prometheus.CounterVec = nil + StepDurationRecord *prometheus.GaugeVec = nil +) + +func StartMetricsCollector(ctx context.Context, c *cli.Command, _store store.Store) { + detailedMetricsEnabled := c.Bool("step-level-metrics") pendingSteps := promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "woodpecker", Name: "pending_steps", @@ -64,6 +76,24 @@ func startMetricsCollector(ctx context.Context, _store store.Store) { Help: "Total number of repos.", }) + if detailedMetricsEnabled { + FailurePipelineStepInfoCount = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "woodpecker", + Name: "step_failures_total", + Help: "Total number of pipeline step failures.", + }, + []string{"pipeline", "repo", "step"}) + + StepDurationRecord = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "woodpecker", + Name: "step_duration_seconds", + Help: "Step duration in seconds.", + }, + []string{"pipeline", "repo", "step"}, + ) + } go func() { log.Info().Msg("queue metric collector started") diff --git a/server/rpc/rpc.go b/server/rpc/rpc.go index 8d3b28c4b..fb7b53b99 100644 --- a/server/rpc/rpc.go +++ b/server/rpc/rpc.go @@ -34,6 +34,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/forge" "go.woodpecker-ci.org/woodpecker/v3/server/logging" + "go.woodpecker-ci.org/woodpecker/v3/server/metric" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/pipeline" "go.woodpecker-ci.org/woodpecker/v3/server/pubsub" @@ -213,6 +214,22 @@ func (s *RPC) Update(c context.Context, strWorkflowID string, state rpc.StepStat log.Error().Err(err).Msg("rpc.update: cannot update step") } + if metric.FailurePipelineStepInfoCount != nil && + state.Exited && + (step.State == model.StatusFailure || + step.State == model.StatusKilled || + step.State == model.StatusError) { + metric.FailurePipelineStepInfoCount.WithLabelValues(strconv.FormatInt(workflow.PipelineID, 10), repo.FullName, step.Name).Inc() + } + + if metric.StepDurationRecord != nil && state.Exited && step.Started > 0 && step.Finished > step.Started { + duration := step.Finished - step.Started + metric.StepDurationRecord.WithLabelValues( + strconv.FormatInt(workflow.PipelineID, 10), + repo.FullName, + step.Name, + ).Set(float64(duration)) + } if state.Exited { server.Config.Services.LogStore.StepFinished(step) }