github/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2026-02-13 21:00:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor pipeline engine (#6073)

Browse Source 
restructure pipeline/*.go to use submodules

<!-- https://claude.ai/chat/1b8965d7-5bca-42c7-86b4-48c2d645c362 -->

- pipeline/error.go -> pipeline/errors/...
- pipeline/pipeline.go#Runtime -> pipeline/runtime/runtime.go
- pipeline/pipeline.go#execAll -> pipeline/runtime/executor.go
- pipeline/shutdown.go -> pipeline/runtime/shutdown.go
- pipeline/logger.go ->pipeline/logging
- pipeline/tracer.go -> pipeline/tracing
- pipeline/pipeline.go#State -> state/state.go
This commit is contained in:
6543
2026-02-13 11:56:43 +01:00
committed by GitHub
parent 8a3fbaaef5
commit a63b93f5ee
22 changed files with 372 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
agent/logger.go
View File

@@ -23,11 +23,12 @@ import (
"go.woodpecker-ci.org/woodpecker/v3/agent/log"
"go.woodpecker-ci.org/woodpecker/v3/pipeline"
backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/logging"
pipeline_utils "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils"
"go.woodpecker-ci.org/woodpecker/v3/rpc"
)
func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger {
func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) logging.Logger {
return func(step *backend.Step, rc io.ReadCloser) error {
defer rc.Close()

25
agent/runner.go
View File

@@ -25,8 +25,9 @@ import (
"github.com/rs/zerolog/log"
"google.golang.org/grpc/metadata"
"go.woodpecker-ci.org/woodpecker/v3/pipeline"
backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
pipeline_runtime "go.woodpecker-ci.org/woodpecker/v3/pipeline/runtime"
"go.woodpecker-ci.org/woodpecker/v3/rpc"
"go.woodpecker-ci.org/woodpecker/v3/shared/constant"
"go.woodpecker-ci.org/woodpecker/v3/shared/utils"
@@ -99,7 +100,7 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error {
workflowCtx = utils.WithContextSigtermCallback(workflowCtx, func() {
logger.Error().Msg("received sigterm termination signal")
// WithContextSigtermCallback would cancel the context too, but we want our own custom error
cancelWorkflowCtx(pipeline.ErrCancel)
cancelWorkflowCtx(pipeline_errors.ErrCancel)
})
// Listen for remote cancel events (UI / API).
@@ -114,7 +115,7 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error {
} else {
if canceled {
logger.Debug().Err(err).Msg("server side cancel signal received")
cancelWorkflowCtx(pipeline.ErrCancel)
cancelWorkflowCtx(pipeline_errors.ErrCancel)
}
// Wait returned without error, meaning the workflow finished normally
logger.Debug().Msg("cancel listener exited normally")
@@ -153,14 +154,14 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error {
var uploads sync.WaitGroup
// Run pipeline
err = pipeline.New(
err = pipeline_runtime.New(
workflow.Config,
pipeline.WithContext(workflowCtx),
pipeline.WithTaskUUID(fmt.Sprint(workflow.ID)),
pipeline.WithLogger(r.createLogger(logger, &uploads, workflow)),
pipeline.WithTracer(r.createTracer(ctxMeta, &uploads, logger, workflow)),
pipeline.WithBackend(*r.backend),
pipeline.WithDescription(map[string]string{
pipeline_runtime.WithContext(workflowCtx),
pipeline_runtime.WithTaskUUID(fmt.Sprint(workflow.ID)),
pipeline_runtime.WithLogger(r.createLogger(logger, &uploads, workflow)),
pipeline_runtime.WithTracer(r.createTracer(ctxMeta, &uploads, logger, workflow)),
pipeline_runtime.WithBackend(*r.backend),
pipeline_runtime.WithDescription(map[string]string{
"workflow_id": workflow.ID,
"repo": repoName,
"pipeline_number": pipelineNumber,
@@ -171,10 +172,10 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error {
if err != nil {
state.Error = err.Error()
if errors.Is(err, pipeline.ErrCancel) {
if errors.Is(err, pipeline_errors.ErrCancel) {
state.Canceled = true
// cleanup joined error messages
state.Error = pipeline.ErrCancel.Error()
state.Error = pipeline_errors.ErrCancel.Error()
}
}

10
agent/tracer.go
View File

@@ -24,12 +24,14 @@ import (
"github.com/rs/zerolog"
"go.woodpecker-ci.org/woodpecker/v3/pipeline"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/state"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing"
"go.woodpecker-ci.org/woodpecker/v3/rpc"
)
func (r *Runner) createTracer(ctxMeta context.Context, uploads *sync.WaitGroup, logger zerolog.Logger, workflow *rpc.Workflow) pipeline.TraceFunc {
return func(state *pipeline.State) error {
func (r *Runner) createTracer(ctxMeta context.Context, uploads *sync.WaitGroup, logger zerolog.Logger, workflow *rpc.Workflow) tracing.TraceFunc {
return func(state *state.State) error {
uploads.Add(1)
defer uploads.Done()
@@ -46,7 +48,7 @@ func (r *Runner) createTracer(ctxMeta context.Context, uploads *sync.WaitGroup,
Exited: state.Process.Exited,
ExitCode: state.Process.ExitCode,
Started: state.Process.Started,
Canceled: errors.Is(state.Process.Error, pipeline.ErrCancel),
Canceled: errors.Is(state.Process.Error, pipeline_errors.ErrCancel),
}
if state.Process.Error != nil {
stepState.Error = state.Process.Error.Error()

17
cli/exec/exec.go
View File

@@ -45,6 +45,9 @@ import (
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/matrix"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/logging"
pipeline_runtime "go.woodpecker-ci.org/woodpecker/v3/pipeline/runtime"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing"
pipeline_utils "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils"
"go.woodpecker-ci.org/woodpecker/v3/shared/constant"
"go.woodpecker-ci.org/woodpecker/v3/shared/utils"
@@ -318,12 +321,12 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
fmt.Printf("ctrl+c received, terminating current pipeline '%s'\n", confStr)
})
return pipeline.New(compiled,
pipeline.WithContext(pipelineCtx), //nolint:contextcheck
pipeline.WithTracer(pipeline.DefaultTracer),
pipeline.WithLogger(defaultLogger),
pipeline.WithBackend(backendEngine),
pipeline.WithDescription(map[string]string{
return pipeline_runtime.New(compiled,
pipeline_runtime.WithContext(pipelineCtx), //nolint:contextcheck
pipeline_runtime.WithTracer(tracing.DefaultTracer),
pipeline_runtime.WithLogger(defaultLogger),
pipeline_runtime.WithBackend(backendEngine),
pipeline_runtime.WithDescription(map[string]string{
"CLI": "exec",
}),
).Run(ctx)
@@ -348,7 +351,7 @@ func convertPathForWindows(path string) string {
return filepath.ToSlash(path)
}
var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error {
var defaultLogger = logging.Logger(func(step *backend_types.Step, rc io.ReadCloser) error {
logWriter := NewLineWriter(step.Name, step.UUID)
return pipeline_utils.CopyLineByLine(logWriter, rc, pipeline.MaxLogLineLength)
})

94
cmd/server/openapi/docs.go
View File

@@ -5017,7 +5017,7 @@ const docTemplate = `{
"errors": {
"type": "array",
"items": {
"$ref": "#/definitions/types.PipelineError"
"$ref": "#/definitions/errors.PipelineError"
}
},
"event": {
@@ -5736,6 +5736,52 @@ const docTemplate = `{
"EventManual"
]
},
"errors.PipelineError": {
"type": "object",
"properties": {
"data": {},
"is_warning": {
"type": "boolean"
},
"message": {
"type": "string"
},
"type": {
"$ref": "#/definitions/errors.PipelineErrorType"
}
}
},
"errors.PipelineErrorType": {
"type": "string",
"enum": [
"linter",
"deprecation",
"compiler",
"generic",
"bad_habit"
],
"x-enum-comments": {
"PipelineErrorTypeBadHabit": "some bad-habit error",
"PipelineErrorTypeCompiler": "some error with the config semantics",
"PipelineErrorTypeDeprecation": "using some deprecated feature",
"PipelineErrorTypeGeneric": "some generic error",
"PipelineErrorTypeLinter": "some error with the config syntax"
},
"x-enum-descriptions": [
"some error with the config syntax",
"using some deprecated feature",
"some error with the config semantics",
"some generic error",
"some bad-habit error"
],
"x-enum-varnames": [
"PipelineErrorTypeLinter",
"PipelineErrorTypeDeprecation",
"PipelineErrorTypeCompiler",
"PipelineErrorTypeGeneric",
"PipelineErrorTypeBadHabit"
]
},
"metadata.Author": {
"type": "object",
"properties": {
@@ -6156,52 +6202,6 @@ const docTemplate = `{
"$ref": "#/definitions/StatusValue"
}
}
},
"types.PipelineError": {
"type": "object",
"properties": {
"data": {},
"is_warning": {
"type": "boolean"
},
"message": {
"type": "string"
},
"type": {
"$ref": "#/definitions/types.PipelineErrorType"
}
}
},
"types.PipelineErrorType": {
"type": "string",
"enum": [
"linter",
"deprecation",
"compiler",
"generic",
"bad_habit"
],
"x-enum-comments": {
"PipelineErrorTypeBadHabit": "some bad-habit error",
"PipelineErrorTypeCompiler": "some error with the config semantics",
"PipelineErrorTypeDeprecation": "using some deprecated feature",
"PipelineErrorTypeGeneric": "some generic error",
"PipelineErrorTypeLinter": "some error with the config syntax"
},
"x-enum-descriptions": [
"some error with the config syntax",
"using some deprecated feature",
"some error with the config semantics",
"some generic error",
"some bad-habit error"
],
"x-enum-varnames": [
"PipelineErrorTypeLinter",
"PipelineErrorTypeDeprecation",
"PipelineErrorTypeCompiler",
"PipelineErrorTypeGeneric",
"PipelineErrorTypeBadHabit"
]
}
}
}`

71
pipeline/errors/error.go
View File

@@ -1,71 +0,0 @@
package errors
import (
"errors"
"go.uber.org/multierr"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
)
type LinterErrorData struct {
File string `json:"file"`
Field string `json:"field"`
}
type DeprecationErrorData struct {
File string `json:"file"`
Field string `json:"field"`
Docs string `json:"docs"`
}
type BadHabitErrorData struct {
File string `json:"file"`
Field string `json:"field"`
Docs string `json:"docs"`
}
func GetLinterData(e *types.PipelineError) *LinterErrorData {
if e.Type != types.PipelineErrorTypeLinter {
return nil
}
if data, ok := e.Data.(*LinterErrorData); ok {
return data
}
return nil
}
func GetPipelineErrors(err error) []*types.PipelineError {
var pipelineErrors []*types.PipelineError
for _, _err := range multierr.Errors(err) {
var err *types.PipelineError
if errors.As(_err, &err) {
pipelineErrors = append(pipelineErrors, err)
} else {
pipelineErrors = append(pipelineErrors, &types.PipelineError{
Message: _err.Error(),
Type: types.PipelineErrorTypeGeneric,
})
}
}
return pipelineErrors
}
func HasBlockingErrors(err error) bool {
if err == nil {
return false
}
errs := GetPipelineErrors(err)
for _, err := range errs {
if !err.IsWarning {
return true
}
}
return false
}

83
pipeline/errors/linter.go Normal file
View File

@@ -0,0 +1,83 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package errors
import (
"errors"
"go.uber.org/multierr"
)
type LinterErrorData struct {
File string `json:"file"`
Field string `json:"field"`
}
type DeprecationErrorData struct {
File string `json:"file"`
Field string `json:"field"`
Docs string `json:"docs"`
}
type BadHabitErrorData struct {
File string `json:"file"`
Field string `json:"field"`
Docs string `json:"docs"`
}
func GetLinterData(e *PipelineError) *LinterErrorData {
if e.Type != PipelineErrorTypeLinter {
return nil
}
if data, ok := e.Data.(*LinterErrorData); ok {
return data
}
return nil
}
func GetPipelineErrors(err error) []*PipelineError {
var pipelineErrors []*PipelineError
for _, _err := range multierr.Errors(err) {
var err *PipelineError
if errors.As(_err, &err) {
pipelineErrors = append(pipelineErrors, err)
} else {
pipelineErrors = append(pipelineErrors, &PipelineError{
Message: _err.Error(),
Type: PipelineErrorTypeGeneric,
})
}
}
return pipelineErrors
}
func HasBlockingErrors(err error) bool {
if err == nil {
return false
}
errs := GetPipelineErrors(err)
for _, err := range errs {
if !err.IsWarning {
return true
}
}
return false
}

51
pipeline/errors/error_test.go → pipeline/errors/linter_test.go
View File

@@ -1,3 +1,17 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package errors_test
import (
@@ -8,7 +22,6 @@ import (
"go.uber.org/multierr"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
)
func TestGetPipelineErrors(t *testing.T) {
@@ -17,7 +30,7 @@ func TestGetPipelineErrors(t *testing.T) {
tests := []struct {
title string
err error
expected []*types.PipelineError
expected []*pipeline_errors.PipelineError
}{
{
title: "nil error",
@@ -26,10 +39,10 @@ func TestGetPipelineErrors(t *testing.T) {
},
{
title: "warning",
err: &types.PipelineError{
err: &pipeline_errors.PipelineError{
IsWarning: true,
},
expected: []*types.PipelineError{
expected: []*pipeline_errors.PipelineError{
{
IsWarning: true,
},
@@ -37,10 +50,10 @@ func TestGetPipelineErrors(t *testing.T) {
},
{
title: "pipeline error",
err: &types.PipelineError{
err: &pipeline_errors.PipelineError{
IsWarning: false,
},
expected: []*types.PipelineError{
expected: []*pipeline_errors.PipelineError{
{
IsWarning: false,
},
@@ -49,14 +62,14 @@ func TestGetPipelineErrors(t *testing.T) {
{
title: "multiple warnings",
err: multierr.Combine(
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: true,
},
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: true,
},
),
expected: []*types.PipelineError{
expected: []*pipeline_errors.PipelineError{
{
IsWarning: true,
},
@@ -68,15 +81,15 @@ func TestGetPipelineErrors(t *testing.T) {
{
title: "multiple errors and warnings",
err: multierr.Combine(
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: true,
},
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: false,
},
errors.New("some error"),
),
expected: []*types.PipelineError{
expected: []*pipeline_errors.PipelineError{
{
IsWarning: true,
},
@@ -84,7 +97,7 @@ func TestGetPipelineErrors(t *testing.T) {
IsWarning: false,
},
{
Type: types.PipelineErrorTypeGeneric,
Type: pipeline_errors.PipelineErrorTypeGeneric,
IsWarning: false,
Message: "some error",
},
@@ -112,14 +125,14 @@ func TestHasBlockingErrors(t *testing.T) {
},
{
title: "warning",
err: &types.PipelineError{
err: &pipeline_errors.PipelineError{
IsWarning: true,
},
expected: false,
},
{
title: "pipeline error",
err: &types.PipelineError{
err: &pipeline_errors.PipelineError{
IsWarning: false,
},
expected: true,
@@ -127,10 +140,10 @@ func TestHasBlockingErrors(t *testing.T) {
{
title: "multiple warnings",
err: multierr.Combine(
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: true,
},
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: true,
},
),
@@ -139,10 +152,10 @@ func TestHasBlockingErrors(t *testing.T) {
{
title: "multiple errors and warnings",
err: multierr.Combine(
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: true,
},
&types.PipelineError{
&pipeline_errors.PipelineError{
IsWarning: false,
},
errors.New("some error"),

16
pipeline/errors/types/errors.go → pipeline/errors/pipeline.go
View File

@@ -1,4 +1,18 @@
package types
// Copyright 2024 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package errors
import (
"fmt"

2
pipeline/error.go → pipeline/errors/runtime.go
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
package errors
import (
"errors"

11
pipeline/frontend/yaml/linter/error.go
View File

@@ -15,15 +15,14 @@
package linter
import (
"go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
)
func newLinterError(message, file, field string, isWarning bool) *errorTypes.PipelineError {
return &errorTypes.PipelineError{
Type: errorTypes.PipelineErrorTypeLinter,
func newLinterError(message, file, field string, isWarning bool) *pipeline_errors.PipelineError {
return &pipeline_errors.PipelineError{
Type: pipeline_errors.PipelineErrorTypeLinter,
Message: message,
Data: &errors.LinterErrorData{File: file, Field: field},
Data: &pipeline_errors.LinterErrorData{File: file, Field: field},
IsWarning: isWarning,
}
}

9
pipeline/frontend/yaml/linter/linter.go
View File

@@ -20,8 +20,7 @@ import (
"codeberg.org/6543/xyaml"
"go.uber.org/multierr"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter/schema"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/types"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/utils"
@@ -345,10 +344,10 @@ func (l *Linter) lintBadHabits(config *WorkflowConfig) (err error) {
}
}
if field != "" {
err = multierr.Append(err, &errorTypes.PipelineError{
Type: errorTypes.PipelineErrorTypeBadHabit,
err = multierr.Append(err, &pipeline_errors.PipelineError{
Type: pipeline_errors.PipelineErrorTypeBadHabit,
Message: "Set an event filter for all steps or the entire workflow on all items of the `when` block",
Data: errors.BadHabitErrorData{
Data: pipeline_errors.BadHabitErrorData{
File: config.File,
Field: field,
Docs: "https://woodpecker-ci.org/docs/usage/linter#event-filter-for-all-steps",

6
pipeline/frontend/yaml/matrix/matrix.go
View File

@@ -19,7 +19,7 @@ import (
"codeberg.org/6543/xyaml"
errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
)
const (
@@ -116,7 +116,7 @@ func parse(raw []byte) (Matrix, error) {
Matrix map[string][]string
}{}
if err := xyaml.Unmarshal(raw, &data); err != nil {
return nil, &errorTypes.PipelineError{Message: err.Error(), Type: errorTypes.PipelineErrorTypeCompiler}
return nil, &pipeline_errors.PipelineError{Message: err.Error(), Type: pipeline_errors.PipelineErrorTypeCompiler}
}
return data.Matrix, nil
}
@@ -129,7 +129,7 @@ func parseList(raw []byte) ([]Axis, error) {
}{}
if err := xyaml.Unmarshal(raw, &data); err != nil {
return nil, &errorTypes.PipelineError{Message: err.Error(), Type: errorTypes.PipelineErrorTypeCompiler}
return nil, &pipeline_errors.PipelineError{Message: err.Error(), Type: pipeline_errors.PipelineErrorTypeCompiler}
}
return data.Matrix.Include, nil
}

2
pipeline/logger.go → pipeline/logging/logger.go
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
package logging
import (
"io"

89
pipeline/pipeline.go → pipeline/runtime/executor.go
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
package runtime
import (
"context"
@@ -21,79 +21,14 @@ import (
"sync"
"time"
"github.com/oklog/ulid/v2"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"golang.org/x/sync/errgroup"
backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
pipelineErrors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/state"
)
// TODO: move runtime into "runtime" subpackage
type (
// State defines the pipeline and process state.
State struct {
// Global state of the pipeline.
Pipeline struct {
// Pipeline time started
Started int64 `json:"time"`
// Current pipeline step
Step *backend.Step `json:"step"`
// Current pipeline error state
Error error `json:"error"`
}
// Current process state.
Process backend.State
}
)
// Runtime represents a workflow state executed by a specific backend.
// Each workflow gets its own state configuration at runtime.
type Runtime struct {
err error
spec *backend.Config
engine backend.Backend
started int64
// The context a workflow is being executed with.
// All normal (non cleanup) operations must use this.
// Cleanup operations should use the runnerCtx passed to Run()
ctx context.Context
tracer Tracer
logger Logger
taskUUID string
Description map[string]string // The runtime descriptors.
}
// New returns a new runtime using the specified runtime
// configuration and runtime engine.
func New(spec *backend.Config, opts ...Option) *Runtime {
r := new(Runtime)
r.Description = map[string]string{}
r.spec = spec
r.ctx = context.Background()
r.taskUUID = ulid.Make().String()
for _, opts := range opts {
opts(r)
}
return r
}
func (r *Runtime) MakeLogger() zerolog.Logger {
logCtx := log.With()
for key, val := range r.Description {
logCtx = logCtx.Str(key, val)
}
return logCtx.Logger()
}
// Run starts the execution of a workflow and waits for it to complete.
func (r *Runtime) Run(runnerCtx context.Context) error {
logger := r.MakeLogger()
@@ -122,9 +57,9 @@ func (r *Runtime) Run(runnerCtx context.Context) error {
r.started = time.Now().Unix()
if err := r.engine.SetupWorkflow(runnerCtx, r.spec, r.taskUUID); err != nil {
var stepErr *pipelineErrors.ErrInvalidWorkflowSetup
var stepErr *pipeline_errors.ErrInvalidWorkflowSetup
if errors.As(err, &stepErr) {
state := new(State)
state := new(state.State)
state.Pipeline.Step = stepErr.Step
state.Pipeline.Error = stepErr.Err
state.Process = backend.State{
@@ -147,7 +82,7 @@ func (r *Runtime) Run(runnerCtx context.Context) error {
for _, stage := range r.spec.Stages {
select {
case <-r.ctx.Done():
return ErrCancel
return pipeline_errors.ErrCancel
case err := <-r.execAll(runnerCtx, stage.Steps):
if err != nil {
r.err = err
@@ -167,7 +102,7 @@ func (r *Runtime) traceStep(processState *backend.State, err error, step *backen
return nil
}
state := new(State)
state := new(state.State)
state.Pipeline.Started = r.started
state.Pipeline.Step = step
state.Pipeline.Error = r.err
@@ -244,7 +179,7 @@ func (r *Runtime) execAll(runnerCtx context.Context, steps []*backend.Step) <-ch
// normalize context cancel error
if errors.Is(err, context.Canceled) {
err = ErrCancel
err = pipeline_errors.ErrCancel
}
// Return the error after tracing it.
@@ -326,7 +261,7 @@ func (r *Runtime) exec(runnerCtx context.Context, step *backend.Step, setupWg *s
waitState, err := r.engine.WaitStep(r.ctx, step, r.taskUUID) //nolint:contextcheck
if err != nil {
if errors.Is(err, context.Canceled) {
waitState.Error = ErrCancel
waitState.Error = pipeline_errors.ErrCancel
} else {
return nil, err
}
@@ -343,16 +278,16 @@ func (r *Runtime) exec(runnerCtx context.Context, step *backend.Step, setupWg *s
// we handle cancel case
if ctxErr := r.ctx.Err(); ctxErr != nil && errors.Is(ctxErr, context.Canceled) {
waitState.Error = ErrCancel
waitState.Error = pipeline_errors.ErrCancel
}
if waitState.OOMKilled {
return waitState, &OomError{
return waitState, &pipeline_errors.OomError{
UUID: step.UUID,
Code: waitState.ExitCode,
}
} else if waitState.ExitCode != 0 {
return waitState, &ExitError{
return waitState, &pipeline_errors.ExitError{
UUID: step.UUID,
Code: waitState.ExitCode,
}

8
pipeline/option.go → pipeline/runtime/option.go
View File

@@ -12,12 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
package runtime
import (
"context"
backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/logging"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing"
)
// Option configures a runtime option.
@@ -31,14 +33,14 @@ func WithBackend(backend backend.Backend) Option {
}
// WithLogger returns an option configured with a runtime logger.
func WithLogger(logger Logger) Option {
func WithLogger(logger logging.Logger) Option {
return func(r *Runtime) {
r.logger = logger
}
}
// WithTracer returns an option configured with a runtime tracer.
func WithTracer(tracer Tracer) Option {
func WithTracer(tracer tracing.Tracer) Option {
return func(r *Runtime) {
r.tracer = tracer
}

70
pipeline/runtime/runtime.go Normal file
View File

@@ -0,0 +1,70 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package runtime
import (
"context"
"github.com/oklog/ulid/v2"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/logging"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing"
)
// Runtime represents a workflow state executed by a specific backend.
// Each workflow gets its own state configuration at runtime.
type Runtime struct {
err error
spec *backend.Config
engine backend.Backend
started int64
// The context a workflow is being executed with.
// All normal (non cleanup) operations must use this.
// Cleanup operations should use the runnerCtx passed to Run()
ctx context.Context
tracer tracing.Tracer
logger logging.Logger
taskUUID string
Description map[string]string // The runtime descriptors.
}
// New returns a new runtime using the specified runtime
// configuration and runtime engine.
func New(spec *backend.Config, opts ...Option) *Runtime {
r := new(Runtime)
r.Description = map[string]string{}
r.spec = spec
r.ctx = context.Background()
r.taskUUID = ulid.Make().String()
for _, opts := range opts {
opts(r)
}
return r
}
func (r *Runtime) MakeLogger() zerolog.Logger {
logCtx := log.With()
for key, val := range r.Description {
logCtx = logCtx.Str(key, val)
}
return logCtx.Logger()
}

2
pipeline/shutdown.go → pipeline/runtime/shutdown.go
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
package runtime
import (
"context"

31
pipeline/error_test.go → pipeline/state/state.go
View File

@@ -12,25 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
package state
import (
"testing"
"github.com/stretchr/testify/assert"
backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
)
func TestExitError(t *testing.T) {
err := ExitError{
UUID: "14534321",
Code: 255,
type (
// State defines the pipeline and process state.
State struct {
// Global state of the pipeline.
Pipeline struct {
// Pipeline time started
Started int64 `json:"time"`
// Current pipeline step
Step *backend.Step `json:"step"`
// Current pipeline error state
Error error `json:"error"`
}
assert.Equal(t, "uuid=14534321: exit code 255", err.Error())
}
func TestOomError(t *testing.T) {
err := OomError{
UUID: "14534321",
// Current process state.
Process backend.State
}
assert.Equal(t, "uuid=14534321: received oom kill", err.Error())
}
)

12
pipeline/tracer.go → pipeline/tracing/tracer.go
View File

@@ -12,30 +12,32 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
package tracing
import (
"strconv"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/state"
)
// Tracer handles process tracing.
type Tracer interface {
Trace(*State) error
Trace(*state.State) error
}
// TraceFunc type is an adapter to allow the use of ordinary
// functions as a Tracer.
type TraceFunc func(*State) error
type TraceFunc func(*state.State) error
// Trace calls f(state).
func (f TraceFunc) Trace(state *State) error {
func (f TraceFunc) Trace(state *state.State) error {
return f(state)
}
// DefaultTracer provides a tracer that updates the CI_ environment
// variables to include the correct timestamp and status.
// TODO: find either a new home or better name for this.
var DefaultTracer = TraceFunc(func(state *State) error {
var DefaultTracer = TraceFunc(func(state *state.State) error {
if state.Process.Exited {
return nil
}

4
server/model/pipeline.go
View File

@@ -16,7 +16,7 @@
package model
import (
"go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
)
type Pipeline struct {
@@ -28,7 +28,7 @@ type Pipeline struct {
Event WebhookEvent `json:"event" xorm:"event"`
EventReason []string `json:"event_reason" xorm:"json 'event_reason'"`
Status StatusValue `json:"status" xorm:"INDEX 'status'"`
Errors []*types.PipelineError `json:"errors" xorm:"json 'errors'"`
Errors []*errors.PipelineError `json:"errors" xorm:"json 'errors'"`
Created int64 `json:"created" xorm:"'created' NOT NULL DEFAULT 0 created"`
Updated int64 `json:"updated" xorm:"'updated' NOT NULL DEFAULT 0 updated"`
Started int64 `json:"started" xorm:"started"`

3
server/pipeline/stepbuilder/step_builder.go
View File

@@ -29,7 +29,6 @@ import (
"go.woodpecker-ci.org/woodpecker/v3/pipeline"
backend_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors"
errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml"
"go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler"
@@ -139,7 +138,7 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A
// parse yaml pipeline
parsed, err := yaml.ParseString(substituted)
if err != nil {
return nil, &errorTypes.PipelineError{Message: err.Error(), Type: errorTypes.PipelineErrorTypeCompiler}
return nil, &pipeline_errors.PipelineError{Message: err.Error(), Type: pipeline_errors.PipelineErrorTypeCompiler}
}
// lint pipeline