From a63b93f5ee3baf3f783994bd78981e790303f8af Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 13 Feb 2026 11:56:43 +0100 Subject: [PATCH] Refactor pipeline engine (#6073) restructure pipeline/*.go to use submodules - pipeline/error.go -> pipeline/errors/... - pipeline/pipeline.go#Runtime -> pipeline/runtime/runtime.go - pipeline/pipeline.go#execAll -> pipeline/runtime/executor.go - pipeline/shutdown.go -> pipeline/runtime/shutdown.go - pipeline/logger.go ->pipeline/logging - pipeline/tracer.go -> pipeline/tracing - pipeline/pipeline.go#State -> state/state.go --- agent/logger.go | 3 +- agent/runner.go | 25 ++--- agent/tracer.go | 10 +- cli/exec/exec.go | 17 ++-- cmd/server/openapi/docs.go | 94 +++++++++---------- pipeline/errors/error.go | 71 -------------- pipeline/errors/linter.go | 83 ++++++++++++++++ .../errors/{error_test.go => linter_test.go} | 51 ++++++---- .../errors/{types/errors.go => pipeline.go} | 16 +++- pipeline/{error.go => errors/runtime.go} | 2 +- pipeline/frontend/yaml/linter/error.go | 11 +-- pipeline/frontend/yaml/linter/linter.go | 9 +- pipeline/frontend/yaml/matrix/matrix.go | 6 +- pipeline/{ => logging}/logger.go | 2 +- pipeline/{pipeline.go => runtime/executor.go} | 89 +++--------------- pipeline/{ => runtime}/option.go | 8 +- pipeline/runtime/runtime.go | 70 ++++++++++++++ pipeline/{ => runtime}/shutdown.go | 2 +- pipeline/{error_test.go => state/state.go} | 33 +++---- pipeline/{ => tracing}/tracer.go | 12 ++- server/model/pipeline.go | 74 +++++++-------- server/pipeline/stepbuilder/step_builder.go | 3 +- 22 files changed, 372 insertions(+), 319 deletions(-) delete mode 100644 pipeline/errors/error.go create mode 100644 pipeline/errors/linter.go rename pipeline/errors/{error_test.go => linter_test.go} (60%) rename pipeline/errors/{types/errors.go => pipeline.go} (66%) rename pipeline/{error.go => errors/runtime.go} (98%) rename pipeline/{ => logging}/logger.go (97%) rename pipeline/{pipeline.go => runtime/executor.go} (78%) rename pipeline/{ => runtime}/option.go (87%) create mode 100644 pipeline/runtime/runtime.go rename pipeline/{ => runtime}/shutdown.go (98%) rename pipeline/{error_test.go => state/state.go} (57%) rename pipeline/{ => tracing}/tracer.go (83%) diff --git a/agent/logger.go b/agent/logger.go index 89ced9c71..528b8830c 100644 --- a/agent/logger.go +++ b/agent/logger.go @@ -23,11 +23,12 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/agent/log" "go.woodpecker-ci.org/woodpecker/v3/pipeline" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/logging" pipeline_utils "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils" "go.woodpecker-ci.org/woodpecker/v3/rpc" ) -func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger { +func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) logging.Logger { return func(step *backend.Step, rc io.ReadCloser) error { defer rc.Close() diff --git a/agent/runner.go b/agent/runner.go index 9d3e205d5..900221588 100644 --- a/agent/runner.go +++ b/agent/runner.go @@ -25,8 +25,9 @@ import ( "github.com/rs/zerolog/log" "google.golang.org/grpc/metadata" - "go.woodpecker-ci.org/woodpecker/v3/pipeline" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" + pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" + pipeline_runtime "go.woodpecker-ci.org/woodpecker/v3/pipeline/runtime" "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/shared/constant" "go.woodpecker-ci.org/woodpecker/v3/shared/utils" @@ -99,7 +100,7 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error { workflowCtx = utils.WithContextSigtermCallback(workflowCtx, func() { logger.Error().Msg("received sigterm termination signal") // WithContextSigtermCallback would cancel the context too, but we want our own custom error - cancelWorkflowCtx(pipeline.ErrCancel) + cancelWorkflowCtx(pipeline_errors.ErrCancel) }) // Listen for remote cancel events (UI / API). @@ -114,7 +115,7 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error { } else { if canceled { logger.Debug().Err(err).Msg("server side cancel signal received") - cancelWorkflowCtx(pipeline.ErrCancel) + cancelWorkflowCtx(pipeline_errors.ErrCancel) } // Wait returned without error, meaning the workflow finished normally logger.Debug().Msg("cancel listener exited normally") @@ -153,14 +154,14 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error { var uploads sync.WaitGroup // Run pipeline - err = pipeline.New( + err = pipeline_runtime.New( workflow.Config, - pipeline.WithContext(workflowCtx), - pipeline.WithTaskUUID(fmt.Sprint(workflow.ID)), - pipeline.WithLogger(r.createLogger(logger, &uploads, workflow)), - pipeline.WithTracer(r.createTracer(ctxMeta, &uploads, logger, workflow)), - pipeline.WithBackend(*r.backend), - pipeline.WithDescription(map[string]string{ + pipeline_runtime.WithContext(workflowCtx), + pipeline_runtime.WithTaskUUID(fmt.Sprint(workflow.ID)), + pipeline_runtime.WithLogger(r.createLogger(logger, &uploads, workflow)), + pipeline_runtime.WithTracer(r.createTracer(ctxMeta, &uploads, logger, workflow)), + pipeline_runtime.WithBackend(*r.backend), + pipeline_runtime.WithDescription(map[string]string{ "workflow_id": workflow.ID, "repo": repoName, "pipeline_number": pipelineNumber, @@ -171,10 +172,10 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error { if err != nil { state.Error = err.Error() - if errors.Is(err, pipeline.ErrCancel) { + if errors.Is(err, pipeline_errors.ErrCancel) { state.Canceled = true // cleanup joined error messages - state.Error = pipeline.ErrCancel.Error() + state.Error = pipeline_errors.ErrCancel.Error() } } diff --git a/agent/tracer.go b/agent/tracer.go index 69a9f6539..df4b0c814 100644 --- a/agent/tracer.go +++ b/agent/tracer.go @@ -24,12 +24,14 @@ import ( "github.com/rs/zerolog" - "go.woodpecker-ci.org/woodpecker/v3/pipeline" + pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/state" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing" "go.woodpecker-ci.org/woodpecker/v3/rpc" ) -func (r *Runner) createTracer(ctxMeta context.Context, uploads *sync.WaitGroup, logger zerolog.Logger, workflow *rpc.Workflow) pipeline.TraceFunc { - return func(state *pipeline.State) error { +func (r *Runner) createTracer(ctxMeta context.Context, uploads *sync.WaitGroup, logger zerolog.Logger, workflow *rpc.Workflow) tracing.TraceFunc { + return func(state *state.State) error { uploads.Add(1) defer uploads.Done() @@ -46,7 +48,7 @@ func (r *Runner) createTracer(ctxMeta context.Context, uploads *sync.WaitGroup, Exited: state.Process.Exited, ExitCode: state.Process.ExitCode, Started: state.Process.Started, - Canceled: errors.Is(state.Process.Error, pipeline.ErrCancel), + Canceled: errors.Is(state.Process.Error, pipeline_errors.ErrCancel), } if state.Process.Error != nil { stepState.Error = state.Process.Error.Error() diff --git a/cli/exec/exec.go b/cli/exec/exec.go index c96c5beb5..c338f435f 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -45,6 +45,9 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/matrix" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/logging" + pipeline_runtime "go.woodpecker-ci.org/woodpecker/v3/pipeline/runtime" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing" pipeline_utils "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils" "go.woodpecker-ci.org/woodpecker/v3/shared/constant" "go.woodpecker-ci.org/woodpecker/v3/shared/utils" @@ -318,12 +321,12 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax fmt.Printf("ctrl+c received, terminating current pipeline '%s'\n", confStr) }) - return pipeline.New(compiled, - pipeline.WithContext(pipelineCtx), //nolint:contextcheck - pipeline.WithTracer(pipeline.DefaultTracer), - pipeline.WithLogger(defaultLogger), - pipeline.WithBackend(backendEngine), - pipeline.WithDescription(map[string]string{ + return pipeline_runtime.New(compiled, + pipeline_runtime.WithContext(pipelineCtx), //nolint:contextcheck + pipeline_runtime.WithTracer(tracing.DefaultTracer), + pipeline_runtime.WithLogger(defaultLogger), + pipeline_runtime.WithBackend(backendEngine), + pipeline_runtime.WithDescription(map[string]string{ "CLI": "exec", }), ).Run(ctx) @@ -348,7 +351,7 @@ func convertPathForWindows(path string) string { return filepath.ToSlash(path) } -var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error { +var defaultLogger = logging.Logger(func(step *backend_types.Step, rc io.ReadCloser) error { logWriter := NewLineWriter(step.Name, step.UUID) return pipeline_utils.CopyLineByLine(logWriter, rc, pipeline.MaxLogLineLength) }) diff --git a/cmd/server/openapi/docs.go b/cmd/server/openapi/docs.go index 5f619bced..c5c78746b 100644 --- a/cmd/server/openapi/docs.go +++ b/cmd/server/openapi/docs.go @@ -5017,7 +5017,7 @@ const docTemplate = `{ "errors": { "type": "array", "items": { - "$ref": "#/definitions/types.PipelineError" + "$ref": "#/definitions/errors.PipelineError" } }, "event": { @@ -5736,6 +5736,52 @@ const docTemplate = `{ "EventManual" ] }, + "errors.PipelineError": { + "type": "object", + "properties": { + "data": {}, + "is_warning": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "type": { + "$ref": "#/definitions/errors.PipelineErrorType" + } + } + }, + "errors.PipelineErrorType": { + "type": "string", + "enum": [ + "linter", + "deprecation", + "compiler", + "generic", + "bad_habit" + ], + "x-enum-comments": { + "PipelineErrorTypeBadHabit": "some bad-habit error", + "PipelineErrorTypeCompiler": "some error with the config semantics", + "PipelineErrorTypeDeprecation": "using some deprecated feature", + "PipelineErrorTypeGeneric": "some generic error", + "PipelineErrorTypeLinter": "some error with the config syntax" + }, + "x-enum-descriptions": [ + "some error with the config syntax", + "using some deprecated feature", + "some error with the config semantics", + "some generic error", + "some bad-habit error" + ], + "x-enum-varnames": [ + "PipelineErrorTypeLinter", + "PipelineErrorTypeDeprecation", + "PipelineErrorTypeCompiler", + "PipelineErrorTypeGeneric", + "PipelineErrorTypeBadHabit" + ] + }, "metadata.Author": { "type": "object", "properties": { @@ -6156,52 +6202,6 @@ const docTemplate = `{ "$ref": "#/definitions/StatusValue" } } - }, - "types.PipelineError": { - "type": "object", - "properties": { - "data": {}, - "is_warning": { - "type": "boolean" - }, - "message": { - "type": "string" - }, - "type": { - "$ref": "#/definitions/types.PipelineErrorType" - } - } - }, - "types.PipelineErrorType": { - "type": "string", - "enum": [ - "linter", - "deprecation", - "compiler", - "generic", - "bad_habit" - ], - "x-enum-comments": { - "PipelineErrorTypeBadHabit": "some bad-habit error", - "PipelineErrorTypeCompiler": "some error with the config semantics", - "PipelineErrorTypeDeprecation": "using some deprecated feature", - "PipelineErrorTypeGeneric": "some generic error", - "PipelineErrorTypeLinter": "some error with the config syntax" - }, - "x-enum-descriptions": [ - "some error with the config syntax", - "using some deprecated feature", - "some error with the config semantics", - "some generic error", - "some bad-habit error" - ], - "x-enum-varnames": [ - "PipelineErrorTypeLinter", - "PipelineErrorTypeDeprecation", - "PipelineErrorTypeCompiler", - "PipelineErrorTypeGeneric", - "PipelineErrorTypeBadHabit" - ] } } }` diff --git a/pipeline/errors/error.go b/pipeline/errors/error.go deleted file mode 100644 index 4adf7fc62..000000000 --- a/pipeline/errors/error.go +++ /dev/null @@ -1,71 +0,0 @@ -package errors - -import ( - "errors" - - "go.uber.org/multierr" - - "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" -) - -type LinterErrorData struct { - File string `json:"file"` - Field string `json:"field"` -} - -type DeprecationErrorData struct { - File string `json:"file"` - Field string `json:"field"` - Docs string `json:"docs"` -} - -type BadHabitErrorData struct { - File string `json:"file"` - Field string `json:"field"` - Docs string `json:"docs"` -} - -func GetLinterData(e *types.PipelineError) *LinterErrorData { - if e.Type != types.PipelineErrorTypeLinter { - return nil - } - - if data, ok := e.Data.(*LinterErrorData); ok { - return data - } - - return nil -} - -func GetPipelineErrors(err error) []*types.PipelineError { - var pipelineErrors []*types.PipelineError - for _, _err := range multierr.Errors(err) { - var err *types.PipelineError - if errors.As(_err, &err) { - pipelineErrors = append(pipelineErrors, err) - } else { - pipelineErrors = append(pipelineErrors, &types.PipelineError{ - Message: _err.Error(), - Type: types.PipelineErrorTypeGeneric, - }) - } - } - - return pipelineErrors -} - -func HasBlockingErrors(err error) bool { - if err == nil { - return false - } - - errs := GetPipelineErrors(err) - - for _, err := range errs { - if !err.IsWarning { - return true - } - } - - return false -} diff --git a/pipeline/errors/linter.go b/pipeline/errors/linter.go new file mode 100644 index 000000000..9bd7d5d3b --- /dev/null +++ b/pipeline/errors/linter.go @@ -0,0 +1,83 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package errors + +import ( + "errors" + + "go.uber.org/multierr" +) + +type LinterErrorData struct { + File string `json:"file"` + Field string `json:"field"` +} + +type DeprecationErrorData struct { + File string `json:"file"` + Field string `json:"field"` + Docs string `json:"docs"` +} + +type BadHabitErrorData struct { + File string `json:"file"` + Field string `json:"field"` + Docs string `json:"docs"` +} + +func GetLinterData(e *PipelineError) *LinterErrorData { + if e.Type != PipelineErrorTypeLinter { + return nil + } + + if data, ok := e.Data.(*LinterErrorData); ok { + return data + } + + return nil +} + +func GetPipelineErrors(err error) []*PipelineError { + var pipelineErrors []*PipelineError + for _, _err := range multierr.Errors(err) { + var err *PipelineError + if errors.As(_err, &err) { + pipelineErrors = append(pipelineErrors, err) + } else { + pipelineErrors = append(pipelineErrors, &PipelineError{ + Message: _err.Error(), + Type: PipelineErrorTypeGeneric, + }) + } + } + + return pipelineErrors +} + +func HasBlockingErrors(err error) bool { + if err == nil { + return false + } + + errs := GetPipelineErrors(err) + + for _, err := range errs { + if !err.IsWarning { + return true + } + } + + return false +} diff --git a/pipeline/errors/error_test.go b/pipeline/errors/linter_test.go similarity index 60% rename from pipeline/errors/error_test.go rename to pipeline/errors/linter_test.go index a1658a797..df6a72bec 100644 --- a/pipeline/errors/error_test.go +++ b/pipeline/errors/linter_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package errors_test import ( @@ -8,7 +22,6 @@ import ( "go.uber.org/multierr" pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" ) func TestGetPipelineErrors(t *testing.T) { @@ -17,7 +30,7 @@ func TestGetPipelineErrors(t *testing.T) { tests := []struct { title string err error - expected []*types.PipelineError + expected []*pipeline_errors.PipelineError }{ { title: "nil error", @@ -26,10 +39,10 @@ func TestGetPipelineErrors(t *testing.T) { }, { title: "warning", - err: &types.PipelineError{ + err: &pipeline_errors.PipelineError{ IsWarning: true, }, - expected: []*types.PipelineError{ + expected: []*pipeline_errors.PipelineError{ { IsWarning: true, }, @@ -37,10 +50,10 @@ func TestGetPipelineErrors(t *testing.T) { }, { title: "pipeline error", - err: &types.PipelineError{ + err: &pipeline_errors.PipelineError{ IsWarning: false, }, - expected: []*types.PipelineError{ + expected: []*pipeline_errors.PipelineError{ { IsWarning: false, }, @@ -49,14 +62,14 @@ func TestGetPipelineErrors(t *testing.T) { { title: "multiple warnings", err: multierr.Combine( - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: true, }, - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: true, }, ), - expected: []*types.PipelineError{ + expected: []*pipeline_errors.PipelineError{ { IsWarning: true, }, @@ -68,15 +81,15 @@ func TestGetPipelineErrors(t *testing.T) { { title: "multiple errors and warnings", err: multierr.Combine( - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: true, }, - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: false, }, errors.New("some error"), ), - expected: []*types.PipelineError{ + expected: []*pipeline_errors.PipelineError{ { IsWarning: true, }, @@ -84,7 +97,7 @@ func TestGetPipelineErrors(t *testing.T) { IsWarning: false, }, { - Type: types.PipelineErrorTypeGeneric, + Type: pipeline_errors.PipelineErrorTypeGeneric, IsWarning: false, Message: "some error", }, @@ -112,14 +125,14 @@ func TestHasBlockingErrors(t *testing.T) { }, { title: "warning", - err: &types.PipelineError{ + err: &pipeline_errors.PipelineError{ IsWarning: true, }, expected: false, }, { title: "pipeline error", - err: &types.PipelineError{ + err: &pipeline_errors.PipelineError{ IsWarning: false, }, expected: true, @@ -127,10 +140,10 @@ func TestHasBlockingErrors(t *testing.T) { { title: "multiple warnings", err: multierr.Combine( - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: true, }, - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: true, }, ), @@ -139,10 +152,10 @@ func TestHasBlockingErrors(t *testing.T) { { title: "multiple errors and warnings", err: multierr.Combine( - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: true, }, - &types.PipelineError{ + &pipeline_errors.PipelineError{ IsWarning: false, }, errors.New("some error"), diff --git a/pipeline/errors/types/errors.go b/pipeline/errors/pipeline.go similarity index 66% rename from pipeline/errors/types/errors.go rename to pipeline/errors/pipeline.go index caa85c08e..ad8351e07 100644 --- a/pipeline/errors/types/errors.go +++ b/pipeline/errors/pipeline.go @@ -1,4 +1,18 @@ -package types +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package errors import ( "fmt" diff --git a/pipeline/error.go b/pipeline/errors/runtime.go similarity index 98% rename from pipeline/error.go rename to pipeline/errors/runtime.go index d2b39c335..774fc79cb 100644 --- a/pipeline/error.go +++ b/pipeline/errors/runtime.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package errors import ( "errors" diff --git a/pipeline/frontend/yaml/linter/error.go b/pipeline/frontend/yaml/linter/error.go index 96c96ddc0..b267773c9 100644 --- a/pipeline/frontend/yaml/linter/error.go +++ b/pipeline/frontend/yaml/linter/error.go @@ -15,15 +15,14 @@ package linter import ( - "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" - errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" + pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" ) -func newLinterError(message, file, field string, isWarning bool) *errorTypes.PipelineError { - return &errorTypes.PipelineError{ - Type: errorTypes.PipelineErrorTypeLinter, +func newLinterError(message, file, field string, isWarning bool) *pipeline_errors.PipelineError { + return &pipeline_errors.PipelineError{ + Type: pipeline_errors.PipelineErrorTypeLinter, Message: message, - Data: &errors.LinterErrorData{File: file, Field: field}, + Data: &pipeline_errors.LinterErrorData{File: file, Field: field}, IsWarning: isWarning, } } diff --git a/pipeline/frontend/yaml/linter/linter.go b/pipeline/frontend/yaml/linter/linter.go index 4b991dc41..0a560dfd3 100644 --- a/pipeline/frontend/yaml/linter/linter.go +++ b/pipeline/frontend/yaml/linter/linter.go @@ -20,8 +20,7 @@ import ( "codeberg.org/6543/xyaml" "go.uber.org/multierr" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" - errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" + pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter/schema" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/types" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/utils" @@ -345,10 +344,10 @@ func (l *Linter) lintBadHabits(config *WorkflowConfig) (err error) { } } if field != "" { - err = multierr.Append(err, &errorTypes.PipelineError{ - Type: errorTypes.PipelineErrorTypeBadHabit, + err = multierr.Append(err, &pipeline_errors.PipelineError{ + Type: pipeline_errors.PipelineErrorTypeBadHabit, Message: "Set an event filter for all steps or the entire workflow on all items of the `when` block", - Data: errors.BadHabitErrorData{ + Data: pipeline_errors.BadHabitErrorData{ File: config.File, Field: field, Docs: "https://woodpecker-ci.org/docs/usage/linter#event-filter-for-all-steps", diff --git a/pipeline/frontend/yaml/matrix/matrix.go b/pipeline/frontend/yaml/matrix/matrix.go index a48ad31d8..7616f07fb 100644 --- a/pipeline/frontend/yaml/matrix/matrix.go +++ b/pipeline/frontend/yaml/matrix/matrix.go @@ -19,7 +19,7 @@ import ( "codeberg.org/6543/xyaml" - errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" + pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" ) const ( @@ -116,7 +116,7 @@ func parse(raw []byte) (Matrix, error) { Matrix map[string][]string }{} if err := xyaml.Unmarshal(raw, &data); err != nil { - return nil, &errorTypes.PipelineError{Message: err.Error(), Type: errorTypes.PipelineErrorTypeCompiler} + return nil, &pipeline_errors.PipelineError{Message: err.Error(), Type: pipeline_errors.PipelineErrorTypeCompiler} } return data.Matrix, nil } @@ -129,7 +129,7 @@ func parseList(raw []byte) ([]Axis, error) { }{} if err := xyaml.Unmarshal(raw, &data); err != nil { - return nil, &errorTypes.PipelineError{Message: err.Error(), Type: errorTypes.PipelineErrorTypeCompiler} + return nil, &pipeline_errors.PipelineError{Message: err.Error(), Type: pipeline_errors.PipelineErrorTypeCompiler} } return data.Matrix.Include, nil } diff --git a/pipeline/logger.go b/pipeline/logging/logger.go similarity index 97% rename from pipeline/logger.go rename to pipeline/logging/logger.go index 765a0333b..e7e55f24e 100644 --- a/pipeline/logger.go +++ b/pipeline/logging/logger.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package logging import ( "io" diff --git a/pipeline/pipeline.go b/pipeline/runtime/executor.go similarity index 78% rename from pipeline/pipeline.go rename to pipeline/runtime/executor.go index b89befefd..edf1b0d19 100644 --- a/pipeline/pipeline.go +++ b/pipeline/runtime/executor.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package runtime import ( "context" @@ -21,79 +21,14 @@ import ( "sync" "time" - "github.com/oklog/ulid/v2" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" - pipelineErrors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" + pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/state" ) -// TODO: move runtime into "runtime" subpackage - -type ( - // State defines the pipeline and process state. - State struct { - // Global state of the pipeline. - Pipeline struct { - // Pipeline time started - Started int64 `json:"time"` - // Current pipeline step - Step *backend.Step `json:"step"` - // Current pipeline error state - Error error `json:"error"` - } - - // Current process state. - Process backend.State - } -) - -// Runtime represents a workflow state executed by a specific backend. -// Each workflow gets its own state configuration at runtime. -type Runtime struct { - err error - spec *backend.Config - engine backend.Backend - started int64 - - // The context a workflow is being executed with. - // All normal (non cleanup) operations must use this. - // Cleanup operations should use the runnerCtx passed to Run() - ctx context.Context - - tracer Tracer - logger Logger - - taskUUID string - - Description map[string]string // The runtime descriptors. -} - -// New returns a new runtime using the specified runtime -// configuration and runtime engine. -func New(spec *backend.Config, opts ...Option) *Runtime { - r := new(Runtime) - r.Description = map[string]string{} - r.spec = spec - r.ctx = context.Background() - r.taskUUID = ulid.Make().String() - for _, opts := range opts { - opts(r) - } - return r -} - -func (r *Runtime) MakeLogger() zerolog.Logger { - logCtx := log.With() - for key, val := range r.Description { - logCtx = logCtx.Str(key, val) - } - return logCtx.Logger() -} - // Run starts the execution of a workflow and waits for it to complete. func (r *Runtime) Run(runnerCtx context.Context) error { logger := r.MakeLogger() @@ -122,9 +57,9 @@ func (r *Runtime) Run(runnerCtx context.Context) error { r.started = time.Now().Unix() if err := r.engine.SetupWorkflow(runnerCtx, r.spec, r.taskUUID); err != nil { - var stepErr *pipelineErrors.ErrInvalidWorkflowSetup + var stepErr *pipeline_errors.ErrInvalidWorkflowSetup if errors.As(err, &stepErr) { - state := new(State) + state := new(state.State) state.Pipeline.Step = stepErr.Step state.Pipeline.Error = stepErr.Err state.Process = backend.State{ @@ -147,7 +82,7 @@ func (r *Runtime) Run(runnerCtx context.Context) error { for _, stage := range r.spec.Stages { select { case <-r.ctx.Done(): - return ErrCancel + return pipeline_errors.ErrCancel case err := <-r.execAll(runnerCtx, stage.Steps): if err != nil { r.err = err @@ -167,7 +102,7 @@ func (r *Runtime) traceStep(processState *backend.State, err error, step *backen return nil } - state := new(State) + state := new(state.State) state.Pipeline.Started = r.started state.Pipeline.Step = step state.Pipeline.Error = r.err @@ -244,7 +179,7 @@ func (r *Runtime) execAll(runnerCtx context.Context, steps []*backend.Step) <-ch // normalize context cancel error if errors.Is(err, context.Canceled) { - err = ErrCancel + err = pipeline_errors.ErrCancel } // Return the error after tracing it. @@ -326,7 +261,7 @@ func (r *Runtime) exec(runnerCtx context.Context, step *backend.Step, setupWg *s waitState, err := r.engine.WaitStep(r.ctx, step, r.taskUUID) //nolint:contextcheck if err != nil { if errors.Is(err, context.Canceled) { - waitState.Error = ErrCancel + waitState.Error = pipeline_errors.ErrCancel } else { return nil, err } @@ -343,16 +278,16 @@ func (r *Runtime) exec(runnerCtx context.Context, step *backend.Step, setupWg *s // we handle cancel case if ctxErr := r.ctx.Err(); ctxErr != nil && errors.Is(ctxErr, context.Canceled) { - waitState.Error = ErrCancel + waitState.Error = pipeline_errors.ErrCancel } if waitState.OOMKilled { - return waitState, &OomError{ + return waitState, &pipeline_errors.OomError{ UUID: step.UUID, Code: waitState.ExitCode, } } else if waitState.ExitCode != 0 { - return waitState, &ExitError{ + return waitState, &pipeline_errors.ExitError{ UUID: step.UUID, Code: waitState.ExitCode, } diff --git a/pipeline/option.go b/pipeline/runtime/option.go similarity index 87% rename from pipeline/option.go rename to pipeline/runtime/option.go index 784cfdbbd..f87d4a108 100644 --- a/pipeline/option.go +++ b/pipeline/runtime/option.go @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package runtime import ( "context" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/logging" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing" ) // Option configures a runtime option. @@ -31,14 +33,14 @@ func WithBackend(backend backend.Backend) Option { } // WithLogger returns an option configured with a runtime logger. -func WithLogger(logger Logger) Option { +func WithLogger(logger logging.Logger) Option { return func(r *Runtime) { r.logger = logger } } // WithTracer returns an option configured with a runtime tracer. -func WithTracer(tracer Tracer) Option { +func WithTracer(tracer tracing.Tracer) Option { return func(r *Runtime) { r.tracer = tracer } diff --git a/pipeline/runtime/runtime.go b/pipeline/runtime/runtime.go new file mode 100644 index 000000000..5fb197b4a --- /dev/null +++ b/pipeline/runtime/runtime.go @@ -0,0 +1,70 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + + "github.com/oklog/ulid/v2" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/logging" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/tracing" +) + +// Runtime represents a workflow state executed by a specific backend. +// Each workflow gets its own state configuration at runtime. +type Runtime struct { + err error + spec *backend.Config + engine backend.Backend + started int64 + + // The context a workflow is being executed with. + // All normal (non cleanup) operations must use this. + // Cleanup operations should use the runnerCtx passed to Run() + ctx context.Context + + tracer tracing.Tracer + logger logging.Logger + + taskUUID string + + Description map[string]string // The runtime descriptors. +} + +// New returns a new runtime using the specified runtime +// configuration and runtime engine. +func New(spec *backend.Config, opts ...Option) *Runtime { + r := new(Runtime) + r.Description = map[string]string{} + r.spec = spec + r.ctx = context.Background() + r.taskUUID = ulid.Make().String() + for _, opts := range opts { + opts(r) + } + return r +} + +func (r *Runtime) MakeLogger() zerolog.Logger { + logCtx := log.With() + for key, val := range r.Description { + logCtx = logCtx.Str(key, val) + } + return logCtx.Logger() +} diff --git a/pipeline/shutdown.go b/pipeline/runtime/shutdown.go similarity index 98% rename from pipeline/shutdown.go rename to pipeline/runtime/shutdown.go index 9479ab971..d53cf1c0d 100644 --- a/pipeline/shutdown.go +++ b/pipeline/runtime/shutdown.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package runtime import ( "context" diff --git a/pipeline/error_test.go b/pipeline/state/state.go similarity index 57% rename from pipeline/error_test.go rename to pipeline/state/state.go index 95a264f2e..6a496700e 100644 --- a/pipeline/error_test.go +++ b/pipeline/state/state.go @@ -12,25 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package state import ( - "testing" - - "github.com/stretchr/testify/assert" + backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" ) -func TestExitError(t *testing.T) { - err := ExitError{ - UUID: "14534321", - Code: 255, - } - assert.Equal(t, "uuid=14534321: exit code 255", err.Error()) -} +type ( + // State defines the pipeline and process state. + State struct { + // Global state of the pipeline. + Pipeline struct { + // Pipeline time started + Started int64 `json:"time"` + // Current pipeline step + Step *backend.Step `json:"step"` + // Current pipeline error state + Error error `json:"error"` + } -func TestOomError(t *testing.T) { - err := OomError{ - UUID: "14534321", + // Current process state. + Process backend.State } - assert.Equal(t, "uuid=14534321: received oom kill", err.Error()) -} +) diff --git a/pipeline/tracer.go b/pipeline/tracing/tracer.go similarity index 83% rename from pipeline/tracer.go rename to pipeline/tracing/tracer.go index f48a7c562..77b9f3a7d 100644 --- a/pipeline/tracer.go +++ b/pipeline/tracing/tracer.go @@ -12,30 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package tracing import ( "strconv" + + "go.woodpecker-ci.org/woodpecker/v3/pipeline/state" ) // Tracer handles process tracing. type Tracer interface { - Trace(*State) error + Trace(*state.State) error } // TraceFunc type is an adapter to allow the use of ordinary // functions as a Tracer. -type TraceFunc func(*State) error +type TraceFunc func(*state.State) error // Trace calls f(state). -func (f TraceFunc) Trace(state *State) error { +func (f TraceFunc) Trace(state *state.State) error { return f(state) } // DefaultTracer provides a tracer that updates the CI_ environment // variables to include the correct timestamp and status. // TODO: find either a new home or better name for this. -var DefaultTracer = TraceFunc(func(state *State) error { +var DefaultTracer = TraceFunc(func(state *state.State) error { if state.Process.Exited { return nil } diff --git a/server/model/pipeline.go b/server/model/pipeline.go index a3707b690..4de1c69bd 100644 --- a/server/model/pipeline.go +++ b/server/model/pipeline.go @@ -16,46 +16,46 @@ package model import ( - "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" ) type Pipeline struct { - ID int64 `json:"id" xorm:"pk autoincr 'id'"` - RepoID int64 `json:"-" xorm:"UNIQUE(s) INDEX 'repo_id'"` - Number int64 `json:"number" xorm:"UNIQUE(s) 'number'"` - Author string `json:"author" xorm:"INDEX 'author'"` - Parent int64 `json:"parent" xorm:"parent"` - Event WebhookEvent `json:"event" xorm:"event"` - EventReason []string `json:"event_reason" xorm:"json 'event_reason'"` - Status StatusValue `json:"status" xorm:"INDEX 'status'"` - Errors []*types.PipelineError `json:"errors" xorm:"json 'errors'"` - Created int64 `json:"created" xorm:"'created' NOT NULL DEFAULT 0 created"` - Updated int64 `json:"updated" xorm:"'updated' NOT NULL DEFAULT 0 updated"` - Started int64 `json:"started" xorm:"started"` - Finished int64 `json:"finished" xorm:"finished"` - DeployTo string `json:"deploy_to" xorm:"deploy"` - DeployTask string `json:"deploy_task" xorm:"deploy_task"` - Commit string `json:"commit" xorm:"commit"` - Branch string `json:"branch" xorm:"branch"` - Ref string `json:"ref" xorm:"ref"` - Refspec string `json:"refspec" xorm:"refspec"` - Title string `json:"title" xorm:"title"` - Message string `json:"message" xorm:"TEXT 'message'"` - Timestamp int64 `json:"timestamp" xorm:"'timestamp'"` - Sender string `json:"sender" xorm:"sender"` // uses reported user for webhooks and name of cron for cron pipelines - Avatar string `json:"author_avatar" xorm:"varchar(500) avatar"` - Email string `json:"author_email" xorm:"varchar(500) email"` - ForgeURL string `json:"forge_url" xorm:"forge_url"` - Reviewer string `json:"reviewed_by" xorm:"reviewer"` - Reviewed int64 `json:"reviewed" xorm:"reviewed"` - CancelInfo *CancelInfo `json:"cancel_info,omitempty" xorm:"json 'cancel_info'"` - Workflows []*Workflow `json:"workflows,omitempty" xorm:"-"` - ChangedFiles []string `json:"changed_files,omitempty" xorm:"LONGTEXT 'changed_files'"` - AdditionalVariables map[string]string `json:"variables,omitempty" xorm:"json 'additional_variables'"` - PullRequestLabels []string `json:"pr_labels,omitempty" xorm:"json 'pr_labels'"` - PullRequestMilestone string `json:"pr_milestone,omitempty" xorm:"pr_milestone"` - IsPrerelease bool `json:"is_prerelease,omitempty" xorm:"is_prerelease"` - FromFork bool `json:"from_fork,omitempty" xorm:"from_fork"` + ID int64 `json:"id" xorm:"pk autoincr 'id'"` + RepoID int64 `json:"-" xorm:"UNIQUE(s) INDEX 'repo_id'"` + Number int64 `json:"number" xorm:"UNIQUE(s) 'number'"` + Author string `json:"author" xorm:"INDEX 'author'"` + Parent int64 `json:"parent" xorm:"parent"` + Event WebhookEvent `json:"event" xorm:"event"` + EventReason []string `json:"event_reason" xorm:"json 'event_reason'"` + Status StatusValue `json:"status" xorm:"INDEX 'status'"` + Errors []*errors.PipelineError `json:"errors" xorm:"json 'errors'"` + Created int64 `json:"created" xorm:"'created' NOT NULL DEFAULT 0 created"` + Updated int64 `json:"updated" xorm:"'updated' NOT NULL DEFAULT 0 updated"` + Started int64 `json:"started" xorm:"started"` + Finished int64 `json:"finished" xorm:"finished"` + DeployTo string `json:"deploy_to" xorm:"deploy"` + DeployTask string `json:"deploy_task" xorm:"deploy_task"` + Commit string `json:"commit" xorm:"commit"` + Branch string `json:"branch" xorm:"branch"` + Ref string `json:"ref" xorm:"ref"` + Refspec string `json:"refspec" xorm:"refspec"` + Title string `json:"title" xorm:"title"` + Message string `json:"message" xorm:"TEXT 'message'"` + Timestamp int64 `json:"timestamp" xorm:"'timestamp'"` + Sender string `json:"sender" xorm:"sender"` // uses reported user for webhooks and name of cron for cron pipelines + Avatar string `json:"author_avatar" xorm:"varchar(500) avatar"` + Email string `json:"author_email" xorm:"varchar(500) email"` + ForgeURL string `json:"forge_url" xorm:"forge_url"` + Reviewer string `json:"reviewed_by" xorm:"reviewer"` + Reviewed int64 `json:"reviewed" xorm:"reviewed"` + CancelInfo *CancelInfo `json:"cancel_info,omitempty" xorm:"json 'cancel_info'"` + Workflows []*Workflow `json:"workflows,omitempty" xorm:"-"` + ChangedFiles []string `json:"changed_files,omitempty" xorm:"LONGTEXT 'changed_files'"` + AdditionalVariables map[string]string `json:"variables,omitempty" xorm:"json 'additional_variables'"` + PullRequestLabels []string `json:"pr_labels,omitempty" xorm:"json 'pr_labels'"` + PullRequestMilestone string `json:"pr_milestone,omitempty" xorm:"pr_milestone"` + IsPrerelease bool `json:"is_prerelease,omitempty" xorm:"is_prerelease"` + FromFork bool `json:"from_fork,omitempty" xorm:"from_fork"` } // @name Pipeline // TableName return database table name for xorm. diff --git a/server/pipeline/stepbuilder/step_builder.go b/server/pipeline/stepbuilder/step_builder.go index ab690ee1f..f98ca4f74 100644 --- a/server/pipeline/stepbuilder/step_builder.go +++ b/server/pipeline/stepbuilder/step_builder.go @@ -29,7 +29,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline" backend_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" - errorTypes "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors/types" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" @@ -139,7 +138,7 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A // parse yaml pipeline parsed, err := yaml.ParseString(substituted) if err != nil { - return nil, &errorTypes.PipelineError{Message: err.Error(), Type: errorTypes.PipelineErrorTypeCompiler} + return nil, &pipeline_errors.PipelineError{Message: err.Error(), Type: pipeline_errors.PipelineErrorTypeCompiler} } // lint pipeline