From 0fe5bc1b44145803c2d16e8bbdf5da77dd458422 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 26 May 2026 23:04:41 +0200 Subject: [PATCH] Use refactored pipeline builder in cli exec (#6453) based on #3967 --- cli/exec/exec.go | 279 ++++++++++++++++---------------------- cli/exec/exec_test.go | 100 ++++++++++++++ cli/exec/metadata.go | 6 +- cli/exec/metadata_test.go | 10 +- 4 files changed, 220 insertions(+), 175 deletions(-) create mode 100644 cli/exec/exec_test.go diff --git a/cli/exec/exec.go b/cli/exec/exec.go index 39039bebe..f572ab0a1 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -18,18 +18,14 @@ import ( "context" "fmt" "io" - "maps" "os" - "path" "path/filepath" "runtime" "slices" "strings" "codeberg.org/6543/xyaml" - "github.com/drone/envsubst" "github.com/oklog/ulid/v2" - "github.com/rs/zerolog/log" "github.com/urfave/cli/v3" "go.uber.org/multierr" @@ -41,11 +37,9 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/kubernetes" "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/local" backend_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/matrix" "go.woodpecker-ci.org/woodpecker/v3/pipeline/logging" pipeline_runtime "go.woodpecker-ci.org/woodpecker/v3/pipeline/runtime" pipeline_utils "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils" @@ -72,6 +66,7 @@ func run(ctx context.Context, c *cli.Command) error { return common.RunPipelineFunc(ctx, c, execFile, execDir) } +// TODO: do parallel runs with output to multiple _windows_ e.g. tmux like func execDir(ctx context.Context, c *cli.Command, dir string) error { // TODO: respect pipeline dependency repoPath := c.String("repo-path") @@ -84,34 +79,25 @@ func execDir(ctx context.Context, c *cli.Command, dir string) error { repoPath = convertPathForWindows(repoPath) } - var execErr error - - // TODO: respect depends_on and do parallel runs with output to multiple _windows_ e.g. tmux like + var yamls []*builder.YamlFile walkErr := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } - - // check if it is a regular file (not dir) if info.Mode().IsRegular() && (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) { - fmt.Println("#", info.Name()) - err := runExec(ctx, c, path, repoPath, false) + dat, err := os.ReadFile(path) if err != nil { - fmt.Print(err) - execErr = multierr.Append(execErr, err) + return err } - fmt.Println("") - return nil + yamls = append(yamls, &builder.YamlFile{Name: path, Data: dat}) } - return nil }) - if walkErr != nil { return walkErr } - return execErr + return runExec(ctx, c, yamls, repoPath) } func execFile(ctx context.Context, c *cli.Command, file string) error { @@ -124,185 +110,137 @@ func execFile(ctx context.Context, c *cli.Command, file string) error { if runtime.GOOS == "windows" && c.String("backend-engine") != "local" { repoPath = convertPathForWindows(repoPath) } - return runExec(ctx, c, file, repoPath, true) -} -func runExec(ctx context.Context, c *cli.Command, file, repoPath string, singleExec bool) error { dat, err := os.ReadFile(file) if err != nil { return err } + return runExec(ctx, c, []*builder.YamlFile{{Name: file, Data: dat}}, repoPath) +} +func runExec(ctx context.Context, c *cli.Command, yamls []*builder.YamlFile, repoPath string) error { // if we use the local backend we should signal to run at $repoPath if c.String("backend-engine") == "local" { local.CLIWorkaroundExecAtDir = repoPath } - axes, err := matrix.ParseString(string(dat)) - if err != nil { - return fmt.Errorf("parse matrix fail") - } - - if len(axes) == 0 { - axes = append(axes, matrix.Axis{}) - } - for _, axis := range axes { - err := execWithAxis(ctx, c, file, repoPath, axis, singleExec) - if err != nil { - return err - } - } - return nil -} - -func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis, singleExec bool) error { - metadataWorkflow := &metadata.Workflow{} - if !singleExec { - // TODO: proper try to use the engine to generate the same metadata for workflows - // https://github.com/woodpecker-ci/woodpecker/pull/3967 - metadataWorkflow.Name = strings.TrimSuffix(strings.TrimSuffix(file, ".yaml"), ".yml") - } - metadata, err := metadataFromContext(ctx, c, axis, metadataWorkflow) - if err != nil { - return fmt.Errorf("could not create metadata: %w", err) - } else if metadata == nil { - return fmt.Errorf("metadata is nil") - } - - environ := metadata.Environ() - maps.Copy(environ, metadata.Workflow.Matrix) + // collect secrets from flags var secrets []compiler.Secret for key, val := range c.StringMap("secrets") { - secrets = append(secrets, compiler.Secret{ - Name: key, - Value: val, - }) + secrets = append(secrets, compiler.Secret{Name: key, Value: val}) } if secretsFile := c.String("secrets-file"); secretsFile != "" { fileContent, err := os.ReadFile(secretsFile) if err != nil { return err } - var m map[string]string - err = xyaml.Unmarshal(fileContent, &m) - if err != nil { + if err := xyaml.Unmarshal(fileContent, &m); err != nil { return err } - for key, val := range m { - secrets = append(secrets, compiler.Secret{ - Name: key, - Value: val, - }) + secrets = append(secrets, compiler.Secret{Name: key, Value: val}) } } + // collect extra env vars from --env flags pipelineEnv := make(map[string]string) for _, env := range c.StringSlice("env") { before, after, _ := strings.Cut(env, "=") pipelineEnv[before] = after - if oldVar, exists := environ[before]; exists { - // override existing values, but print a warning - log.Warn().Msgf("environment variable '%s' had value '%s', but got overwritten", before, oldVar) - } - environ[before] = after - } - - tmpl, err := envsubst.ParseFile(file) - if err != nil { - return err - } - confStr, err := tmpl.Execute(func(name string) string { - return environ[name] - }) - if err != nil { - return err - } - - conf, err := yaml.ParseString(confStr) - if err != nil { - return err - } - - // emulate server behavior https://github.com/woodpecker-ci/woodpecker/blob/eebaa10d104cbc3fa7ce4c0e344b0b7978405135/server/pipeline/stepbuilder/stepBuilder.go#L289-L295 - prefix := "wp_" + ulid.Make().String() - - // configure volumes for local execution - volumes := c.StringSlice("volumes") - if c.Bool("local") { - var ( - workspaceBase = conf.Workspace.Base - workspacePath = conf.Workspace.Path - ) - if workspaceBase == "" { - workspaceBase = c.String("workspace-base") - } - if workspacePath == "" { - workspacePath = c.String("workspace-path") - } - - volumes = append(volumes, prefix+"_default:"+workspaceBase) - volumes = append(volumes, repoPath+":"+path.Join(workspaceBase, workspacePath)) } privilegedPlugins := c.StringSlice("plugins-privileged") - // lint the yaml file - err = linter.New( - linter.WithTrusted(linter.TrustedConfiguration{ - Security: c.Bool("repo-trusted-security"), - Network: c.Bool("repo-trusted-network"), - Volumes: c.Bool("repo-trusted-volumes"), - }), - linter.PrivilegedPlugins(privilegedPlugins), - linter.WithTrustedClonePlugins(constant.TrustedClonePlugins), - ).Lint([]*linter.WorkflowConfig{{ - File: path.Base(file), - RawConfig: confStr, - Workflow: conf, - }}) - if err != nil { - str, err := lint.FormatLintError(file, err, false) - fmt.Print(str) - if err != nil { - return err - } - } + // emulate server prefix for volume/network naming + prefix := "wp_" + ulid.Make().String() - // compiles the yaml file - compiled, err := compiler.New( - compiler.WithEscalated( - privilegedPlugins..., - ), - compiler.WithVolumes(volumes...), - compiler.WithWorkspace( - c.String("workspace-base"), - c.String("workspace-path"), - ), - compiler.WithNetworks( - c.StringSlice("network")..., - ), + // build compiler options — mirrors server behavior + compilerOpts := []compiler.Option{ + compiler.WithEscalated(privilegedPlugins...), + compiler.WithNetworks(c.StringSlice("network")...), compiler.WithPrefix(prefix), compiler.WithProxy(compiler.ProxyOptions{ NoProxy: c.String("backend-no-proxy"), HTTPProxy: c.String("backend-http-proxy"), HTTPSProxy: c.String("backend-https-proxy"), }), - compiler.WithLocal( - c.Bool("local"), - ), + compiler.WithLocal(c.Bool("local")), compiler.WithNetrc( c.String("netrc-username"), c.String("netrc-password"), c.String("netrc-machine"), ), - compiler.WithMetadata(*metadata), compiler.WithSecret(secrets...), compiler.WithEnviron(pipelineEnv), - ).Compile(conf) + } + + // configure volumes for local execution + volumes := c.StringSlice("volumes") + if c.Bool("local") { + compilerOpts = append(compilerOpts, + compiler.WithWorkspace( + c.String("workspace-base"), + c.String("workspace-path"), + ), + ) + volumes = append(volumes, + prefix+"_default:"+c.String("workspace-base"), + repoPath+":"+c.String("workspace-base")+"/"+c.String("workspace-path"), + ) + } else { + compilerOpts = append(compilerOpts, + compiler.WithWorkspace( + c.String("workspace-base"), + c.String("workspace-path"), + ), + ) + } + compilerOpts = append(compilerOpts, compiler.WithVolumes(volumes...)) + + // build the metadata once — the CLI has a single pipeline context for all + // workflows, so every workflow gets the same metadata. + baseMetadata, err := metadataFromContext(ctx, c, nil) if err != nil { - return err + return fmt.Errorf("could not create metadata: %w", err) + } + + b := builder.PipelineBuilder{ + Yamls: yamls, + Envs: pipelineEnv, + RepoTrusted: &metadata.TrustedConfiguration{ + Network: c.Bool("repo-trusted-network"), + Volumes: c.Bool("repo-trusted-volumes"), + Security: c.Bool("repo-trusted-security"), + }, + TrustedClonePlugins: constant.TrustedClonePlugins, + PrivilegedPlugins: privilegedPlugins, + CompilerOptions: compilerOpts, + // GetWorkflowMetadata provides per-workflow metadata. In the CLI there + // is no server context, so we derive it from the base metadata and + // populate the workflow name/matrix from the builder.Workflow. + GetWorkflowMetadata: func(w *builder.Workflow) metadata.Metadata { + m := *baseMetadata + m.Workflow = metadata.Workflow{ + Name: w.Name, + Number: w.PID, + Matrix: w.Environ, + } + return m + }, + } + + items, err := b.Build() + if err != nil { + str, fmtErr := lint.FormatLintError("pipeline", err, false) + fmt.Print(str) + if fmtErr != nil { + return fmtErr + } + } + + if len(items) == 0 { + return fmt.Errorf("no workflows to execute (all filtered out)") } backendCtx := context.WithValue(ctx, backend_types.CliCommand, c) @@ -310,25 +248,36 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax if err != nil { return err } - if _, err = backendEngine.Load(backendCtx); err != nil { return err } - pipelineCtx, cancel := context.WithTimeout(context.Background(), c.Duration("timeout")) - defer cancel() - pipelineCtx = utils.WithContextSigtermCallback(pipelineCtx, func() { - fmt.Printf("ctrl+c received, terminating current pipeline '%s'\n", confStr) - }) + var execErr error + // TODO: respect depends_on and run in parallel where possible + for _, item := range items { + fmt.Println("#", item.Workflow.Name) - return pipeline_runtime.New( - compiled, backendEngine, - pipeline_runtime.WithContext(pipelineCtx), //nolint:contextcheck - pipeline_runtime.WithLogger(defaultLogger), - pipeline_runtime.WithDescription(map[string]string{ - "CLI": "exec", - }), - ).Run(ctx) + pipelineCtx, cancel := context.WithTimeout(context.Background(), c.Duration("timeout")) + defer cancel() + pipelineCtx = utils.WithContextSigtermCallback(pipelineCtx, func() { + fmt.Printf("ctrl+c received, terminating workflow '%s'\n", item.Workflow.Name) + }) + + err := pipeline_runtime.New( + item.Config, backendEngine, + pipeline_runtime.WithContext(pipelineCtx), //nolint:contextcheck + pipeline_runtime.WithLogger(defaultLogger), + pipeline_runtime.WithDescription(map[string]string{ + "CLI": "exec", + }), + ).Run(ctx) + if err != nil { + fmt.Println(err) + execErr = multierr.Append(execErr, err) + } + fmt.Println("") + } + return execErr } // convertPathForWindows converts a path to use slash separators diff --git a/cli/exec/exec_test.go b/cli/exec/exec_test.go new file mode 100644 index 000000000..27ab99d82 --- /dev/null +++ b/cli/exec/exec_test.go @@ -0,0 +1,100 @@ +// Copyright 2026 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build test + +package exec + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExecDummy(t *testing.T) { + repoDir := t.TempDir() + workflowPath := filepath.Join(repoDir, "workflow.yaml") + require.NoError(t, os.WriteFile(workflowPath, []byte(` +when: + - event: manual + +steps: + - name: build + image: alpine + commands: + - echo hello +`), 0o600)) + + // LineWriter writes to os.Stderr directly, so redirect the fd + oldStderr := os.Stderr + r, w, err := os.Pipe() + require.NoError(t, err) + os.Stderr = w + + t.Cleanup(func() { + os.Stderr = oldStderr + }) + + // This is important, else it will work on your system but if run in woodpecker, + // the exec will use the metadata the current test is running in. + clearEnv(t) + + err = Command.Run(t.Context(), []string{ + "woodpecker-cli", + "--backend-engine", "dummy", + "--repo-path", repoDir, + workflowPath, + }) + require.NoError(t, err) + + // close write end so Read below doesn't block + w.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, r) + require.NoError(t, err) + r.Close() + stdout := buf.String() + + assert.Contains(t, stdout, + `[build:L0:0s] StepName: build +[build:L1:0s] StepType: commands +[build:L2:0s] StepUUID: `, + ) + assert.Contains(t, stdout, + `[build:L3:0s] StepCommands: +[build:L4:0s] ------------------ +[build:L5:0s] echo hello +[build:L6:0s] ------------------`, + ) + + require.NoError(t, err) +} + +func clearEnv(t *testing.T) { + t.Helper() + osEnv := os.Environ() + t.Cleanup(func() { + for _, env := range osEnv { + k, v, _ := strings.Cut(env, "=") + _ = os.Setenv(k, v) //nolint:usetesting + } + }) + os.Clearenv() +} diff --git a/cli/exec/metadata.go b/cli/exec/metadata.go index a529c3617..ba3fbb82a 100644 --- a/cli/exec/metadata.go +++ b/cli/exec/metadata.go @@ -30,13 +30,9 @@ import ( ) // return the metadata from the cli context. -func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis, w *metadata.Workflow) (*metadata.Metadata, error) { +func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (*metadata.Metadata, error) { m := &metadata.Metadata{} - if w != nil { - m.Workflow = *w - } - if c.IsSet("metadata-file") { metadataFile, err := os.Open(c.String("metadata-file")) if err != nil { diff --git a/cli/exec/metadata_test.go b/cli/exec/metadata_test.go index ae28ca014..41eae3571 100644 --- a/cli/exec/metadata_test.go +++ b/cli/exec/metadata_test.go @@ -55,7 +55,7 @@ func TestMetadataFromContext(t *testing.T) { runCommand(flags, func(c *cli.Command) { _ = c.Set("metadata-file", tempFileName) - m, err := metadataFromContext(t.Context(), c, nil, nil) + m, err := metadataFromContext(t.Context(), c, nil) require.NoError(t, err) assert.Equal(t, "test-repo", m.Repo.Name) assert.Equal(t, int64(5), m.Curr.Number) @@ -76,7 +76,7 @@ func TestMetadataFromContext(t *testing.T) { _ = c.Set("repo-name", "aUser/override-repo") _ = c.Set("pipeline-number", "10") - m, err := metadataFromContext(t.Context(), c, nil, nil) + m, err := metadataFromContext(t.Context(), c, nil) require.NoError(t, err) assert.Equal(t, "override-repo", m.Repo.Name) assert.Equal(t, int64(10), m.Curr.Number) @@ -98,7 +98,7 @@ func TestMetadataFromContext(t *testing.T) { runCommand(flags, func(c *cli.Command) { _ = c.Set("metadata-file", tempFile.Name()) - _, err = metadataFromContext(t.Context(), c, nil, nil) + _, err = metadataFromContext(t.Context(), c, nil) assert.Error(t, err) }) }) @@ -110,7 +110,7 @@ func TestMetadataFromContext(t *testing.T) { } runCommand(flags, func(c *cli.Command) { - m, err := metadataFromContext(t.Context(), c, nil, nil) + m, err := metadataFromContext(t.Context(), c, nil) require.NoError(t, err) if assert.NotNil(t, m) { assert.Equal(t, "test", m.Repo.Owner) @@ -123,7 +123,7 @@ func TestMetadataFromContext(t *testing.T) { t.Run("MatrixAxis", func(t *testing.T) { runCommand([]cli.Flag{}, func(c *cli.Command) { axis := matrix.Axis{"go": "1.16", "os": "linux"} - m, err := metadataFromContext(t.Context(), c, axis, nil) + m, err := metadataFromContext(t.Context(), c, axis) require.NoError(t, err) assert.EqualValues(t, map[string]string{"go": "1.16", "os": "linux"}, m.Workflow.Matrix) })