diff --git a/server/pipeline/stepbuilder/stepBuilder.go b/pipeline/frontend/yaml/stepbuilder/step_builder.go similarity index 55% rename from server/pipeline/stepbuilder/stepBuilder.go rename to pipeline/frontend/yaml/stepbuilder/step_builder.go index 3d0b55e7e..f0b027132 100644 --- a/server/pipeline/stepbuilder/stepBuilder.go +++ b/pipeline/frontend/yaml/stepbuilder/step_builder.go @@ -18,7 +18,6 @@ package stepbuilder import ( "fmt" "maps" - "path/filepath" "strings" "github.com/oklog/ulid/v2" @@ -34,25 +33,21 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/matrix" yaml_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/types" - "go.woodpecker-ci.org/woodpecker/v3/server" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" ) // StepBuilder Takes the hook data and the yaml and returns in internal data model. type StepBuilder struct { - Repo *model.Repo - Curr *model.Pipeline - Prev *model.Pipeline - Netrc *model.Netrc - Secs []*model.Secret - Regs []*model.Registry - Host string - Yamls []*forge_types.FileMeta - Envs map[string]string - Forge metadata.ServerForge - DefaultLabels map[string]string - ProxyOpts compiler.ProxyOptions + Yamls []*forge_types.FileMeta + CompilerOptions []compiler.Option + WorkflowMetadataFunc func(*model.Workflow) metadata.Metadata + RepoTrusted *metadata.TrustedConfiguration + TrustedClonePlugins []string + PrivilegedPlugins []string + Host string + Envs map[string]string + DefaultLabels map[string]string } type Item struct { @@ -117,17 +112,12 @@ func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) { } func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.Axis, data string) (item *Item, errorsAndWarnings error) { - workflowMetadata := MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Prev, workflow, b.Host) - environ := b.environmentVariables(workflowMetadata, axis) + workflowMetadata := b.WorkflowMetadataFunc(workflow) - // add global environment variables for substituting - for k, v := range b.Envs { - if _, exists := environ[k]; exists { - // don't override existing values - continue - } - environ[k] = v - } + environ := map[string]string{} + maps.Copy(environ, b.Envs) // global environment data + maps.Copy(environ, workflowMetadata.Environ()) // workflow environment data like CI_REPO_NAME + maps.Copy(environ, axis) // substitute vars substituted, err := metadata.EnvVarSubst(data, environ) @@ -144,12 +134,12 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A // lint pipeline errorsAndWarnings = multierr.Append(errorsAndWarnings, linter.New( linter.WithTrusted(linter.TrustedConfiguration{ - Network: b.Repo.Trusted.Network, - Volumes: b.Repo.Trusted.Volumes, - Security: b.Repo.Trusted.Security, + Network: b.RepoTrusted.Network, + Volumes: b.RepoTrusted.Volumes, + Security: b.RepoTrusted.Security, }), - linter.PrivilegedPlugins(server.Config.Pipeline.PrivilegedPlugins), - linter.WithTrustedClonePlugins(server.Config.Pipeline.TrustedClonePlugins), + linter.PrivilegedPlugins(b.PrivilegedPlugins), + linter.WithTrustedClonePlugins(b.TrustedClonePlugins), ).Lint([]*linter.WorkflowConfig{{ Workflow: parsed, File: workflow.Name, @@ -172,7 +162,7 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A return nil, multierr.Append(errorsAndWarnings, err) } - ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID) + ir, err := b.compileWorkflow(parsed, environ, workflowMetadata, workflow.ID) if err != nil { return nil, multierr.Append(errorsAndWarnings, err) } @@ -197,102 +187,13 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A return item, errorsAndWarnings } -func stepListContainsItemsToRun(items []*Item) bool { - for i := range items { - if items[i].Workflow.State == model.StatusPending { - return true - } - } - return false -} - -func filterItemsWithMissingDependencies(items []*Item) []*Item { - itemsToRemove := make([]*Item, 0) - - for _, item := range items { - for _, dep := range item.DependsOn { - if !containsItemWithName(dep, items) { - itemsToRemove = append(itemsToRemove, item) - } - } - } - - if len(itemsToRemove) > 0 { - filtered := make([]*Item, 0) - for _, item := range items { - if !containsItemWithName(item.Workflow.Name, itemsToRemove) { - filtered = append(filtered, item) - } - } - // Recursive to handle transitive deps - return filterItemsWithMissingDependencies(filtered) - } - - return items -} - -func containsItemWithName(name string, items []*Item) bool { - for _, item := range items { - if name == item.Workflow.Name { - return true - } - } - return false -} - -func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string { - environ := metadata.Environ() - for k, v := range axis { - environ[k] = v - } - return environ -} - -func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, workflowID int64) (*backend_types.Config, error) { - var secrets []compiler.Secret - for _, sec := range b.Secs { - var events []string - for _, event := range sec.Events { - events = append(events, string(event)) - } - - secrets = append(secrets, compiler.Secret{ - Name: sec.Name, - Value: sec.Value, - AllowedPlugins: sec.Images, - Events: events, - }) - } - - var registries []compiler.Registry - for _, reg := range b.Regs { - registries = append(registries, compiler.Registry{ - Hostname: reg.Address, - Username: reg.Username, - Password: reg.Password, - }) - } - - return compiler.New( +func (b *StepBuilder) compileWorkflow(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, workflowID int64) (*backend_types.Config, error) { + options := []compiler.Option{} + options = append(options, compiler.WithEnviron(environ), compiler.WithEnviron(b.Envs), - // TODO: server deps should be moved into StepBuilder fields and set on StepBuilder creation - compiler.WithEscalated(server.Config.Pipeline.PrivilegedPlugins...), - compiler.WithVolumes(server.Config.Pipeline.Volumes...), - compiler.WithNetworks(server.Config.Pipeline.Networks...), - compiler.WithLocal(false), - compiler.WithOption( - compiler.WithNetrc( - b.Netrc.Login, - b.Netrc.Password, - b.Netrc.Machine, - ), - b.Repo.IsSCMPrivate || server.Config.Pipeline.AuthenticatePublicRepos, - ), - compiler.WithDefaultClonePlugin(server.Config.Pipeline.DefaultClonePlugin), - compiler.WithTrustedClonePlugins(append(b.Repo.NetrcTrustedPlugins, server.Config.Pipeline.TrustedClonePlugins...)), - compiler.WithRegistry(registries...), - compiler.WithSecret(secrets...), + compiler.WithEscalated(b.PrivilegedPlugins...), + compiler.WithTrustedClonePlugins(b.TrustedClonePlugins), // TODO: append repo trusted clone plugins compiler.WithPrefix( fmt.Sprintf( "wp_%s_%d", @@ -300,17 +201,10 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, envi workflowID, ), ), - compiler.WithProxy(b.ProxyOpts), - compiler.WithWorkspaceFromURL(compiler.DefaultWorkspaceBase, b.Repo.ForgeURL), compiler.WithMetadata(metadata), - compiler.WithTrustedSecurity(b.Repo.Trusted.Security), - ).Compile(parsed) -} + compiler.WithTrustedSecurity(b.RepoTrusted.Security), + ) + options = append(options, b.CompilerOptions...) -func SanitizePath(path string) string { - path = filepath.Base(path) - path = strings.TrimSuffix(path, ".yml") - path = strings.TrimSuffix(path, ".yaml") - path = strings.TrimPrefix(path, ".") - return path + return compiler.New(options...).Compile(parsed) } diff --git a/server/pipeline/stepbuilder/stepBuilder_test.go b/pipeline/frontend/yaml/stepbuilder/step_builder_test.go similarity index 75% rename from server/pipeline/stepbuilder/stepBuilder_test.go rename to pipeline/frontend/yaml/stepbuilder/step_builder_test.go index bca38e798..c4759e20c 100644 --- a/server/pipeline/stepbuilder/stepBuilder_test.go +++ b/pipeline/frontend/yaml/stepbuilder/step_builder_test.go @@ -22,31 +22,40 @@ import ( "github.com/stretchr/testify/assert" "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/server/forge" "go.woodpecker-ci.org/woodpecker/v3/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" + server_metadata "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/metadata" ) +func getMockMetadata(t *testing.T) func(*model.Workflow) metadata.Metadata { + repo := &model.Repo{} + curr := &model.Pipeline{ + Message: "aaa", + Event: model.EventPush, + } + last := &model.Pipeline{} + host := "" + return server_metadata.MetadataFromStruct(getMockForge(t), repo, curr, last, host) +} + func TestGlobalEnvsubst(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), Envs: map[string]string{ "KEY_K": "VALUE_V", "IMAGE": "scratch", }, - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Message: "aaa", - Event: model.EventPush, + WorkflowMetadataFunc: getMockMetadata(t), + Host: "", + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -71,21 +80,17 @@ func TestMissingGlobalEnvsubst(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), Envs: map[string]string{ "KEY_K": "VALUE_V", "NO_IMAGE": "scratch", }, - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Message: "aaa", - Event: model.EventPush, + WorkflowMetadataFunc: getMockMetadata(t), + Host: "", + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -110,17 +115,13 @@ func TestMultilineEnvsubst(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Message: `aaa -bbb`, + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -154,16 +155,13 @@ func TestMultiPipeline(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Event: model.EventPush, + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -195,16 +193,13 @@ func TestDependsOn(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Event: model.EventPush, + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Name: "lint", Data: []byte(` when: @@ -250,16 +245,13 @@ func TestRunsOn(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Event: model.EventPush, + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -291,16 +283,13 @@ func TestPipelineName(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{Config: ".woodpecker"}, - Curr: &model.Pipeline{ - Event: model.EventPush, + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Name: ".woodpecker/lint.yml", Data: []byte(` when: @@ -333,17 +322,13 @@ func TestBranchFilter(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Branch: "dev", - Event: model.EventPush, + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, }, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -379,14 +364,13 @@ func TestRootWhenFilter(t *testing.T) { t.Parallel() b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: &model.Pipeline{Event: "tag"}, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, + }, Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -423,20 +407,19 @@ steps: func TestZeroSteps(t *testing.T) { t.Parallel() - pipeline := &model.Pipeline{ - Branch: "dev", - Event: model.EventPush, - } + // pipeline := &model.Pipeline{ + // Branch: "dev", + // Event: model.EventPush, + // } b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: pipeline, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, + }, Yamls: []*forge_types.FileMeta{ {Data: []byte(` when: @@ -463,20 +446,19 @@ steps: func TestZeroStepsAsMultiPipelineDeps(t *testing.T) { t.Parallel() - pipeline := &model.Pipeline{ - Branch: "dev", - Event: model.EventPush, - } + // pipeline := &model.Pipeline{ + // Branch: "dev", + // Event: model.EventPush, + // } b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: pipeline, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, + }, Yamls: []*forge_types.FileMeta{ {Name: "zerostep", Data: []byte(` when: @@ -521,20 +503,19 @@ depends_on: [ zerostep ] func TestZeroStepsAsMultiPipelineTransitiveDeps(t *testing.T) { t.Parallel() - pipeline := &model.Pipeline{ - Branch: "dev", - Event: model.EventPush, - } + // pipeline := &model.Pipeline{ + // Branch: "dev", + // Event: model.EventPush, + // } b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - Curr: pipeline, - Prev: &model.Pipeline{}, - Netrc: &model.Netrc{}, - Secs: []*model.Secret{}, - Regs: []*model.Registry{}, - Host: "", + Host: "", + WorkflowMetadataFunc: getMockMetadata(t), + RepoTrusted: &metadata.TrustedConfiguration{ + Network: false, + Volumes: false, + Security: false, + }, Yamls: []*forge_types.FileMeta{ {Name: "zerostep", Data: []byte(` when: diff --git a/pipeline/frontend/yaml/stepbuilder/utils.go b/pipeline/frontend/yaml/stepbuilder/utils.go new file mode 100644 index 000000000..fbde0032a --- /dev/null +++ b/pipeline/frontend/yaml/stepbuilder/utils.go @@ -0,0 +1,74 @@ +// Copyright 2025 Woodpecker Authors +// Copyright 2018 Drone.IO Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stepbuilder + +import ( + "path/filepath" + "strings" + + "go.woodpecker-ci.org/woodpecker/v3/server/model" +) + +func SanitizePath(path string) string { + path = filepath.Base(path) + path = strings.TrimSuffix(path, ".yml") + path = strings.TrimSuffix(path, ".yaml") + path = strings.TrimPrefix(path, ".") + return path +} + +func stepListContainsItemsToRun(items []*Item) bool { + for i := range items { + if items[i].Workflow.State == model.StatusPending { + return true + } + } + return false +} + +func filterItemsWithMissingDependencies(items []*Item) []*Item { + itemsToRemove := make([]*Item, 0) + + for _, item := range items { + for _, dep := range item.DependsOn { + if !containsItemWithName(dep, items) { + itemsToRemove = append(itemsToRemove, item) + } + } + } + + if len(itemsToRemove) > 0 { + filtered := make([]*Item, 0) + for _, item := range items { + if !containsItemWithName(item.Workflow.Name, itemsToRemove) { + filtered = append(filtered, item) + } + } + // Recursive to handle transitive deps + return filterItemsWithMissingDependencies(filtered) + } + + return items +} + +func containsItemWithName(name string, items []*Item) bool { + for _, item := range items { + if name == item.Workflow.Name { + return true + } + } + return false +} diff --git a/server/api/pipeline.go b/server/api/pipeline.go index ce3d1396f..fc2186f83 100644 --- a/server/api/pipeline.go +++ b/server/api/pipeline.go @@ -31,7 +31,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/pipeline" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/stepbuilder" + "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/metadata" "go.woodpecker-ci.org/woodpecker/v3/server/router/middleware/session" "go.woodpecker-ci.org/woodpecker/v3/server/store" "go.woodpecker-ci.org/woodpecker/v3/server/store/types" @@ -460,8 +460,8 @@ func GetPipelineMetadata(c *gin.Context) { return } - metadata := stepbuilder.MetadataFromStruct(forge, repo, currentPipeline, prevPipeline, nil, server.Config.Server.Host) - c.JSON(http.StatusOK, metadata) + m := metadata.MetadataFromStruct(forge, repo, currentPipeline, prevPipeline, server.Config.Server.Host)(nil) + c.JSON(http.StatusOK, m) } // CancelPipeline diff --git a/server/pipeline/config.go b/server/pipeline/config.go index 0e550d54c..5615186c7 100644 --- a/server/pipeline/config.go +++ b/server/pipeline/config.go @@ -15,9 +15,9 @@ package pipeline import ( + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/stepbuilder" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/stepbuilder" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) diff --git a/server/pipeline/items.go b/server/pipeline/items.go index 72080414b..32bd06e70 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -18,16 +18,19 @@ import ( "context" "database/sql" "errors" + "maps" "github.com/rs/zerolog/log" pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" + pipeline_metadata "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/stepbuilder" "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/forge" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/stepbuilder" + "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/metadata" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) @@ -49,12 +52,36 @@ func parsePipeline(forge forge.Forge, store store.Store, currentPipeline *model. log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) } + var secrets []compiler.Secret + for _, sec := range secs { + var events []string + for _, event := range sec.Events { + events = append(events, string(event)) + } + + secrets = append(secrets, compiler.Secret{ + Name: sec.Name, + Value: sec.Value, + AllowedPlugins: sec.Images, + Events: events, + }) + } + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) regs, err := registryService.RegistryListPipeline(repo, currentPipeline) if err != nil { log.Error().Err(err).Msgf("error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) } + var registries []compiler.Registry + for _, reg := range regs { + registries = append(registries, compiler.Registry{ + Hostname: reg.Address, + Username: reg.Username, + Password: reg.Password, + }) + } + if envs == nil { envs = map[string]string{} } @@ -67,26 +94,44 @@ func parsePipeline(forge forge.Forge, store store.Store, currentPipeline *model. } } - for k, v := range currentPipeline.AdditionalVariables { - envs[k] = v - } + maps.Copy(envs, currentPipeline.AdditionalVariables) + + workflowMetadataFunc := metadata.MetadataFromStruct(forge, repo, currentPipeline, prev, server.Config.Server.Host) b := stepbuilder.StepBuilder{ - Repo: repo, - Curr: currentPipeline, - Prev: prev, - Netrc: netrc, - Secs: secs, - Regs: regs, - Envs: envs, - Host: server.Config.Server.Host, - Yamls: yamls, - Forge: forge, - DefaultLabels: server.Config.Pipeline.DefaultWorkflowLabels, - ProxyOpts: compiler.ProxyOptions{ - NoProxy: server.Config.Pipeline.Proxy.No, - HTTPProxy: server.Config.Pipeline.Proxy.HTTP, - HTTPSProxy: server.Config.Pipeline.Proxy.HTTPS, + Yamls: yamls, + WorkflowMetadataFunc: workflowMetadataFunc, + RepoTrusted: &pipeline_metadata.TrustedConfiguration{ + Network: repo.Trusted.Network, + Volumes: repo.Trusted.Volumes, + Security: repo.Trusted.Security, + }, + Host: server.Config.Server.Host, + Envs: envs, + TrustedClonePlugins: append(server.Config.Pipeline.TrustedClonePlugins, repo.NetrcTrustedPlugins...), + PrivilegedPlugins: server.Config.Pipeline.PrivilegedPlugins, + DefaultLabels: server.Config.Pipeline.DefaultWorkflowLabels, + CompilerOptions: []compiler.Option{ + compiler.WithVolumes(server.Config.Pipeline.Volumes...), + compiler.WithNetworks(server.Config.Pipeline.Networks...), + compiler.WithLocal(false), + compiler.WithOption( + compiler.WithNetrc( + netrc.Login, + netrc.Password, + netrc.Machine, + ), + repo.IsSCMPrivate || server.Config.Pipeline.AuthenticatePublicRepos, + ), + compiler.WithDefaultClonePlugin(server.Config.Pipeline.DefaultClonePlugin), + compiler.WithRegistry(registries...), + compiler.WithSecret(secrets...), + compiler.WithProxy(compiler.ProxyOptions{ + NoProxy: server.Config.Pipeline.Proxy.No, + HTTPProxy: server.Config.Pipeline.Proxy.HTTP, + HTTPSProxy: server.Config.Pipeline.Proxy.HTTPS, + }), + compiler.WithWorkspaceFromURL(compiler.DefaultWorkspaceBase, repo.ForgeURL), }, } return b.Build() diff --git a/server/pipeline/items_test.go b/server/pipeline/items_test.go index 53b9db126..a93b15051 100644 --- a/server/pipeline/items_test.go +++ b/server/pipeline/items_test.go @@ -4,8 +4,8 @@ import ( "testing" "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/stepbuilder" "go.woodpecker-ci.org/woodpecker/v3/server/model" - sharedPipeline "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/stepbuilder" ) func TestSetPipelineStepsOnPipeline(t *testing.T) { @@ -16,7 +16,7 @@ func TestSetPipelineStepsOnPipeline(t *testing.T) { Event: model.EventPush, } - pipelineItems := []*sharedPipeline.Item{{ + pipelineItems := []*stepbuilder.Item{{ Workflow: &model.Workflow{ PID: 1, }, diff --git a/server/pipeline/stepbuilder/metadata.go b/server/pipeline/metadata/metadata.go similarity index 78% rename from server/pipeline/stepbuilder/metadata.go rename to server/pipeline/metadata/metadata.go index 8ceb0dbcd..d15ebd6d5 100644 --- a/server/pipeline/stepbuilder/metadata.go +++ b/server/pipeline/metadata/metadata.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package stepbuilder +package metadata import ( "fmt" @@ -25,7 +25,7 @@ import ( ) // MetadataFromStruct return the metadata from a pipeline will run with. -func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, prev *model.Pipeline, workflow *model.Workflow, sysURL string) metadata.Metadata { +func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, prev *model.Pipeline, sysURL string) func(workflow *model.Workflow) metadata.Metadata { host := sysURL uri, err := url.Parse(sysURL) if err == nil { @@ -69,29 +69,31 @@ func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, } } - fWorkflow := metadata.Workflow{} - if workflow != nil { - fWorkflow = metadata.Workflow{ - Name: workflow.Name, - Number: workflow.PID, - Matrix: workflow.Environ, + return func(workflow *model.Workflow) metadata.Metadata { + fWorkflow := metadata.Workflow{} + if workflow != nil { + fWorkflow = metadata.Workflow{ + Name: workflow.Name, + Number: workflow.PID, + Matrix: workflow.Environ, + } } - } - return metadata.Metadata{ - Repo: fRepo, - Curr: metadataPipelineFromModelPipeline(pipeline, true), - Prev: metadataPipelineFromModelPipeline(prev, false), - Workflow: fWorkflow, - Step: metadata.Step{}, - Sys: metadata.System{ - Name: "woodpecker", - URL: sysURL, - Host: host, - Platform: "", // will be set by pipeline platform option or by agent - Version: version.Version, - }, - Forge: fForge, + return metadata.Metadata{ + Repo: fRepo, + Curr: metadataPipelineFromModelPipeline(pipeline, true), + Prev: metadataPipelineFromModelPipeline(prev, false), + Workflow: fWorkflow, + Step: metadata.Step{}, + Sys: metadata.System{ + Name: "woodpecker", + URL: sysURL, + Host: host, + Platform: "", // will be set by pipeline platform option or by agent + Version: version.Version, + }, + Forge: fForge, + } } } diff --git a/server/pipeline/stepbuilder/metadata_test.go b/server/pipeline/metadata/metadata_test.go similarity index 98% rename from server/pipeline/stepbuilder/metadata_test.go rename to server/pipeline/metadata/metadata_test.go index c63680d94..e6dc6b62d 100644 --- a/server/pipeline/stepbuilder/metadata_test.go +++ b/server/pipeline/metadata/metadata_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package stepbuilder +package metadata import ( "testing" @@ -91,7 +91,7 @@ func TestMetadataFromStruct(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - result := MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.prev, testCase.workflow, testCase.sysURL) + result := MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.prev, testCase.sysURL)(testCase.workflow) assert.EqualValues(t, testCase.expectedMetadata, result) assert.EqualValues(t, testCase.expectedEnviron, result.Environ()) }) diff --git a/server/pipeline/queue.go b/server/pipeline/queue.go index 002dbf3a9..2def5e09e 100644 --- a/server/pipeline/queue.go +++ b/server/pipeline/queue.go @@ -20,10 +20,10 @@ import ( "fmt" "maps" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/stepbuilder" "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/stepbuilder" ) func queuePipeline(ctx context.Context, repo *model.Repo, pipelineItems []*stepbuilder.Item) error { diff --git a/server/pipeline/start.go b/server/pipeline/start.go index 82625319f..e582cabcf 100644 --- a/server/pipeline/start.go +++ b/server/pipeline/start.go @@ -19,9 +19,9 @@ import ( "github.com/rs/zerolog/log" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/stepbuilder" "go.woodpecker-ci.org/woodpecker/v3/server/forge" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/stepbuilder" "go.woodpecker-ci.org/woodpecker/v3/server/store" )