// Copyright 2022 Woodpecker Authors // Copyright 2018 Drone.IO Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package step_builder import ( "fmt" "maps" "path/filepath" "slices" "strconv" "strings" "github.com/oklog/ulid/v2" "github.com/rs/zerolog/log" "go.uber.org/multierr" "go.woodpecker-ci.org/woodpecker/v3/pipeline" backend_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/matrix" yaml_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/types" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" ) // StepBuilder Takes the hook data and the yaml and returns the internal data model. type StepBuilder struct { Repo *model.Repo // TODO: get rid of server dependency Curr *model.Pipeline // TODO: get rid of server dependency Prev *model.Pipeline // TODO: get rid of server dependency Host string Yamls []*forge_types.FileMeta Envs map[string]string Forge metadata.ServerForge DefaultLabels map[string]string RepoTrusted *metadata.TrustedConfiguration TrustedClonePlugins []string PrivilegedPlugins []string CompilerOptions []compiler.Option } type Item struct { Workflow *model.Workflow // TODO: get rid of server dependency Labels map[string]string DependsOn []string RunsOn []string Config *backend_types.Config } func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) { b.Yamls = forge_types.SortByName(b.Yamls) pidSequence := 1 for _, y := range b.Yamls { // matrix axes axes, err := matrix.ParseString(string(y.Data)) if err != nil { return nil, err } if len(axes) == 0 { axes = append(axes, matrix.Axis{}) } for i, axis := range axes { workflow := &model.Workflow{ PID: pidSequence, State: model.StatusPending, Environ: axis, Name: SanitizePath(y.Name), } if len(axes) > 1 { workflow.AxisID = i + 1 } item, err := b.genItemForWorkflow(workflow, axis, string(y.Data)) if err != nil && pipeline_errors.HasBlockingErrors(err) { return nil, err } else if err != nil { errorsAndWarnings = multierr.Append(errorsAndWarnings, err) } if item == nil { continue } items = append(items, item) pidSequence++ } // TODO: add summary workflow that send status back based on workflows generated by matrix function // depend on https://github.com/woodpecker-ci/woodpecker/issues/778 } items = filterItemsWithMissingDependencies(items) return items, errorsAndWarnings } func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.Axis, data string) (item *Item, errorsAndWarnings error) { workflowMetadata := MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Prev, workflow, b.Host) environ := b.environmentVariables(workflowMetadata, axis) // add global environment variables for substituting for k, v := range b.Envs { if _, exists := environ[k]; exists { // don't override existing values continue } environ[k] = v } // substitute vars substituted, err := metadata.EnvVarSubst(data, environ) if err != nil { return nil, multierr.Append(errorsAndWarnings, err) } // parse yaml pipeline parsed, err := yaml.ParseString(substituted) if err != nil { return nil, &pipeline_errors.PipelineError{Message: err.Error(), Type: pipeline_errors.PipelineErrorTypeCompiler} } // lint pipeline errorsAndWarnings = multierr.Append(errorsAndWarnings, linter.New( linter.WithTrusted(linter.TrustedConfiguration{ Network: b.Repo.Trusted.Network, Volumes: b.Repo.Trusted.Volumes, Security: b.Repo.Trusted.Security, }), linter.PrivilegedPlugins(b.PrivilegedPlugins), linter.WithTrustedClonePlugins(b.TrustedClonePlugins), ).Lint([]*linter.WorkflowConfig{{ Workflow: parsed, File: workflow.Name, RawConfig: data, }})) if pipeline_errors.HasBlockingErrors(errorsAndWarnings) { return nil, errorsAndWarnings } // checking if filtered. if match, err := parsed.When.Match(workflowMetadata, true, environ); !match && err == nil { log.Debug().Str("pipeline", workflow.Name).Msg( "marked as skipped, does not match metadata", ) return nil, nil } else if err != nil { log.Debug().Str("pipeline", workflow.Name).Msg( "pipeline config could not be parsed", ) return nil, multierr.Append(errorsAndWarnings, err) } ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID) if err != nil { return nil, multierr.Append(errorsAndWarnings, err) } if len(ir.Stages) == 0 { return nil, nil } item = &Item{ Workflow: workflow, Config: ir, Labels: parsed.Labels, DependsOn: parsed.DependsOn, RunsOn: parsed.RunsOn, //nolint:staticcheck // TODO: remove in next major. } if len(item.Labels) == 0 { item.Labels = make(map[string]string, len(b.DefaultLabels)) // Set default labels if no labels are defined in the pipeline maps.Copy(item.Labels, b.DefaultLabels) } if !slices.Contains(item.RunsOn, "failure") && parsed.When.IncludesStatusFailure(workflowMetadata, true, environ) { item.RunsOn = append(item.RunsOn, "failure") } if !slices.Contains(item.RunsOn, "success") && parsed.When.IncludesStatusFailure(workflowMetadata, true, environ) { item.RunsOn = append(item.RunsOn, "success") } // "woodpecker-ci.org" namespace is reserved for internal use for key := range item.Labels { if strings.HasPrefix(key, pipeline.InternalLabelPrefix) { log.Debug().Str("forge", b.Forge.Name()).Str("repo", b.Repo.FullName).Str("label", key).Msg("dropped pipeline label with reserved prefix woodpecker-ci.org") delete(item.Labels, key) } } // Add Woodpecker managed labels to the pipeline item.Labels[pipeline.LabelForgeRemoteID] = b.Forge.Name() item.Labels[pipeline.LabelRepoForgeID] = string(b.Repo.ForgeRemoteID) item.Labels[pipeline.LabelRepoID] = strconv.FormatInt(b.Repo.ID, 10) item.Labels[pipeline.LabelRepoName] = b.Repo.Name item.Labels[pipeline.LabelRepoFullName] = b.Repo.FullName item.Labels[pipeline.LabelBranch] = b.Repo.Branch item.Labels[pipeline.LabelOrgID] = strconv.FormatInt(b.Repo.OrgID, 10) for stageI := range item.Config.Stages { for stepI := range item.Config.Stages[stageI].Steps { item.Config.Stages[stageI].Steps[stepI].WorkflowLabels = item.Labels item.Config.Stages[stageI].Steps[stepI].OrgID = b.Repo.OrgID } } return item, errorsAndWarnings } func filterItemsWithMissingDependencies(items []*Item) []*Item { itemsToRemove := make([]*Item, 0) for _, item := range items { for _, dep := range item.DependsOn { if !containsItemWithName(dep, items) { itemsToRemove = append(itemsToRemove, item) } } } if len(itemsToRemove) > 0 { filtered := make([]*Item, 0) for _, item := range items { if !containsItemWithName(item.Workflow.Name, itemsToRemove) { filtered = append(filtered, item) } } // Recursive to handle transitive deps return filterItemsWithMissingDependencies(filtered) } return items } func containsItemWithName(name string, items []*Item) bool { for _, item := range items { if name == item.Workflow.Name { return true } } return false } func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string { environ := metadata.Environ() maps.Copy(environ, axis) return environ } func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, workflowID int64) (*backend_types.Config, error) { options := []compiler.Option{} options = append(options, compiler.WithEnviron(environ), compiler.WithEnviron(b.Envs), compiler.WithEscalated(b.PrivilegedPlugins...), compiler.WithTrustedClonePlugins(b.TrustedClonePlugins), compiler.WithPrefix( fmt.Sprintf( "wp_%s_%d", strings.ToLower(ulid.Make().String()), workflowID, ), ), compiler.WithMetadata(metadata), compiler.WithTrustedSecurity(b.RepoTrusted.Security), ) // by adding the passed in options last, we allow them // to override any of the default options set above options = append(options, b.CompilerOptions...) return compiler.New(options...).Compile(parsed) } func SanitizePath(path string) string { path = filepath.Base(path) path = strings.TrimSuffix(path, ".yml") path = strings.TrimSuffix(path, ".yaml") path = strings.TrimPrefix(path, ".") return path }