From ff6895308a73a815f0b83a6fdfe26970e6789bf3 Mon Sep 17 00:00:00 2001 From: qwerty287 <80460567+qwerty287@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:13:00 +0100 Subject: [PATCH] Support exclusive extensions (#5978) Co-authored-by: 6543 <6543@obermui.de> --- cmd/server/flags.go | 7 +- cmd/server/openapi/docs.go | 9 ++ .../40-configuration-extension.md | 26 +++-- .../10-configuration/10-server.md | 106 +++--------------- docs/docs/92-development/02-core-ideas.md | 2 +- docs/src/pages/migrations.md | 1 - server/api/repo.go | 3 + server/model/repo.go | 2 + server/services/config/http.go | 19 +++- server/services/manager.go | 3 + server/services/setup.go | 3 + web/src/assets/locales/en.json | 2 + web/src/lib/api/types/repo.ts | 4 +- web/src/views/repo/settings/Extensions.vue | 9 ++ 14 files changed, 83 insertions(+), 113 deletions(-) diff --git a/cmd/server/flags.go b/cmd/server/flags.go index ac5a3fd66..779d91400 100644 --- a/cmd/server/flags.go +++ b/cmd/server/flags.go @@ -276,7 +276,12 @@ var flags = append([]cli.Flag{ &cli.StringFlag{ Sources: cli.EnvVars("WOODPECKER_CONFIG_SERVICE_ENDPOINT"), Name: "config-service-endpoint", - Usage: "url used for calling configuration service endpoint", + Usage: "url used for calling global configuration service endpoint", + }, + &cli.BoolFlag{ + Sources: cli.EnvVars("WOODPECKER_CONFIG_SERVICE_EXCLUSIVE"), + Name: "config-service-exclusive", + Usage: "whether global configuration service endpoint should be exclusive (skip forge)", }, &cli.StringFlag{ Sources: cli.EnvVars("WOODPECKER_EXTENSIONS_ALLOWED_HOSTS"), diff --git a/cmd/server/openapi/docs.go b/cmd/server/openapi/docs.go index bbdcf6287..0d5539279 100644 --- a/cmd/server/openapi/docs.go +++ b/cmd/server/openapi/docs.go @@ -5223,6 +5223,9 @@ const docTemplate = `{ "config_extension_endpoint": { "type": "string" }, + "config_extension_exclusive": { + "type": "boolean" + }, "config_file": { "type": "string" }, @@ -5316,6 +5319,9 @@ const docTemplate = `{ "config_extension_endpoint": { "type": "string" }, + "config_extension_exclusive": { + "type": "boolean" + }, "config_file": { "type": "string" }, @@ -5400,6 +5406,9 @@ const docTemplate = `{ "config_extension_endpoint": { "type": "string" }, + "config_extension_exclusive": { + "type": "boolean" + }, "config_file": { "type": "string" }, diff --git a/docs/docs/20-usage/72-extensions/40-configuration-extension.md b/docs/docs/20-usage/72-extensions/40-configuration-extension.md index c8f41ea48..baa27e25d 100644 --- a/docs/docs/20-usage/72-extensions/40-configuration-extension.md +++ b/docs/docs/20-usage/72-extensions/40-configuration-extension.md @@ -23,7 +23,7 @@ As Woodpecker will pass private information like tokens and will execute the ret In addition to the ability to configure the extension per repository, you can also configure a global endpoint in the Woodpecker server configuration. This can be useful if you want to use the extension for all repositories. Be careful if you share your Woodpecker server with others as they will also use your configuration extension. -The global configuration will be called before the repository specific configuration extension if both are configured. +The global configuration will be called before the repository specific configuration extension if both are configured and the repository has not enabled the exclusive setting. ```ini title="Server" WOODPECKER_CONFIG_SERVICE_ENDPOINT=https://example.com/ciconfig @@ -33,6 +33,8 @@ WOODPECKER_CONFIG_SERVICE_ENDPOINT=https://example.com/ciconfig When a pipeline is triggered Woodpecker will fetch the pipeline configuration from the repository, then make a HTTP POST request to the configured extension with a JSON payload containing some data like the repository, pipeline information and the current config files retrieved from the repository. The extension can then send back modified or even new pipeline configurations following Woodpeckers official yaml format that should be used. +You can enable the exclusive setting (both globally and on a per-repo level). Then Woodpecker will only call your extension, but nothing else. This allows you to completely skip the forge. Requests sent to the extension will not have the configuration files added. + ### Request The extension receives an HTTP POST request with the following JSON payload: @@ -42,6 +44,11 @@ class Request { repo: Repo; pipeline: Pipeline; netrc: Netrc; + configuration: { + // list of configurations. Not send if there was none. + name: string; // filename of the configuration file + data: string; // content of the configuration file + }[]; } ``` @@ -52,15 +59,12 @@ Checkout the following models for more information: - [netrc model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/netrc.go) :::tip -The `netrc` data is pretty powerful as it contains credentials to access the repository. You can use this to fetch files or other information (like changed files, issues) from the repository using the forge api or even clone the repository. +The `netrc` data is pretty powerful as it contains credentials to access the repository. You can use this to clone the repository or even use the forge (Github or Gitlab, ...) API to get more information about the repository. ::: Example request: ```json -// Please check the latest structure in the models mentioned above. -// This example is likely outdated. - { "repo": { "id": 100, @@ -122,12 +126,12 @@ Example request: "updated_at": 0, "verified": false }, - "netrc": { - "machine": "myforge.com", - "login": "myUser", - "password": "myPassword", - "type": "forge" - } + "configuration": [ + { + "name": ".woodpecker.yaml", + "data": "steps:\n - name: backend\n image: alpine\n commands:\n - echo \"Hello there from Repo (.woodpecker.yaml)\"\n" + } + ] } ``` diff --git a/docs/docs/30-administration/10-configuration/10-server.md b/docs/docs/30-administration/10-configuration/10-server.md index 61e67b109..8994af44f 100644 --- a/docs/docs/30-administration/10-configuration/10-server.md +++ b/docs/docs/30-administration/10-configuration/10-server.md @@ -402,97 +402,6 @@ woodpecker_waiting_steps 0 woodpecker_worker_count 4 ``` -## External Configuration API - -To provide additional management and preprocessing capabilities for pipeline configurations Woodpecker supports an HTTP API which can be enabled to call an external config service. -Before the run or restart of any pipeline Woodpecker will make a POST request to an external HTTP API sending the current repository, build information and all current config files retrieved from the repository. The external API can then send back new pipeline configurations that will be used immediately or respond with `HTTP 204` to tell the system to use the existing configuration. - -Every request sent by Woodpecker is signed using a [http-signature](https://datatracker.ietf.org/doc/html/rfc9421) by a private key (ed25519) generated on the first start of the Woodpecker server. You can get the public key for the verification of the http-signature from `http(s)://your-woodpecker-server/api/signature/public-key`. - -A simplistic example configuration service can be found here: [https://github.com/woodpecker-ci/example-config-service](https://github.com/woodpecker-ci/example-config-service) - -:::warning -You need to trust the external config service as it is getting secret information about the repository and pipeline and has the ability to change pipeline configs that could run malicious tasks. -::: - -### Configuration - -```ini title="Server" -WOODPECKER_CONFIG_SERVICE_ENDPOINT=https://example.com/ciconfig -``` - -#### Example request made by Woodpecker - -```json -{ - "repo": { - "id": 100, - "uid": "", - "user_id": 0, - "namespace": "", - "name": "woodpecker-test-pipe", - "slug": "", - "scm": "git", - "git_http_url": "", - "git_ssh_url": "", - "link": "", - "default_branch": "", - "private": true, - "visibility": "private", - "active": true, - "config": "", - "trusted": false, - "protected": false, - "ignore_forks": false, - "ignore_pulls": false, - "cancel_pulls": false, - "timeout": 60, - "counter": 0, - "synced": 0, - "created": 0, - "updated": 0, - "version": 0 - }, - "pipeline": { - "author": "myUser", - "author_avatar": "https://myforge.com/avatars/d6b3f7787a685fcdf2a44e2c685c7e03", - "author_email": "my@email.com", - "branch": "main", - "changed_files": ["some-file-name.txt"], - "commit": "2fff90f8d288a4640e90f05049fe30e61a14fd50", - "created_at": 0, - "deploy_to": "", - "enqueued_at": 0, - "error": "", - "event": "push", - "finished_at": 0, - "id": 0, - "link_url": "https://myforge.com/myUser/woodpecker-testpipe/commit/2fff90f8d288a4640e90f05049fe30e61a14fd50", - "message": "test old config\n", - "number": 0, - "parent": 0, - "ref": "refs/heads/main", - "refspec": "", - "clone_url": "", - "reviewed_at": 0, - "reviewed_by": "", - "sender": "myUser", - "signed": false, - "started_at": 0, - "status": "", - "timestamp": 1645962783, - "title": "", - "updated_at": 0, - "verified": false - }, - "netrc": { - "machine": "https://example.com", - "login": "user", - "password": "password" - } -} -``` - #### Example response structure ```json @@ -1067,7 +976,20 @@ Supported variables: - Name: `WOODPECKER_CONFIG_SERVICE_ENDPOINT` - Default: none -Specify a configuration service endpoint, see [Configuration Extension](#external-configuration-api) +Specify a configuration service endpoint, see [Configuration Extension](../../20-usage/72-extensions/40-configuration-extension.md) + +--- + +### CONFIG_SERVICE_EXCLUSIVE + +- Name: `CONFIG_SERVICE_EXCLUSIVE` +- Default: false + +Whether the forge request should be skipped for the global configuration endpoint. + +:::warning +If you enable this, all repos will exclusively use the global config service endpoint. There is no possibility to directly define pipelines in the forge, except the extension handles this case itself as well. +::: --- diff --git a/docs/docs/92-development/02-core-ideas.md b/docs/docs/92-development/02-core-ideas.md index 2f9570685..49a276d25 100644 --- a/docs/docs/92-development/02-core-ideas.md +++ b/docs/docs/92-development/02-core-ideas.md @@ -8,7 +8,7 @@ ## Addons and extensions If you are wondering whether your contribution will be accepted to be merged in the Woodpecker core, or whether it's better to write an -[addon](../30-administration/10-configuration/100-addons.md), [extension](../30-administration/10-configuration/10-server.md#external-configuration-api) or an +[addon](../30-administration/10-configuration/100-addons.md), [extension](../20-usage/72-extensions/40-configuration-extension.md) or an [external custom backend](../30-administration/10-configuration/11-backends/50-custom.md), please check these points: - Is your change very specific to your setup and unlikely to be used by anyone else? diff --git a/docs/src/pages/migrations.md b/docs/src/pages/migrations.md index 13c588408..34fe95477 100644 --- a/docs/src/pages/migrations.md +++ b/docs/src/pages/migrations.md @@ -187,7 +187,6 @@ The Webhook tokens have been changed for enhanced security and therefore existin Image pull secrets must now be set explicitly via env var `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES` ([#4005](https://github.com/woodpecker-ci/woodpecker/pull/4005)) - Webhook signatures now use the `rfc9421` protocol -- Replaced `configs` object by `netrc` in external configuration APIs - Git is now the only officially supported SCM. No others were supported previously, but the existence of the env var `CI_REPO_SCM` indicated that others might be. diff --git a/server/api/repo.go b/server/api/repo.go index 774b232dc..790454064 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -287,6 +287,9 @@ func PatchRepo(c *gin.Context) { if in.ConfigExtensionEndpoint != nil { repo.ConfigExtensionEndpoint = *in.ConfigExtensionEndpoint } + if in.ConfigExtensionExclusive != nil { + repo.ConfigExtensionExclusive = *in.ConfigExtensionExclusive + } err := _store.UpdateRepo(repo) if err != nil { diff --git a/server/model/repo.go b/server/model/repo.go index f2e009f60..e64367588 100644 --- a/server/model/repo.go +++ b/server/model/repo.go @@ -73,6 +73,7 @@ type Repo struct { CancelPreviousPipelineEvents []WebhookEvent `json:"cancel_previous_pipeline_events" xorm:"json 'cancel_previous_pipeline_events'"` NetrcTrustedPlugins []string `json:"netrc_trusted" xorm:"json 'netrc_trusted'"` ConfigExtensionEndpoint string `json:"config_extension_endpoint" xorm:"varchar(500) 'config_extension_endpoint'"` + ConfigExtensionExclusive bool `json:"config_extension_exclusive" xorm:"DEFAULT FALSE 'config_extension_exclusive'"` } // @name Repo // TableName return database table name for xorm. @@ -144,6 +145,7 @@ type RepoPatch struct { NetrcTrusted *[]string `json:"netrc_trusted"` Trusted *TrustedConfigurationPatch `json:"trusted"` ConfigExtensionEndpoint *string `json:"config_extension_endpoint,omitempty"` + ConfigExtensionExclusive *bool `json:"config_extension_exclusive"` } // @name RepoPatch type ForgeRemoteID string diff --git a/server/services/config/http.go b/server/services/config/http.go index 0f1057aba..6c2d3f34c 100644 --- a/server/services/config/http.go +++ b/server/services/config/http.go @@ -37,9 +37,10 @@ type configData struct { } type requestStructure struct { - Repo *model.Repo `json:"repo"` - Pipeline *model.Pipeline `json:"pipeline"` - Netrc *model.Netrc `json:"netrc"` + Repo *model.Repo `json:"repo"` + Pipeline *model.Pipeline `json:"pipeline"` + Netrc *model.Netrc `json:"netrc"` + Configuration []*configData `json:"configuration,omitempty"` } type responseStructure struct { @@ -56,11 +57,17 @@ func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, r return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) } + configuration := make([]*configData, len(oldConfigData)) + for i, oldConfig := range oldConfigData { + configuration[i] = &configData{Name: oldConfig.Name, Data: string(oldConfig.Data)} + } + response := new(responseStructure) body := requestStructure{ - Repo: repo, - Pipeline: pipeline, - Netrc: netrc, + Repo: repo, + Pipeline: pipeline, + Netrc: netrc, + Configuration: configuration, } status, err := h.client.Send(ctx, net_http.MethodPost, h.endpoint, body, response) diff --git a/server/services/manager.go b/server/services/manager.go index 43e370cd5..7c6417ed6 100644 --- a/server/services/manager.go +++ b/server/services/manager.go @@ -119,6 +119,9 @@ func (m *manager) RegistryService() registry.Service { func (m *manager) ConfigServiceFromRepo(repo *model.Repo) config.Service { if repo.ConfigExtensionEndpoint != "" { + if repo.ConfigExtensionExclusive { + return config.NewHTTP(strings.TrimRight(repo.ConfigExtensionEndpoint, "/"), m.client) + } return config.NewCombined(m.config, config.NewHTTP(strings.TrimRight(repo.ConfigExtensionEndpoint, "/"), m.client)) } diff --git a/server/services/setup.go b/server/services/setup.go index 1079326f6..c6677aec2 100644 --- a/server/services/setup.go +++ b/server/services/setup.go @@ -68,6 +68,9 @@ func setupConfigService(c *cli.Command, client *utils.Client) (config.Service, e if endpoint := c.String("config-service-endpoint"); endpoint != "" { httpFetcher := config.NewHTTP(endpoint, client) + if c.Bool("config-service-exclusive") { + return httpFetcher, nil + } return config.NewCombined(configFetcher, httpFetcher), nil } diff --git a/web/src/assets/locales/en.json b/web/src/assets/locales/en.json index b46f815a3..f3ad0db9b 100644 --- a/web/src/assets/locales/en.json +++ b/web/src/assets/locales/en.json @@ -525,6 +525,8 @@ "extensions_description": "Extensions are HTTP services that can be called by Woodpecker instead of using the builtin ones.", "extension_endpoint_placeholder": "e.g. https://example.com/api", "config_extension_endpoint": "Config extension endpoint", + "config_extension_exclusive": "Exclusive", + "config_extension_exclusive_desc": "If enabled, will skip all other ways of configuration fetching, including the forge.", "extensions_signatures_public_key": "Public key for signatures", "extensions_signatures_public_key_description": "This public key should be used by your extensions to verify webhook calls from Woodpecker.", "extensions_configuration_saved": "Extensions configuration saved", diff --git a/web/src/lib/api/types/repo.ts b/web/src/lib/api/types/repo.ts index bfca1b5ac..bd5bf4c22 100644 --- a/web/src/lib/api/types/repo.ts +++ b/web/src/lib/api/types/repo.ts @@ -82,6 +82,8 @@ export interface Repo { // Endpoint for config extensions config_extension_endpoint: string; + + config_extension_exclusive: boolean; } /* eslint-disable no-unused-vars */ @@ -113,7 +115,7 @@ export type RepoSettings = Pick< | 'netrc_trusted' >; -export type ExtensionSettings = Pick; +export type ExtensionSettings = Pick; export interface RepoPermissions { pull: boolean; diff --git a/web/src/views/repo/settings/Extensions.vue b/web/src/views/repo/settings/Extensions.vue index 945d8666f..1571f3287 100644 --- a/web/src/views/repo/settings/Extensions.vue +++ b/web/src/views/repo/settings/Extensions.vue @@ -9,6 +9,13 @@ + +