diff --git a/.woodpecker/web.yaml b/.woodpecker/web.yaml index b64f33f5c..85d01e854 100644 --- a/.woodpecker/web.yaml +++ b/.woodpecker/web.yaml @@ -58,6 +58,7 @@ steps: test: depends_on: - install-dependencies + - format-check # wait for it else test artifacts are falsely detected as wrong image: *node_image directory: web/ commands: diff --git a/agent/rpc/client_grpc.go b/agent/rpc/client_grpc.go index 21d6af30f..ff6d6d0ec 100644 --- a/agent/rpc/client_grpc.go +++ b/agent/rpc/client_grpc.go @@ -25,29 +25,44 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + grpcproto "google.golang.org/protobuf/proto" backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc/proto" ) -// Set grpc version on compile time to compare against server version response. -const ClientGrpcVersion int32 = proto.Version +const ( + // Set grpc version on compile time to compare against server version response. + ClientGrpcVersion int32 = proto.Version + + // Maximum size of an outgoing log message. + // Picked to prevent it from going over GRPC size limit (4 MiB) with a large safety margin. + maxLogBatchSize int = 1 * 1024 * 1024 + + // Maximum amount of time between sending consecutive batched log messages. + // Controls the delay between the CI job generating a log record, and web users receiving it. + maxLogFlushPeriod time.Duration = time.Second +) type client struct { client proto.WoodpeckerClient conn *grpc.ClientConn + logs chan *proto.LogEntry } // NewGrpcClient returns a new grpc Client. -func NewGrpcClient(conn *grpc.ClientConn) rpc.Peer { +func NewGrpcClient(ctx context.Context, conn *grpc.ClientConn) rpc.Peer { client := new(client) client.client = proto.NewWoodpeckerClient(conn) client.conn = conn + client.logs = make(chan *proto.LogEntry, 10) // max memory use: 10 lines * 1 MiB + go client.processLogs(ctx) return client } func (c *client) Close() error { + close(c.logs) return c.conn.Close() } @@ -367,18 +382,69 @@ func (c *client) Update(ctx context.Context, workflowID string, state rpc.StepSt return nil } -// Log writes the step log entry. -func (c *client) Log(ctx context.Context, logEntry *rpc.LogEntry) (err error) { - retry := c.newBackOff() - req := new(proto.LogRequest) - req.LogEntry = new(proto.LogEntry) - req.LogEntry.StepUuid = logEntry.StepUUID - req.LogEntry.Data = logEntry.Data - req.LogEntry.Line = int32(logEntry.Line) - req.LogEntry.Time = logEntry.Time - req.LogEntry.Type = int32(logEntry.Type) +// EnqueueLog queues the log entry to be written in a batch later. +func (c *client) EnqueueLog(logEntry *rpc.LogEntry) { + c.logs <- &proto.LogEntry{ + StepUuid: logEntry.StepUUID, + Data: logEntry.Data, + Line: int32(logEntry.Line), + Time: logEntry.Time, + Type: int32(logEntry.Type), + } +} + +func (c *client) processLogs(ctx context.Context) { + var entries []*proto.LogEntry + var bytes int + + send := func() { + if len(entries) == 0 { + return + } + + log.Debug(). + Int("entries", len(entries)). + Int("bytes", bytes). + Msg("log drain: sending queued logs") + + if err := c.sendLogs(ctx, entries); err != nil { + log.Error().Err(err).Msg("log drain: could not send logs to server") + } + + // even if send failed, we don't have infinite memory; retry has already been used + entries = entries[:0] + bytes = 0 + } + + // ctx.Done() is covered by the log channel being closed for { - _, err = c.client.Log(ctx, req) + select { + case entry, ok := <-c.logs: + if !ok { + log.Info().Msg("log drain: channel closed") + send() + return + } + + entries = append(entries, entry) + bytes += grpcproto.Size(entry) // cspell:words grpcproto + + if bytes >= maxLogBatchSize { + send() + } + + case <-time.After(maxLogFlushPeriod): + send() + } + } +} + +func (c *client) sendLogs(ctx context.Context, entries []*proto.LogEntry) error { + req := &proto.LogRequest{LogEntries: entries} + retry := c.newBackOff() + + for { + _, err := c.client.Log(ctx, req) if err == nil { break } diff --git a/agent/runner.go b/agent/runner.go index 0024b7888..27363aa66 100644 --- a/agent/runner.go +++ b/agent/runner.go @@ -28,6 +28,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/pipeline" backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v2/shared/constant" "go.woodpecker-ci.org/woodpecker/v2/shared/utils" ) @@ -118,7 +119,7 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error { //nolint:co logger.Debug().Msg("pipeline done") return - case <-time.After(time.Minute): + case <-time.After(constant.TaskTimeout / 3): logger.Debug().Msg("pipeline lease renewed") if err := r.client.Extend(workflowCtx, workflow.ID); err != nil { log.Error().Err(err).Msg("extending pipeline deadline failed") diff --git a/cli/exec/exec.go b/cli/exec/exec.go index c705de267..59bbe47e6 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -129,7 +129,10 @@ func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error { } func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis) error { - metadata := metadataFromContext(ctx, c, axis) + metadata, err := metadataFromContext(ctx, c, axis) + if err != nil { + return fmt.Errorf("could not create metadata: %w", err) + } environ := metadata.Environ() var secrets []compiler.Secret for key, val := range metadata.Workflow.Matrix { diff --git a/cli/exec/flags.go b/cli/exec/flags.go index 51bcb082f..1acc44ee3 100644 --- a/cli/exec/flags.go +++ b/cli/exec/flags.go @@ -126,6 +126,10 @@ var flags = []cli.Flag{ Sources: cli.EnvVars("CI_SYSTEM_PLATFORM"), Name: "system-platform", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_SYSTEM_HOST"), + Name: "system-host", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_SYSTEM_NAME"), Name: "system-name", @@ -149,6 +153,16 @@ var flags = []cli.Flag{ Sources: cli.EnvVars("CI_REPO_URL"), Name: "repo-url", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_REPO_SCM"), + Name: "repo-scm", + Value: "git", + }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_REPO_DEFAULT_BRANCH"), + Name: "repo-default-branch", + Value: "main", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_REPO_CLONE_URL"), Name: "repo-clone-url", @@ -187,17 +201,22 @@ var flags = []cli.Flag{ Value: "manual", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PIPELINE_URL"), + Sources: cli.EnvVars("CI_PIPELINE_FORGE_URL"), Name: "pipeline-url", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TARGET", "CI_PIPELINE_TARGET"), // TODO: remove CI_PIPELINE_TARGET in 3.x + Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TARGET"), Name: "pipeline-deploy-to", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TASK", "CI_PIPELINE_TASK"), // TODO: remove CI_PIPELINE_TASK in 3.x + Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TASK"), Name: "pipeline-deploy-task", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_PIPELINE_FILES"), + Usage: "either json formatted list of strings, or comma separated string list", + Name: "pipeline-files", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_COMMIT_SHA"), Name: "commit-sha", @@ -213,13 +232,14 @@ var flags = []cli.Flag{ &cli.StringFlag{ Sources: cli.EnvVars("CI_COMMIT_BRANCH"), Name: "commit-branch", + Value: "main", }, &cli.StringFlag{ Sources: cli.EnvVars("CI_COMMIT_MESSAGE"), Name: "commit-message", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_COMMIT_AUTHOR_NAME"), + Sources: cli.EnvVars("CI_COMMIT_AUTHOR"), Name: "commit-author-name", }, &cli.StringFlag{ @@ -230,6 +250,14 @@ var flags = []cli.Flag{ Sources: cli.EnvVars("CI_COMMIT_AUTHOR_EMAIL"), Name: "commit-author-email", }, + &cli.StringSliceFlag{ + Sources: cli.EnvVars("CI_COMMIT_PULL_REQUEST_LABELS"), + Name: "commit-pull-labels", + }, + &cli.BoolFlag{ + Sources: cli.EnvVars("CI_COMMIT_PRERELEASE"), + Name: "commit-release-is-pre", + }, &cli.IntFlag{ Sources: cli.EnvVars("CI_PREV_PIPELINE_NUMBER"), Name: "prev-pipeline-number", @@ -255,9 +283,17 @@ var flags = []cli.Flag{ Name: "prev-pipeline-event", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PREV_PIPELINE_URL"), + Sources: cli.EnvVars("CI_PREV_PIPELINE_FORGE_URL"), Name: "prev-pipeline-url", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_PREV_PIPELINE_DEPLOY_TARGET"), + Name: "prev-pipeline-deploy-to", + }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_PREV_PIPELINE_DEPLOY_TASK"), + Name: "prev-pipeline-deploy-task", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_PREV_COMMIT_SHA"), Name: "prev-commit-sha", @@ -279,7 +315,7 @@ var flags = []cli.Flag{ Name: "prev-commit-message", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PREV_COMMIT_AUTHOR_NAME"), + Sources: cli.EnvVars("CI_PREV_COMMIT_AUTHOR"), Name: "prev-commit-author-name", }, &cli.StringFlag{ diff --git a/cli/exec/metadata.go b/cli/exec/metadata.go index 20a89da66..db2f9d95f 100644 --- a/cli/exec/metadata.go +++ b/cli/exec/metadata.go @@ -16,6 +16,8 @@ package exec import ( "context" + "encoding/json" + "fmt" "runtime" "strings" @@ -27,7 +29,7 @@ import ( ) // return the metadata from the cli context. -func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) metadata.Metadata { +func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (metadata.Metadata, error) { platform := c.String("system-platform") if platform == "" { platform = runtime.GOOS + "/" + runtime.GOARCH @@ -41,12 +43,26 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me repoName = fullRepoName[idx+1:] } + var changedFiles []string + changedFilesRaw := c.String("pipeline-files") + if len(changedFilesRaw) != 0 && changedFilesRaw[0] == '[' { + if err := json.Unmarshal([]byte(changedFilesRaw), &changedFiles); err != nil { + return metadata.Metadata{}, fmt.Errorf("pipeline-files detected json but could not parse it: %w", err) + } + } else { + for _, file := range strings.Split(changedFilesRaw, ",") { + changedFiles = append(changedFiles, strings.TrimSpace(file)) + } + } + return metadata.Metadata{ Repo: metadata.Repo{ Name: repoName, Owner: repoOwner, RemoteID: c.String("repo-remote-id"), ForgeURL: c.String("repo-url"), + SCM: c.String("repo-scm"), + Branch: c.String("repo-default-branch"), CloneURL: c.String("repo-clone-url"), CloneSSHURL: c.String("repo-clone-ssh-url"), Private: c.Bool("repo-private"), @@ -74,6 +90,9 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me Email: c.String("commit-author-email"), Avatar: c.String("commit-author-avatar"), }, + PullRequestLabels: c.StringSlice("commit-pull-labels"), + IsPrerelease: c.Bool("commit-release-is-pre"), + ChangedFiles: changedFiles, }, }, Prev: metadata.Pipeline{ @@ -109,6 +128,7 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me Sys: metadata.System{ Name: c.String("system-name"), URL: c.String("system-url"), + Host: c.String("system-host"), Platform: platform, Version: version.Version, }, @@ -116,5 +136,5 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me Type: c.String("forge-type"), URL: c.String("forge-url"), }, - } + }, nil } diff --git a/cmd/agent/core/agent.go b/cmd/agent/core/agent.go index dea39a8a5..f0c7f0071 100644 --- a/cmd/agent/core/agent.go +++ b/cmd/agent/core/agent.go @@ -156,7 +156,7 @@ func run(ctx context.Context, c *cli.Command, backends []types.Backend) error { } defer conn.Close() - client := agent_rpc.NewGrpcClient(conn) + client := agent_rpc.NewGrpcClient(ctx, conn) agentConfigPersisted := atomic.Bool{} grpcCtx := metadata.NewOutgoingContext(grpcClientCtx, metadata.Pairs("hostname", hostname)) diff --git a/cmd/server/docs/docs.go b/cmd/server/docs/docs.go index 2b8437fd1..ac80d2883 100644 --- a/cmd/server/docs/docs.go +++ b/cmd/server/docs/docs.go @@ -5064,12 +5064,6 @@ const docTemplate = `{ "agent_id": { "type": "integer" }, - "data": { - "type": "array", - "items": { - "type": "integer" - } - }, "dep_status": { "type": "object", "additionalProperties": { diff --git a/cmd/server/grpc_server.go b/cmd/server/grpc_server.go index 16da47f05..f983bb158 100644 --- a/cmd/server/grpc_server.go +++ b/cmd/server/grpc_server.go @@ -33,7 +33,7 @@ import ( func runGrpcServer(ctx context.Context, c *cli.Command, _store store.Store) error { lis, err := net.Listen("tcp", c.String("grpc-addr")) if err != nil { - log.Fatal().Err(err).Msg("failed to listen on grpc-addr") //nolint:forbidigo + return fmt.Errorf("failed to listen on grpc-addr: %w", err) } jwtSecret := c.String("grpc-secret") diff --git a/docker/Dockerfile.cli.alpine.multiarch b/docker/Dockerfile.cli.alpine.multiarch index 53b8da12c..de5fd0688 100644 --- a/docker/Dockerfile.cli.alpine.multiarch +++ b/docker/Dockerfile.cli.alpine.multiarch @@ -8,7 +8,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ make build-cli FROM docker.io/alpine:3.20 +WORKDIR /woodpecker + RUN apk add -U --no-cache ca-certificates + ENV GODEBUG=netdns=go ENV WOODPECKER_DISABLE_UPDATE_CHECK=true diff --git a/docker/Dockerfile.cli.multiarch b/docker/Dockerfile.cli.multiarch index fcb5f95d9..2e1a7009f 100644 --- a/docker/Dockerfile.cli.multiarch +++ b/docker/Dockerfile.cli.multiarch @@ -8,6 +8,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ make build-cli FROM scratch +WORKDIR /woodpecker + ENV GODEBUG=netdns=go ENV WOODPECKER_DISABLE_UPDATE_CHECK=true diff --git a/docs/docs/30-administration/00-getting-started.md b/docs/docs/30-administration/00-getting-started.md index 8bb1b0a71..e5d573e56 100644 --- a/docs/docs/30-administration/00-getting-started.md +++ b/docs/docs/30-administration/00-getting-started.md @@ -32,9 +32,9 @@ In addition you need at least some kind of database which requires additional re ## Installation -You can install Woodpecker on multiple ways. If you are not sure which one to choose, we recommend using the [docker-compose](./05-deployment-methods/10-docker-compose.md) method for the beginning: +You can install Woodpecker on multiple ways. If you are not sure which one to choose, we recommend using the [docker compose](./05-deployment-methods/10-docker-compose.md) method for the beginning: -- Using [docker-compose](./05-deployment-methods/10-docker-compose.md) with the official [container images](./05-deployment-methods/10-docker-compose.md#docker-images) +- Using [docker compose](./05-deployment-methods/10-docker-compose.md) with the official [container images](./05-deployment-methods/10-docker-compose.md#docker-images) - Using [Kubernetes](./05-deployment-methods/20-kubernetes.md) via the Woodpecker Helm chart - Using binaries, DEBs or RPMs you can download from [latest release](https://github.com/woodpecker-ci/woodpecker/releases/latest) - Or using a [third-party installation method](./05-deployment-methods/30-third-party.md) @@ -55,5 +55,5 @@ Check the [server configuration](./10-server-config.md) and [agent configuration The agent is the worker which executes the [workflows](../20-usage/15-terminology/index.md). Woodpecker agents can execute work using a [backend](../20-usage/15-terminology/index.md) like [docker](./22-backends/10-docker.md) or [kubernetes](./22-backends/40-kubernetes.md). -By default if you choose to deploy an agent using [docker-compose](./05-deployment-methods/10-docker-compose.md) the agent simply use docker for the backend as well. +By default if you choose to deploy an agent using [docker compose](./05-deployment-methods/10-docker-compose.md) the agent simply use docker for the backend as well. So nothing to worry about here. If you still prefer to adjust the agent to your needs, check the [agent configuration](./15-agent-config.md) page. diff --git a/docs/docs/30-administration/05-deployment-methods/10-docker-compose.md b/docs/docs/30-administration/05-deployment-methods/10-docker-compose.md index 5af7e85fc..161e75bd8 100644 --- a/docs/docs/30-administration/05-deployment-methods/10-docker-compose.md +++ b/docs/docs/30-administration/05-deployment-methods/10-docker-compose.md @@ -1,12 +1,10 @@ -# docker-compose +# docker compose -The below [docker-compose](https://docs.docker.com/compose/) configuration can be used to start a Woodpecker server with a single agent. +The below [docker compose](https://docs.docker.com/compose/) configuration can be used to start a Woodpecker server with a single agent. -It relies on a number of environment variables that you must set before running `docker-compose up`. The variables are described below. +It relies on a number of environment variables that you must set before running `docker compose up`. The variables are described below. ```yaml title="docker-compose.yaml" -version: '3' - services: woodpecker-server: image: woodpeckerci/woodpecker-server:latest @@ -43,8 +41,6 @@ volumes: Woodpecker needs to know its own address. You must therefore provide the public address of it in `://` format. Please omit trailing slashes: ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] @@ -57,7 +53,6 @@ Woodpecker can also have its port's configured. It uses a separate port for gRPC They can be configured with `*_ADDR` variables: ```diff title="docker-compose.yaml" - version: '3' services: woodpecker-server: [...] @@ -70,7 +65,6 @@ They can be configured with `*_ADDR` variables: Reverse proxying can also be [configured for gRPC](../40-advanced/10-proxy.md#caddy). If the agents are connecting over the internet, it should also be SSL encrypted. The agent then needs to be configured to be secure: ```diff title="docker-compose.yaml" - version: '3' services: woodpecker-server: [...] @@ -83,8 +77,6 @@ Reverse proxying can also be [configured for gRPC](../40-advanced/10-proxy.md#ca As agents run pipeline steps as docker containers they require access to the host machine's Docker daemon: ```diff title="docker-compose.yaml" - version: '3' - services: [...] woodpecker-agent: @@ -96,8 +88,6 @@ As agents run pipeline steps as docker containers they require access to the hos Agents require the server address for agent-to-server communication. The agent connects to the server's gRPC port: ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-agent: [...] @@ -108,8 +98,6 @@ Agents require the server address for agent-to-server communication. The agent c The server and agents use a shared secret to authenticate communication. This should be a random string of your choosing and should be kept private. You can generate such string with `openssl rand -hex 32`: ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] diff --git a/docs/docs/30-administration/10-database.md b/docs/docs/30-administration/10-database.md index e3e33ba7d..b1b5aa688 100644 --- a/docs/docs/30-administration/10-database.md +++ b/docs/docs/30-administration/10-database.md @@ -7,8 +7,6 @@ The default database engine of Woodpecker is an embedded SQLite database which r By default Woodpecker uses a SQLite database stored under `/var/lib/woodpecker/`. If using containers, you can mount a [data volume](https://docs.docker.com/storage/volumes/#create-and-manage-volumes) to persist the SQLite database. ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] diff --git a/docs/docs/30-administration/10-server-config.md b/docs/docs/30-administration/10-server-config.md index c7f544e25..02fde63a0 100644 --- a/docs/docs/30-administration/10-server-config.md +++ b/docs/docs/30-administration/10-server-config.md @@ -63,17 +63,15 @@ Point it to your server's docker config. WOODPECKER_DOCKER_CONFIG=/root/.docker/config.json ``` -## Handling sensitive data in docker-compose and docker-swarm +## Handling sensitive data in **docker compose** and **docker swarm** -To handle sensitive data in docker-compose or docker-swarm configurations there are several options: +To handle sensitive data in `docker compose` or `docker swarm` configurations there are several options: -For docker-compose you can use a `.env` file next to your compose configuration to store the secrets outside of the compose file. While this separates configuration from secrets it is still not very secure. +For docker compose you can use a `.env` file next to your compose configuration to store the secrets outside of the compose file. While this separates configuration from secrets it is still not very secure. Alternatively use docker-secrets. As it may be difficult to use docker secrets for environment variables Woodpecker allows to read sensible data from files by providing a `*_FILE` option of all sensible configuration variables. Woodpecker will try to read the value directly from this file. Keep in mind that when the original environment variable gets specified at the same time it will override the value read from the file. ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] diff --git a/docs/docs/30-administration/11-forges/30-gitea.md b/docs/docs/30-administration/11-forges/30-gitea.md index 62fe40cb6..82d77a54a 100644 --- a/docs/docs/30-administration/11-forges/30-gitea.md +++ b/docs/docs/30-administration/11-forges/30-gitea.md @@ -22,8 +22,6 @@ Otherwise, the communication should go via the `docker0` gateway (usually 172.17 To configure the Docker network if the network's name is `gitea`, configure it like this: ```diff title="docker-compose.yaml" - version: '3' - services: [...] woodpecker-agent: diff --git a/docs/docs/30-administration/11-forges/60-bitbucket_datacenter.md b/docs/docs/30-administration/11-forges/60-bitbucket_datacenter.md index 53926fa73..742180008 100644 --- a/docs/docs/30-administration/11-forges/60-bitbucket_datacenter.md +++ b/docs/docs/30-administration/11-forges/60-bitbucket_datacenter.md @@ -11,8 +11,6 @@ Woodpecker comes with experimental support for Bitbucket Datacenter / Server, fo To enable Bitbucket Server you should configure the Woodpecker container using the following environment variables: ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] diff --git a/docs/docs/30-administration/40-advanced/10-proxy.md b/docs/docs/30-administration/40-advanced/10-proxy.md index 2afcc778d..8771eed44 100644 --- a/docs/docs/30-administration/40-advanced/10-proxy.md +++ b/docs/docs/30-administration/40-advanced/10-proxy.md @@ -137,8 +137,6 @@ To install the Woodpecker server behind a [Traefik](https://traefik.io/) load ba ```yaml -version: '3.8' - services: server: image: woodpeckerci/woodpecker-server:latest diff --git a/docs/docs/30-administration/40-advanced/20-ssl.md b/docs/docs/30-administration/40-advanced/20-ssl.md index 755ba205d..6fda26d3d 100644 --- a/docs/docs/30-administration/40-advanced/20-ssl.md +++ b/docs/docs/30-administration/40-advanced/20-ssl.md @@ -52,8 +52,6 @@ SSL support is provided using the [ListenAndServeTLS](https://golang.org/pkg/net Update your configuration to expose the following ports: ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] @@ -66,8 +64,6 @@ Update your configuration to expose the following ports: Update your configuration to mount your certificate and key: ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] @@ -79,8 +75,6 @@ Update your configuration to mount your certificate and key: Update your configuration to provide the paths of your certificate and key: ```diff title="docker-compose.yaml" - version: '3' - services: woodpecker-server: [...] diff --git a/docs/docs/30-administration/40-advanced/30-autoscaler.md b/docs/docs/30-administration/40-advanced/30-autoscaler.md index ce9ee914a..0ad43a30b 100644 --- a/docs/docs/30-administration/40-advanced/30-autoscaler.md +++ b/docs/docs/30-administration/40-advanced/30-autoscaler.md @@ -6,13 +6,11 @@ Please note that the autoscaler is not feature-complete yet. You can follow the ## Setup -### docker-compose +### docker compose -If you are using docker-compose you can add the following to your `docker-compose.yaml` file: +If you are using docker compose you can add the following to your `docker-compose.yaml` file: ```yaml -version: '3' - services: woodpecker-server: image: woodpeckerci/woodpecker-server:next diff --git a/docs/docs/91-migrations.md b/docs/docs/91-migrations.md index b4d3e7a40..2d1870276 100644 --- a/docs/docs/91-migrations.md +++ b/docs/docs/91-migrations.md @@ -4,6 +4,7 @@ Some versions need some changes to the server configuration or the pipeline conf ## `next` +- Set `/woodpecker` as defautl workdir for the **woodpecker-cli** container - Removed built-in environment variables: - `CI_COMMIT_URL` use `CI_PIPELINE_FORGE_URL` - `CI_STEP_FINISHED` as empty during execution @@ -34,6 +35,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Replaced `configs` object by `netrc` in external configuration APIs - Removed old API routes: `registry/` -> `registries`, `/authorize/token` - Replaced `registry` command with `repo registry` in cli +- Disallow upgrades from 1.x, upgrade to 2.x first ## 2.0.0 diff --git a/pipeline/frontend/metadata/environment.go b/pipeline/frontend/metadata/environment.go index c48efb258..1a018867f 100644 --- a/pipeline/frontend/metadata/environment.go +++ b/pipeline/frontend/metadata/environment.go @@ -56,7 +56,7 @@ func (m *Metadata) Environ() map[string]string { "CI_REPO_NAME": m.Repo.Name, "CI_REPO_OWNER": m.Repo.Owner, "CI_REPO_REMOTE_ID": m.Repo.RemoteID, - "CI_REPO_SCM": "git", + "CI_REPO_SCM": m.Repo.SCM, "CI_REPO_URL": m.Repo.ForgeURL, "CI_REPO_CLONE_URL": m.Repo.CloneURL, "CI_REPO_CLONE_SSH_URL": m.Repo.CloneSSHURL, diff --git a/pipeline/frontend/metadata/types.go b/pipeline/frontend/metadata/types.go index 9cf7e64a1..9d3529fd9 100644 --- a/pipeline/frontend/metadata/types.go +++ b/pipeline/frontend/metadata/types.go @@ -29,17 +29,17 @@ type ( // Repo defines runtime metadata for a repository. Repo struct { - ID int64 `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Owner string `json:"owner,omitempty"` - RemoteID string `json:"remote_id,omitempty"` - ForgeURL string `json:"forge_url,omitempty"` - CloneURL string `json:"clone_url,omitempty"` - CloneSSHURL string `json:"clone_url_ssh,omitempty"` - Private bool `json:"private,omitempty"` - Secrets []Secret `json:"secrets,omitempty"` - Branch string `json:"default_branch,omitempty"` - Trusted bool `json:"trusted,omitempty"` + ID int64 `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + RemoteID string `json:"remote_id,omitempty"` + ForgeURL string `json:"forge_url,omitempty"` + SCM string `json:"scm,omitempty"` + CloneURL string `json:"clone_url,omitempty"` + CloneSSHURL string `json:"clone_url_ssh,omitempty"` + Private bool `json:"private,omitempty"` + Branch string `json:"default_branch,omitempty"` + Trusted bool `json:"trusted,omitempty"` } // Pipeline defines runtime metadata for a pipeline. @@ -91,14 +91,6 @@ type ( Number int `json:"number,omitempty"` } - // Secret defines a runtime secret. - Secret struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` - Mount string `json:"mount,omitempty"` - Mask bool `json:"mask,omitempty"` - } - // System defines runtime metadata for a ci/cd system. System struct { Name string `json:"name,omitempty"` diff --git a/pipeline/frontend/yaml/linter/schema/.woodpecker/test-service.yaml b/pipeline/frontend/yaml/linter/schema/.woodpecker/test-service.yaml index 03564e9fc..16429df75 100644 --- a/pipeline/frontend/yaml/linter/schema/.woodpecker/test-service.yaml +++ b/pipeline/frontend/yaml/linter/schema/.woodpecker/test-service.yaml @@ -10,3 +10,4 @@ services: image: mysql cache: image: redis + directory: /tmp/ diff --git a/pipeline/frontend/yaml/linter/schema/schema.json b/pipeline/frontend/yaml/linter/schema/schema.json index f7a57c7aa..6128ce15b 100644 --- a/pipeline/frontend/yaml/linter/schema/schema.json +++ b/pipeline/frontend/yaml/linter/schema/schema.json @@ -842,6 +842,9 @@ "environment": { "$ref": "#/definitions/step_environment" }, + "directory": { + "$ref": "#/definitions/step_directory" + }, "secrets": { "$ref": "#/definitions/step_secrets" }, diff --git a/pipeline/log/line_writer.go b/pipeline/log/line_writer.go index 0c5058f45..ff8706f36 100644 --- a/pipeline/log/line_writer.go +++ b/pipeline/log/line_writer.go @@ -16,7 +16,6 @@ package log import ( - "context" "io" "strings" "sync" @@ -67,9 +66,6 @@ func (w *LineWriter) Write(p []byte) (n int, err error) { w.num++ - if err := w.peer.Log(context.Background(), line); err != nil { - return 0, err - } - + w.peer.EnqueueLog(line) return len(data), nil } diff --git a/pipeline/log/line_writer_test.go b/pipeline/log/line_writer_test.go index 8bf8f6251..8a2d6b120 100644 --- a/pipeline/log/line_writer_test.go +++ b/pipeline/log/line_writer_test.go @@ -27,7 +27,7 @@ import ( func TestLineWriter(t *testing.T) { peer := mocks.NewPeer(t) - peer.On("Log", mock.Anything, mock.Anything).Return(nil) + peer.On("EnqueueLog", mock.Anything) secrets := []string{"world"} lw := log.NewLineWriter(peer, "e9ea76a5-44a1-4059-9c4a-6956c478b26d", secrets...) @@ -37,7 +37,7 @@ func TestLineWriter(t *testing.T) { _, err = lw.Write([]byte("the previous line had no newline at the end")) assert.NoError(t, err) - peer.AssertCalled(t, "Log", mock.Anything, &rpc.LogEntry{ + peer.AssertCalled(t, "EnqueueLog", &rpc.LogEntry{ StepUUID: "e9ea76a5-44a1-4059-9c4a-6956c478b26d", Time: 0, Type: rpc.LogEntryStdout, @@ -45,7 +45,7 @@ func TestLineWriter(t *testing.T) { Data: []byte("hello ********"), }) - peer.AssertCalled(t, "Log", mock.Anything, &rpc.LogEntry{ + peer.AssertCalled(t, "EnqueueLog", &rpc.LogEntry{ StepUUID: "e9ea76a5-44a1-4059-9c4a-6956c478b26d", Time: 0, Type: rpc.LogEntryStdout, diff --git a/pipeline/log/utils_test.go b/pipeline/log/utils_test.go index b094d1409..cbc15b735 100644 --- a/pipeline/log/utils_test.go +++ b/pipeline/log/utils_test.go @@ -83,7 +83,7 @@ func TestCopyLineByLine(t *testing.T) { assert.Lenf(t, writes, 2, "expected 2 writes, got: %v", writes) // wait for the goroutine to write the data - time.Sleep(time.Millisecond) + time.Sleep(10 * time.Millisecond) writtenData := strings.Join(writes, "-") assert.Equal(t, "12345\n-678\n", writtenData, "unexpected writtenData: %s", writtenData) diff --git a/pipeline/rpc/mocks/peer.go b/pipeline/rpc/mocks/peer.go index 8d0c45d18..c98456284 100644 --- a/pipeline/rpc/mocks/peer.go +++ b/pipeline/rpc/mocks/peer.go @@ -35,6 +35,11 @@ func (_m *Peer) Done(c context.Context, workflowID string, state rpc.WorkflowSta return r0 } +// EnqueueLog provides a mock function with given fields: logEntry +func (_m *Peer) EnqueueLog(logEntry *rpc.LogEntry) { + _m.Called(logEntry) +} + // Extend provides a mock function with given fields: c, workflowID func (_m *Peer) Extend(c context.Context, workflowID string) error { ret := _m.Called(c, workflowID) @@ -71,24 +76,6 @@ func (_m *Peer) Init(c context.Context, workflowID string, state rpc.WorkflowSta return r0 } -// Log provides a mock function with given fields: c, logEntry -func (_m *Peer) Log(c context.Context, logEntry *rpc.LogEntry) error { - ret := _m.Called(c, logEntry) - - if len(ret) == 0 { - panic("no return value specified for Log") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *rpc.LogEntry) error); ok { - r0 = rf(c, logEntry) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // Next provides a mock function with given fields: c, f func (_m *Peer) Next(c context.Context, f rpc.Filter) (*rpc.Workflow, error) { ret := _m.Called(c, f) diff --git a/pipeline/rpc/peer.go b/pipeline/rpc/peer.go index b38b01051..050a2ab11 100644 --- a/pipeline/rpc/peer.go +++ b/pipeline/rpc/peer.go @@ -82,8 +82,8 @@ type Peer interface { // Update updates the step state Update(c context.Context, workflowID string, state StepState) error - // Log writes the step log entry - Log(c context.Context, logEntry *LogEntry) error + // EnqueueLog queues the step log entry for delayed sending + EnqueueLog(logEntry *LogEntry) // RegisterAgent register our agent to the server RegisterAgent(ctx context.Context, platform, backend, version string, capacity int) (int64, error) diff --git a/pipeline/rpc/proto/version.go b/pipeline/rpc/proto/version.go index 2de7645f3..8f79192ce 100644 --- a/pipeline/rpc/proto/version.go +++ b/pipeline/rpc/proto/version.go @@ -16,4 +16,4 @@ package proto // Version is the version of the woodpecker.proto file, // IMPORTANT: increased by 1 each time it get changed. -const Version int32 = 9 +const Version int32 = 10 diff --git a/pipeline/rpc/proto/woodpecker.pb.go b/pipeline/rpc/proto/woodpecker.pb.go index e7cfcad3c..dd3423175 100644 --- a/pipeline/rpc/proto/woodpecker.pb.go +++ b/pipeline/rpc/proto/woodpecker.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.1 -// protoc v4.25.3 +// protoc v4.25.4 // source: woodpecker.proto package proto @@ -685,7 +685,7 @@ type LogRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - LogEntry *LogEntry `protobuf:"bytes,1,opt,name=logEntry,proto3" json:"logEntry,omitempty"` + LogEntries []*LogEntry `protobuf:"bytes,1,rep,name=logEntries,proto3" json:"logEntries,omitempty"` } func (x *LogRequest) Reset() { @@ -720,9 +720,9 @@ func (*LogRequest) Descriptor() ([]byte, []int) { return file_woodpecker_proto_rawDescGZIP(), []int{11} } -func (x *LogRequest) GetLogEntry() *LogEntry { +func (x *LogRequest) GetLogEntries() []*LogEntry { if x != nil { - return x.LogEntry + return x.LogEntries } return nil } @@ -1212,91 +1212,91 @@ var file_woodpecker_proto_rawDesc = []byte{ 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x26, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x65, 0x70, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x39, 0x0a, 0x0a, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x2d, 0x0a, 0x13, - 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x14, - 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, - 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07, - 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, - 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x22, 0x5b, 0x0a, 0x0f, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x67, 0x72, 0x70, 0x63, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x3b, 0x0a, - 0x0c, 0x4e, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, - 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x22, 0x32, 0x0a, 0x15, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x49, - 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, - 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x19, - 0x0a, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x07, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x64, 0x0a, 0x0c, 0x41, 0x75, 0x74, - 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x07, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, - 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x32, - 0xbb, 0x04, 0x0a, 0x0a, 0x57, 0x6f, 0x6f, 0x64, 0x70, 0x65, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x31, - 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x31, 0x0a, 0x04, 0x4e, 0x65, 0x78, 0x74, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x4e, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x2a, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x12, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x3d, 0x0a, 0x0a, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x6f, 0x67, + 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x22, 0x2d, 0x0a, 0x13, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, + 0x82, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, + 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, + 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, + 0x12, 0x18, 0x0a, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x5b, 0x0a, 0x0f, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x67, + 0x72, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x22, 0x3b, 0x0a, 0x0c, 0x4e, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x2b, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x22, 0x32, + 0x0a, 0x15, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x61, 0x67, 0x65, 0x6e, 0x74, + 0x49, 0x64, 0x22, 0x49, 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x64, 0x0a, + 0x0c, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x64, + 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x32, 0xbb, 0x04, 0x0a, 0x0a, 0x57, 0x6f, 0x6f, 0x64, 0x70, 0x65, 0x63, 0x6b, + 0x65, 0x72, 0x12, 0x31, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0c, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x31, 0x0a, 0x04, 0x4e, 0x65, 0x78, 0x74, 0x12, 0x12, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x65, 0x78, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2a, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, + 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x22, 0x00, 0x12, 0x2a, 0x0a, 0x04, 0x57, 0x61, 0x69, 0x74, 0x12, 0x12, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, - 0x12, 0x2a, 0x0a, 0x04, 0x57, 0x61, 0x69, 0x74, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x2a, 0x0a, 0x04, - 0x44, 0x6f, 0x6e, 0x65, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x44, 0x6f, 0x6e, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x06, 0x45, 0x78, 0x74, 0x65, - 0x6e, 0x64, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x28, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, - 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x12, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, - 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x2f, 0x0a, 0x0f, 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, - 0x00, 0x12, 0x3a, 0x0a, 0x0c, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x32, 0x43, 0x0a, - 0x0e, 0x57, 0x6f, 0x6f, 0x64, 0x70, 0x65, 0x63, 0x6b, 0x65, 0x72, 0x41, 0x75, 0x74, 0x68, 0x12, - 0x31, 0x0a, 0x04, 0x41, 0x75, 0x74, 0x68, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x6f, 0x2e, 0x77, 0x6f, 0x6f, 0x64, 0x70, 0x65, 0x63, - 0x6b, 0x65, 0x72, 0x2d, 0x63, 0x69, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x77, 0x6f, 0x6f, 0x64, 0x70, - 0x65, 0x63, 0x6b, 0x65, 0x72, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x12, 0x2a, 0x0a, 0x04, 0x44, 0x6f, 0x6e, 0x65, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x44, 0x6f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x06, + 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x2e, 0x0a, 0x06, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x28, 0x0a, 0x03, + 0x4c, 0x6f, 0x67, 0x12, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x2f, 0x0a, 0x0f, 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x0c, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, + 0x00, 0x32, 0x43, 0x0a, 0x0e, 0x57, 0x6f, 0x6f, 0x64, 0x70, 0x65, 0x63, 0x6b, 0x65, 0x72, 0x41, + 0x75, 0x74, 0x68, 0x12, 0x31, 0x0a, 0x04, 0x41, 0x75, 0x74, 0x68, 0x12, 0x12, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x6f, 0x2e, 0x77, 0x6f, 0x6f, + 0x64, 0x70, 0x65, 0x63, 0x6b, 0x65, 0x72, 0x2d, 0x63, 0x69, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x77, + 0x6f, 0x6f, 0x64, 0x70, 0x65, 0x63, 0x6b, 0x65, 0x72, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1341,7 +1341,7 @@ var file_woodpecker_proto_depIdxs = []int32{ 1, // 2: proto.InitRequest.state:type_name -> proto.WorkflowState 1, // 3: proto.DoneRequest.state:type_name -> proto.WorkflowState 0, // 4: proto.UpdateRequest.state:type_name -> proto.StepState - 2, // 5: proto.LogRequest.logEntry:type_name -> proto.LogEntry + 2, // 5: proto.LogRequest.logEntries:type_name -> proto.LogEntry 4, // 6: proto.NextResponse.workflow:type_name -> proto.Workflow 12, // 7: proto.Woodpecker.Version:input_type -> proto.Empty 5, // 8: proto.Woodpecker.Next:input_type -> proto.NextRequest diff --git a/pipeline/rpc/proto/woodpecker.proto b/pipeline/rpc/proto/woodpecker.proto index 8fc577586..ea6c987b9 100644 --- a/pipeline/rpc/proto/woodpecker.proto +++ b/pipeline/rpc/proto/woodpecker.proto @@ -106,7 +106,7 @@ message UpdateRequest { } message LogRequest { - LogEntry logEntry = 1; + repeated LogEntry logEntries = 1; } message Empty { diff --git a/pipeline/rpc/proto/woodpecker_grpc.pb.go b/pipeline/rpc/proto/woodpecker_grpc.pb.go index 5b4ee588c..64437aeb1 100644 --- a/pipeline/rpc/proto/woodpecker_grpc.pb.go +++ b/pipeline/rpc/proto/woodpecker_grpc.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v4.25.3 +// - protoc v4.25.4 // source: woodpecker.proto package proto diff --git a/server/api/repo.go b/server/api/repo.go index 84b9b8002..b96eeaae6 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -615,10 +615,14 @@ func repairRepo(c *gin.Context, repo *model.Repo, withPerms, skipOnErr bool) { user, err := _store.GetUser(repo.UserID) if err != nil { if errors.Is(err, types.RecordNotExist) { - if !skipOnErr { - c.AbortWithStatus(http.StatusNotFound) + oldUserID := repo.UserID + user = session.User(c) + repo.UserID = user.ID + err = _store.UpdateRepo(repo) + if err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) } - log.Error().Err(err).Msg("could not get user on repo repair") + log.Debug().Msgf("Could not find repo user with ID %d during repo repair, set to repair request user with ID %d", oldUserID, user.ID) } else { _ = c.AbortWithError(http.StatusInternalServerError, err) } diff --git a/server/api/stream.go b/server/api/stream.go index 2d3c7ee86..a620dd068 100644 --- a/server/api/stream.go +++ b/server/api/stream.go @@ -28,12 +28,19 @@ import ( "github.com/rs/zerolog/log" "go.woodpecker-ci.org/woodpecker/v2/server" + "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/router/middleware/session" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) +const ( + // How many batches of logs to keep for each client before starting to + // drop them if the client is not consuming them faster than they arrive. + maxQueuedBatchesPerClient int = 30 +) + // EventStreamSSE // // @Summary Stream events like pipeline updates @@ -213,17 +220,32 @@ func LogStreamSSE(c *gin.Context) { } go func() { - err := server.Config.Services.Logs.Tail(ctx, step.ID, func(entries ...*model.LogEntry) { - for _, entry := range entries { - select { - case <-ctx.Done(): - return - default: - ee, _ := json.Marshal(entry) - logChan <- ee + batches := make(logging.LogChan, maxQueuedBatchesPerClient) + + go func() { + defer func() { + if r := recover(); r != nil { + log.Error().Msgf("error sending log message: %v", r) + } + }() + + for entries := range batches { + for _, entry := range entries { + select { + case <-ctx.Done(): + return + default: + if ee, err := json.Marshal(entry); err == nil { + logChan <- ee + } else { + log.Error().Err(err).Msg("unable to serialize log entry") + } + } } } - }) + }() + + err := server.Config.Services.Logs.Tail(ctx, step.ID, batches) if err != nil { log.Error().Err(err).Msg("tail of logs failed") } diff --git a/server/grpc/rpc.go b/server/grpc/rpc.go index 718d82709..52edf1f99 100644 --- a/server/grpc/rpc.go +++ b/server/grpc/rpc.go @@ -336,27 +336,12 @@ func (s *RPC) Done(c context.Context, strWorkflowID string, state rpc.WorkflowSt } // Log writes a log entry to the database and publishes it to the pubsub. -func (s *RPC) Log(c context.Context, rpcLogEntry *rpc.LogEntry) error { - // convert rpc log_entry to model.log_entry - step, err := s.store.StepByUUID(rpcLogEntry.StepUUID) +// An explicit stepUUID makes it obvious that all entries must come from the same step. +func (s *RPC) Log(c context.Context, stepUUID string, rpcLogEntries []*rpc.LogEntry) error { + step, err := s.store.StepByUUID(stepUUID) if err != nil { - return fmt.Errorf("could not find step with uuid %s in store: %w", rpcLogEntry.StepUUID, err) + return fmt.Errorf("could not find step with uuid %s in store: %w", stepUUID, err) } - logEntry := &model.LogEntry{ - StepID: step.ID, - Time: rpcLogEntry.Time, - Line: rpcLogEntry.Line, - Data: rpcLogEntry.Data, - Type: model.LogEntryType(rpcLogEntry.Type), - } - - // make sure writes to pubsub are non blocking (https://github.com/woodpecker-ci/woodpecker/blob/c919f32e0b6432a95e1a6d3d0ad662f591adf73f/server/logging/log.go#L9) - go func() { - // write line to listening web clients - if err := s.logger.Write(c, logEntry.StepID, logEntry); err != nil { - log.Error().Err(err).Msgf("rpc server could not write to logger") - } - }() agent, err := s.getAgentFromContext(c) if err != nil { @@ -368,7 +353,34 @@ func (s *RPC) Log(c context.Context, rpcLogEntry *rpc.LogEntry) error { return err } - return server.Config.Services.LogStore.LogAppend(logEntry) + var logEntries []*model.LogEntry + + for _, rpcLogEntry := range rpcLogEntries { + if rpcLogEntry.StepUUID != stepUUID { + return fmt.Errorf("expected step UUID %s, got %s", stepUUID, rpcLogEntry.StepUUID) + } + logEntries = append(logEntries, &model.LogEntry{ + StepID: step.ID, + Time: rpcLogEntry.Time, + Line: rpcLogEntry.Line, + Data: rpcLogEntry.Data, + Type: model.LogEntryType(rpcLogEntry.Type), + }) + } + + // make sure writes to pubsub are non blocking (https://github.com/woodpecker-ci/woodpecker/blob/c919f32e0b6432a95e1a6d3d0ad662f591adf73f/server/logging/log.go#L9) + go func() { + // write line to listening web clients + if err := s.logger.Write(c, step.ID, logEntries); err != nil { + log.Error().Err(err).Msgf("rpc server could not write to logger") + } + }() + + if err = server.Config.Services.LogStore.LogAppend(step, logEntries); err != nil { + log.Error().Err(err).Msg("could not store log entries") + } + + return nil } func (s *RPC) RegisterAgent(ctx context.Context, platform, backend, version string, capacity int32) (int64, error) { @@ -513,8 +525,8 @@ func (s *RPC) getHostnameFromContext(ctx context.Context) (string, error) { } func (s *RPC) updateAgentLastWork(agent *model.Agent) error { - // only update agent.LastWork if not done recently - if time.Unix(agent.LastWork, 0).Add(updateAgentLastWorkDelay).Before(time.Now()) { + // only update agent.LastWork if not recently updated + if time.Unix(agent.LastWork, 0).Add(updateAgentLastWorkDelay).After(time.Now()) { return nil } diff --git a/server/grpc/rpc_test.go b/server/grpc/rpc_test.go index 1fd850b8a..01c57676b 100644 --- a/server/grpc/rpc_test.go +++ b/server/grpc/rpc_test.go @@ -17,9 +17,11 @@ package grpc import ( "context" "testing" + "time" "github.com/franela/goblin" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "google.golang.org/grpc/metadata" "go.woodpecker-ci.org/woodpecker/v2/server/model" @@ -107,3 +109,51 @@ func TestRegisterAgent(t *testing.T) { }) }) } + +func TestUpdateAgentLastWork(t *testing.T) { + t.Run("When last work was never updated it should update last work timestamp", func(t *testing.T) { + agent := model.Agent{ + LastWork: 0, + } + store := mocks_store.NewStore(t) + rpc := RPC{ + store: store, + } + store.On("AgentUpdate", mock.Anything).Once().Return(nil) + + err := rpc.updateAgentLastWork(&agent) + assert.NoError(t, err) + + assert.NotZero(t, agent.LastWork) + }) + + t.Run("When last work was updated over a minute ago it should update last work timestamp", func(t *testing.T) { + lastWork := time.Now().Add(-time.Hour).Unix() + agent := model.Agent{ + LastWork: lastWork, + } + store := mocks_store.NewStore(t) + rpc := RPC{ + store: store, + } + store.On("AgentUpdate", mock.Anything).Once().Return(nil) + + err := rpc.updateAgentLastWork(&agent) + assert.NoError(t, err) + + assert.NotEqual(t, lastWork, agent.LastWork) + }) + + t.Run("When last work was updated in the last minute it should not update last work timestamp again", func(t *testing.T) { + lastWork := time.Now().Add(-time.Second * 30).Unix() + agent := model.Agent{ + LastWork: lastWork, + } + rpc := RPC{} + + err := rpc.updateAgentLastWork(&agent) + assert.NoError(t, err) + + assert.Equal(t, lastWork, agent.LastWork) + }) +} diff --git a/server/grpc/server.go b/server/grpc/server.go index d6882c0f8..633a7ee99 100644 --- a/server/grpc/server.go +++ b/server/grpc/server.go @@ -20,6 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus" prometheus_auto "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/rs/zerolog/log" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc/proto" @@ -133,15 +134,39 @@ func (s *WoodpeckerServer) Extend(c context.Context, req *proto.ExtendRequest) ( } func (s *WoodpeckerServer) Log(c context.Context, req *proto.LogRequest) (*proto.Empty, error) { - logEntry := &rpc.LogEntry{ - Data: req.GetLogEntry().GetData(), - Line: int(req.GetLogEntry().GetLine()), - Time: req.GetLogEntry().GetTime(), - StepUUID: req.GetLogEntry().GetStepUuid(), - Type: int(req.GetLogEntry().GetType()), + var ( + entries []*rpc.LogEntry + stepUUID string + ) + + write := func() error { + if len(entries) > 0 { + if err := s.peer.Log(c, stepUUID, entries); err != nil { + log.Error().Err(err).Msg("could not write log entries") + return err + } + } + return nil } + + for _, reqEntry := range req.GetLogEntries() { + entry := &rpc.LogEntry{ + Data: reqEntry.GetData(), + Line: int(reqEntry.GetLine()), + Time: reqEntry.GetTime(), + StepUUID: reqEntry.GetStepUuid(), + Type: int(reqEntry.GetType()), + } + if entry.StepUUID != stepUUID { + _ = write() + stepUUID = entry.StepUUID + entries = entries[:0] + } + entries = append(entries, entry) + } + res := new(proto.Empty) - err := s.peer.Log(c, logEntry) + err := write() return res, err } diff --git a/server/logging/log.go b/server/logging/log.go index 1db5475e5..0e713644f 100644 --- a/server/logging/log.go +++ b/server/logging/log.go @@ -18,6 +18,8 @@ import ( "context" "sync" + logger "github.com/rs/zerolog/log" + "go.woodpecker-ci.org/woodpecker/v2/server/model" ) @@ -38,7 +40,7 @@ import ( // sub.start()... event loop type subscriber struct { - handler Handler + receiver LogChan } type stream struct { @@ -77,7 +79,7 @@ func (l *log) Open(_ context.Context, stepID int64) error { return nil } -func (l *log) Write(ctx context.Context, stepID int64, logEntry *model.LogEntry) error { +func (l *log) Write(ctx context.Context, stepID int64, entries []*model.LogEntry) error { l.Lock() s, ok := l.streams[stepID] l.Unlock() @@ -92,15 +94,20 @@ func (l *log) Write(ctx context.Context, stepID int64, logEntry *model.LogEntry) } s.Lock() - s.list = append(s.list, logEntry) + s.list = append(s.list, entries...) for sub := range s.subs { - go sub.handler(logEntry) + select { + case sub.receiver <- entries: + default: + logger.Info().Msgf("subscriber channel is full -- dropping logs for step %d", stepID) + } } s.Unlock() + return nil } -func (l *log) Tail(c context.Context, stepID int64, handler Handler) error { +func (l *log) Tail(c context.Context, stepID int64, receiver LogChan) error { l.Lock() s, ok := l.streams[stepID] l.Unlock() @@ -109,11 +116,11 @@ func (l *log) Tail(c context.Context, stepID int64, handler Handler) error { } sub := &subscriber{ - handler: handler, + receiver: receiver, } s.Lock() if len(s.list) != 0 { - sub.handler(s.list...) + sub.receiver <- s.list } s.subs[sub] = struct{}{} s.Unlock() diff --git a/server/logging/log_test.go b/server/logging/log_test.go index 90565a48f..f84a5b8f3 100644 --- a/server/logging/log_test.go +++ b/server/logging/log_test.go @@ -39,28 +39,37 @@ func TestLogging(t *testing.T) { context.Background(), ) + receiver := make(LogChan, 10) + defer close(receiver) + + go func() { + for range receiver { + wg.Done() + } + }() + logger := New() assert.NoError(t, logger.Open(ctx, testStepID)) go func() { - assert.NoError(t, logger.Tail(ctx, testStepID, func(_ ...*model.LogEntry) { wg.Done() })) + assert.NoError(t, logger.Tail(ctx, testStepID, receiver)) }() go func() { - assert.NoError(t, logger.Tail(ctx, testStepID, func(_ ...*model.LogEntry) { wg.Done() })) + assert.NoError(t, logger.Tail(ctx, testStepID, receiver)) }() <-time.After(500 * time.Millisecond) wg.Add(4) go func() { - assert.NoError(t, logger.Write(ctx, testStepID, testEntry)) - assert.NoError(t, logger.Write(ctx, testStepID, testEntry)) + assert.NoError(t, logger.Write(ctx, testStepID, []*model.LogEntry{testEntry})) + assert.NoError(t, logger.Write(ctx, testStepID, []*model.LogEntry{testEntry})) }() wg.Wait() wg.Add(1) go func() { - assert.NoError(t, logger.Tail(ctx, testStepID, func(_ ...*model.LogEntry) { wg.Done() })) + assert.NoError(t, logger.Tail(ctx, testStepID, receiver)) }() <-time.After(500 * time.Millisecond) diff --git a/server/logging/logging.go b/server/logging/logging.go index 400273def..e8eb80117 100644 --- a/server/logging/logging.go +++ b/server/logging/logging.go @@ -24,8 +24,8 @@ import ( // ErrNotFound is returned when the log does not exist. var ErrNotFound = errors.New("stream: not found") -// Handler defines a callback function for handling log entries. -type Handler func(...*model.LogEntry) +// LogChan defines a channel type for receiving ordered batches of log entries. +type LogChan chan []*model.LogEntry // Log defines a log multiplexer. type Log interface { @@ -33,10 +33,10 @@ type Log interface { Open(c context.Context, stepID int64) error // Write writes the entry to the log. - Write(c context.Context, stepID int64, entry *model.LogEntry) error + Write(c context.Context, stepID int64, entries []*model.LogEntry) error // Tail tails the log. - Tail(c context.Context, stepID int64, handler Handler) error + Tail(c context.Context, stepID int64, handler LogChan) error // Close closes the log. Close(c context.Context, stepID int64) error diff --git a/server/model/task.go b/server/model/task.go index bd24ca665..3f73bebed 100644 --- a/server/model/task.go +++ b/server/model/task.go @@ -22,7 +22,7 @@ import ( // Task defines scheduled pipeline Task. type Task struct { ID string `json:"id" xorm:"PK UNIQUE 'id'"` - Data []byte `json:"data" xorm:"LONGBLOB 'data'"` + Data []byte `json:"-" xorm:"LONGBLOB 'data'"` Labels map[string]string `json:"labels" xorm:"json 'labels'"` Dependencies []string `json:"dependencies" xorm:"json 'dependencies'"` RunOn []string `json:"run_on" xorm:"json 'run_on'"` diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index da0888511..3c997a290 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -36,10 +36,8 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin return nil, errors.New(msg) } - switch lastPipeline.Status { - case model.StatusDeclined, - model.StatusBlocked: - return nil, &ErrBadRequest{Msg: fmt.Sprintf("cannot restart a pipeline with status %s", lastPipeline.Status)} + if lastPipeline.Status == model.StatusBlocked { + return nil, &ErrBadRequest{Msg: "cannot restart a pipeline with status blocked"} } // fetch the old pipeline config from the database diff --git a/server/pipeline/stepbuilder/metadata.go b/server/pipeline/stepbuilder/metadata.go index 6e48e4158..3ea13f9e4 100644 --- a/server/pipeline/stepbuilder/metadata.go +++ b/server/pipeline/stepbuilder/metadata.go @@ -48,6 +48,7 @@ func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, Owner: repo.Owner, RemoteID: fmt.Sprint(repo.ForgeRemoteID), ForgeURL: repo.ForgeURL, + SCM: string(repo.SCMKind), CloneURL: repo.Clone, CloneSSHURL: repo.CloneSSH, Private: repo.IsSCMPrivate, diff --git a/server/pipeline/stepbuilder/metadata_test.go b/server/pipeline/stepbuilder/metadata_test.go index fd85f8d07..368eae47e 100644 --- a/server/pipeline/stepbuilder/metadata_test.go +++ b/server/pipeline/stepbuilder/metadata_test.go @@ -53,7 +53,7 @@ func TestMetadataFromStruct(t *testing.T) { "CI_PREV_COMMIT_MESSAGE": "", "CI_PREV_COMMIT_REF": "", "CI_PREV_COMMIT_REFSPEC": "", "CI_PREV_COMMIT_SHA": "", "CI_PREV_COMMIT_URL": "", "CI_PREV_PIPELINE_CREATED": "0", "CI_PREV_PIPELINE_DEPLOY_TARGET": "", "CI_PREV_PIPELINE_DEPLOY_TASK": "", "CI_PREV_PIPELINE_EVENT": "", "CI_PREV_PIPELINE_FINISHED": "0", "CI_PREV_PIPELINE_NUMBER": "0", "CI_PREV_PIPELINE_PARENT": "0", "CI_PREV_PIPELINE_STARTED": "0", "CI_PREV_PIPELINE_STATUS": "", "CI_PREV_PIPELINE_URL": "/repos/0/pipeline/0", "CI_PREV_PIPELINE_FORGE_URL": "", "CI_REPO": "", "CI_REPO_CLONE_URL": "", "CI_REPO_CLONE_SSH_URL": "", "CI_REPO_DEFAULT_BRANCH": "", "CI_REPO_REMOTE_ID": "", - "CI_REPO_NAME": "", "CI_REPO_OWNER": "", "CI_REPO_PRIVATE": "false", "CI_REPO_SCM": "git", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "", + "CI_REPO_NAME": "", "CI_REPO_OWNER": "", "CI_REPO_PRIVATE": "false", "CI_REPO_SCM": "", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "", "CI_STEP_NAME": "", "CI_STEP_NUMBER": "0", "CI_STEP_STARTED": "", "CI_STEP_URL": "/repos/0/pipeline/0", "CI_SYSTEM_HOST": "", "CI_SYSTEM_NAME": "woodpecker", "CI_SYSTEM_PLATFORM": "", "CI_SYSTEM_URL": "", "CI_SYSTEM_VERSION": "", "CI_WORKFLOW_NAME": "", "CI_WORKFLOW_NUMBER": "0", }, @@ -61,7 +61,7 @@ func TestMetadataFromStruct(t *testing.T) { { name: "Test with forge", forge: forge, - repo: &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true}, + repo: &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true, SCMKind: "git"}, pipeline: &model.Pipeline{Number: 3, ChangedFiles: []string{"test.go", "markdown file.md"}}, last: &model.Pipeline{Number: 2}, workflow: &model.Workflow{Name: "hello"}, @@ -69,7 +69,7 @@ func TestMetadataFromStruct(t *testing.T) { expectedMetadata: metadata.Metadata{ Forge: metadata.Forge{Type: "gitea", URL: "https://gitea.com"}, Sys: metadata.System{Name: "woodpecker", Host: "example.com", URL: "https://example.com"}, - Repo: metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true}, + Repo: metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true, SCM: "git"}, Curr: metadata.Pipeline{ Number: 3, Commit: metadata.Commit{ChangedFiles: []string{"test.go", "markdown file.md"}}, diff --git a/server/queue/fifo.go b/server/queue/fifo.go index d6fe664c9..5e5a17edd 100644 --- a/server/queue/fifo.go +++ b/server/queue/fifo.go @@ -24,6 +24,7 @@ import ( "github.com/rs/zerolog/log" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/shared/constant" ) type entry struct { @@ -43,6 +44,7 @@ type worker struct { type fifo struct { sync.Mutex + ctx context.Context workers map[*worker]struct{} running map[string]*entry pending *list.List @@ -51,18 +53,23 @@ type fifo struct { paused bool } +// processTimeInterval is the time till the queue rearranges things, +// as the agent pull in 10 milliseconds we should also give them work asap. +const processTimeInterval = 100 * time.Millisecond + // New returns a new fifo queue. -// -//nolint:mnd -func New(_ context.Context) Queue { - return &fifo{ +func New(ctx context.Context) Queue { + q := &fifo{ + ctx: ctx, workers: map[*worker]struct{}{}, running: map[string]*entry{}, pending: list.New(), waitingOnDeps: list.New(), - extension: time.Minute * 10, + extension: constant.TaskTimeout, paused: false, } + go q.process() + return q } // Push pushes a task to the tail of this queue. @@ -70,7 +77,6 @@ func (q *fifo) Push(_ context.Context, task *model.Task) error { q.Lock() q.pending.PushBack(task) q.Unlock() - go q.process() return nil } @@ -81,7 +87,6 @@ func (q *fifo) PushAtOnce(_ context.Context, tasks []*model.Task) error { q.pending.PushBack(task) } q.Unlock() - go q.process() return nil } @@ -98,7 +103,6 @@ func (q *fifo) Poll(c context.Context, agentID int64, f FilterFn) (*model.Task, } q.workers[w] = struct{}{} q.Unlock() - go q.process() for { select { @@ -237,7 +241,6 @@ func (q *fifo) Resume() { q.Lock() q.paused = false q.Unlock() - go q.process() } // KickAgentWorkers kicks all workers for a given agent. @@ -254,28 +257,36 @@ func (q *fifo) KickAgentWorkers(agentID int64) { } // helper function that loops through the queue and attempts to -// match the item to a single subscriber. +// match the item to a single subscriber until context got cancel. func (q *fifo) process() { - q.Lock() - defer q.Unlock() - - if q.paused { - return - } - - q.resubmitExpiredPipelines() - q.filterWaiting() - for pending, worker := q.assignToWorker(); pending != nil && worker != nil; pending, worker = q.assignToWorker() { - task, _ := pending.Value.(*model.Task) - task.AgentID = worker.agentID - delete(q.workers, worker) - q.pending.Remove(pending) - q.running[task.ID] = &entry{ - item: task, - done: make(chan bool), - deadline: time.Now().Add(q.extension), + for { + select { + case <-time.After(processTimeInterval): + case <-q.ctx.Done(): + return } - worker.channel <- task + + q.Lock() + if q.paused { + q.Unlock() + continue + } + + q.resubmitExpiredPipelines() + q.filterWaiting() + for pending, worker := q.assignToWorker(); pending != nil && worker != nil; pending, worker = q.assignToWorker() { + task, _ := pending.Value.(*model.Task) + task.AgentID = worker.agentID + delete(q.workers, worker) + q.pending.Remove(pending) + q.running[task.ID] = &entry{ + item: task, + done: make(chan bool), + deadline: time.Now().Add(q.extension), + } + worker.channel <- task + } + q.Unlock() } } diff --git a/server/queue/fifo_test.go b/server/queue/fifo_test.go index d1d3a2ebc..0de92e3f2 100644 --- a/server/queue/fifo_test.go +++ b/server/queue/fifo_test.go @@ -52,17 +52,23 @@ func TestFifo(t *testing.T) { func TestFifoExpire(t *testing.T) { want := &model.Task{ID: "1"} + ctx, cancel := context.WithCancelCause(context.Background()) - q, _ := New(context.Background()).(*fifo) + q, _ := New(ctx).(*fifo) q.extension = 0 - assert.NoError(t, q.Push(noContext, want)) - info := q.Info(noContext) + assert.NoError(t, q.Push(ctx, want)) + info := q.Info(ctx) assert.Len(t, info.Pending, 1, "expect task in pending queue") - got, err := q.Poll(noContext, 1, func(*model.Task) bool { return true }) + got, err := q.Poll(ctx, 1, func(*model.Task) bool { return true }) assert.NoError(t, err) assert.Equal(t, want, got) + // cancel the context to let the process func end + go func() { + time.Sleep(time.Millisecond) + cancel(nil) + }() q.process() assert.Len(t, info.Pending, 1, "expect task re-added to pending queue") } diff --git a/server/services/log/file/file.go b/server/services/log/file/file.go index 551a05d89..f19931beb 100644 --- a/server/services/log/file/file.go +++ b/server/services/log/file/file.go @@ -8,6 +8,8 @@ import ( "path/filepath" "strings" + logger "github.com/rs/zerolog/log" + "go.woodpecker-ci.org/woodpecker/v2/pipeline" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/services/log" @@ -70,19 +72,30 @@ func (l logStore) LogFind(step *model.Step) ([]*model.LogEntry, error) { return entries, nil } -func (l logStore) LogAppend(logEntry *model.LogEntry) error { - file, err := os.OpenFile(l.filePath(logEntry.StepID), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) +func (l logStore) LogAppend(step *model.Step, logEntries []*model.LogEntry) error { + path := l.filePath(step.ID) + + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { + logger.Error().Err(err).Msgf("could not open log file %s", path) return err } - jsonData, err := json.Marshal(logEntry) - if err != nil { - return err + + var bytes []byte + + for _, logEntry := range logEntries { + if jsonLine, err := json.Marshal(logEntry); err == nil { + bytes = append(bytes, jsonLine...) + bytes = append(bytes, byte('\n')) + } else { + logger.Error().Err(err).Msg("could not convert log entry to JSON") + } } - _, err = file.Write(append(jsonData, byte('\n'))) - if err != nil { - return err + + if _, err = file.Write(bytes); err != nil { + logger.Error().Err(err).Msg("could not write out log entries") } + return file.Close() } diff --git a/server/services/log/service.go b/server/services/log/service.go index 5cc53d1e9..da8f98146 100644 --- a/server/services/log/service.go +++ b/server/services/log/service.go @@ -4,6 +4,6 @@ import "go.woodpecker-ci.org/woodpecker/v2/server/model" type Service interface { LogFind(step *model.Step) ([]*model.LogEntry, error) - LogAppend(logEntry *model.LogEntry) error + LogAppend(step *model.Step, logEntries []*model.LogEntry) error LogDelete(step *model.Step) error } diff --git a/server/store/datastore/log.go b/server/store/datastore/log.go index 68c708c63..3aedf4650 100644 --- a/server/store/datastore/log.go +++ b/server/store/datastore/log.go @@ -15,16 +15,33 @@ package datastore import ( + "github.com/rs/zerolog/log" + "go.woodpecker-ci.org/woodpecker/v2/server/model" ) +// Maximum number of records to store in one PostgreSQL statement. +// Too large a value results in `pq: got XX parameters but PostgreSQL only supports 65535 parameters`. +const pgBatchSize = 1000 + func (s storage) LogFind(step *model.Step) ([]*model.LogEntry, error) { var logEntries []*model.LogEntry return logEntries, s.engine.Asc("id").Where("step_id = ?", step.ID).Find(&logEntries) } -func (s storage) LogAppend(logEntry *model.LogEntry) error { - _, err := s.engine.Insert(logEntry) +func (s storage) LogAppend(_ *model.Step, logEntries []*model.LogEntry) error { + var err error + + // TODO: adapted from slices.Chunk(); switch to it in Go 1.23+ + for i := 0; i < len(logEntries); i += pgBatchSize { + end := min(pgBatchSize, len(logEntries[i:])) + chunk := logEntries[i : i+end] + + if _, err = s.engine.Insert(chunk); err != nil { + log.Error().Err(err).Msg("could not store log entries to db") + } + } + return err } diff --git a/server/store/datastore/log_test.go b/server/store/datastore/log_test.go index f2ee1ffc6..94897268c 100644 --- a/server/store/datastore/log_test.go +++ b/server/store/datastore/log_test.go @@ -45,9 +45,7 @@ func TestLogCreateFindDelete(t *testing.T) { }, } - for _, logEntry := range logEntries { - assert.NoError(t, store.LogAppend(logEntry)) - } + assert.NoError(t, store.LogAppend(&step, logEntries)) // we want to find our inserted logs _logEntries, err := store.LogFind(&step) @@ -83,9 +81,7 @@ func TestLogAppend(t *testing.T) { }, } - for _, logEntry := range logEntries { - assert.NoError(t, store.LogAppend(logEntry)) - } + assert.NoError(t, store.LogAppend(&step, logEntries)) logEntry := &model.LogEntry{ StepID: step.ID, @@ -94,7 +90,7 @@ func TestLogAppend(t *testing.T) { Time: 20, } - assert.NoError(t, store.LogAppend(logEntry)) + assert.NoError(t, store.LogAppend(&step, []*model.LogEntry{logEntry})) _logEntries, err := store.LogFind(&step) assert.NoError(t, err) diff --git a/server/store/datastore/migration/023_add_org_id.go b/server/store/datastore/migration/001_add_org_id.go similarity index 95% rename from server/store/datastore/migration/023_add_org_id.go rename to server/store/datastore/migration/001_add_org_id.go index 37d59833a..877ab58cd 100644 --- a/server/store/datastore/migration/023_add_org_id.go +++ b/server/store/datastore/migration/001_add_org_id.go @@ -26,12 +26,12 @@ import ( var addOrgID = xormigrate.Migration{ ID: "add-org-id", MigrateSession: func(sess *xorm.Session) error { - if err := sess.Sync(new(userV031)); err != nil { + if err := sess.Sync(new(userV009)); err != nil { return fmt.Errorf("sync new models failed: %w", err) } // get all users - var users []*userV031 + var users []*userV009 if err := sess.Find(&users); err != nil { return fmt.Errorf("find all repos failed: %w", err) } diff --git a/server/store/datastore/migration/001_legacy_to_xorm.go b/server/store/datastore/migration/001_legacy_to_xorm.go deleted file mode 100644 index 47a526147..000000000 --- a/server/store/datastore/migration/001_legacy_to_xorm.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2021 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "fmt" - - "github.com/rs/zerolog/log" - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" - "xorm.io/xorm/schemas" -) - -var legacy2Xorm = xormigrate.Migration{ - ID: "xorm", - MigrateSession: func(sess *xorm.Session) error { - // make sure we have required migrations - else fail and point to last major version - for _, mig := range []string{ - // users - "create-table-users", - "update-table-set-users-token-and-secret-length", - // repos - "create-table-repos", - "alter-table-add-repo-visibility", - "update-table-set-repo-visibility", - "alter-table-add-repo-seq", - "update-table-set-repo-seq", - "update-table-set-repo-seq-default", - "alter-table-add-repo-active", - "update-table-set-repo-active", - "alter-table-add-repo-fallback", // needed to drop col - // builds - "create-table-builds", - "create-index-builds-repo", - "create-index-builds-author", - // procs - "create-table-procs", - "create-index-procs-build", - // files - "create-table-files", - "create-index-files-builds", - "create-index-files-procs", - "alter-table-add-file-pid", - "alter-table-add-file-meta-passed", - "alter-table-add-file-meta-failed", - "alter-table-add-file-meta-skipped", - "alter-table-update-file-meta", - // secrets - "create-table-secrets", - "create-index-secrets-repo", - // registry - "create-table-registry", - "create-index-registry-repo", - // senders - "create-table-senders", - "create-index-sender-repos", - // perms - "create-table-perms", - "create-index-perms-repo", - "create-index-perms-user", - // build_config - "create-table-build-config", - "populate-build-config", - } { - exist, err := sess.Exist(&xormigrate.Migration{ID: mig}) - if err != nil { - return fmt.Errorf("test migration existence: %w", err) - } - if !exist { - log.Error().Msgf("migration step '%s' missing, please upgrade to last stable v0.14.x version first", mig) - return fmt.Errorf("legacy migration step missing") - } - } - - { // recreate build_config - type BuildConfig struct { - ConfigID int64 `xorm:"NOT NULL 'config_id'"` // xorm.Sync() do not use index info of sess -> so it tries to create it twice - BuildID int64 `xorm:"NOT NULL 'build_id'"` - } - if err := renameTable(sess, "build_config", "old_build_config"); err != nil { - return err - } - if err := sess.Sync(new(BuildConfig)); err != nil { - return err - } - if _, err := sess.Exec("INSERT INTO build_config (config_id, build_id) SELECT config_id,build_id FROM old_build_config;"); err != nil { - return fmt.Errorf("unable to set copy data into temp table %s. Error: %w", "old_build_config", err) - } - if err := sess.DropTable("old_build_config"); err != nil { - return fmt.Errorf("could not drop table '%s': %w", "old_build_config", err) - } - } - - dialect := sess.Engine().Dialect().URI().DBType - switch dialect { - case schemas.MYSQL: - for _, exec := range []string{ - "DROP INDEX IF EXISTS build_number ON builds;", - "DROP INDEX IF EXISTS ix_build_repo ON builds;", - "DROP INDEX IF EXISTS ix_build_author ON builds;", - "DROP INDEX IF EXISTS proc_build_ix ON procs;", - "DROP INDEX IF EXISTS file_build_ix ON files;", - "DROP INDEX IF EXISTS file_proc_ix ON files;", - "DROP INDEX IF EXISTS ix_secrets_repo ON secrets;", - "DROP INDEX IF EXISTS ix_registry_repo ON registry;", - "DROP INDEX IF EXISTS sender_repo_ix ON senders;", - "DROP INDEX IF EXISTS ix_perms_repo ON perms;", - "DROP INDEX IF EXISTS ix_perms_user ON perms;", - } { - if _, err := sess.Exec(exec); err != nil { - return fmt.Errorf("exec: '%s' failed: %w", exec, err) - } - } - case schemas.SQLITE, schemas.POSTGRES: - for _, exec := range []string{ - "DROP INDEX IF EXISTS ix_build_status_running;", - "DROP INDEX IF EXISTS ix_build_repo;", - "DROP INDEX IF EXISTS ix_build_author;", - "DROP INDEX IF EXISTS proc_build_ix;", - "DROP INDEX IF EXISTS file_build_ix;", - "DROP INDEX IF EXISTS file_proc_ix;", - "DROP INDEX IF EXISTS ix_secrets_repo;", - "DROP INDEX IF EXISTS ix_registry_repo;", - "DROP INDEX IF EXISTS sender_repo_ix;", - "DROP INDEX IF EXISTS ix_perms_repo;", - "DROP INDEX IF EXISTS ix_perms_user;", - } { - if _, err := sess.Exec(exec); err != nil { - return fmt.Errorf("exec: '%s' failed: %w", exec, err) - } - } - default: - return fmt.Errorf("dialect '%s' not supported", dialect) - } - - return nil - }, -} diff --git a/server/store/datastore/migration/002_repos_drop_fallback.go b/server/store/datastore/migration/002_repos_drop_fallback.go deleted file mode 100644 index 111bd3eb0..000000000 --- a/server/store/datastore/migration/002_repos_drop_fallback.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2021 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var alterTableReposDropFallback = xormigrate.Migration{ - ID: "alter-table-drop-repo-fallback", - MigrateSession: func(sess *xorm.Session) error { - return dropTableColumns(sess, "repos", "repo_fallback") - }, -} diff --git a/server/store/datastore/migration/024_task_data_type.go b/server/store/datastore/migration/002_task_data_type.go similarity index 100% rename from server/store/datastore/migration/024_task_data_type.go rename to server/store/datastore/migration/002_task_data_type.go diff --git a/server/store/datastore/migration/025_config_data_type.go b/server/store/datastore/migration/003_config_data_type.go similarity index 100% rename from server/store/datastore/migration/025_config_data_type.go rename to server/store/datastore/migration/003_config_data_type.go diff --git a/server/store/datastore/migration/003_repos_drop_allow_deploys_allow_tags.go b/server/store/datastore/migration/003_repos_drop_allow_deploys_allow_tags.go deleted file mode 100644 index d99610f56..000000000 --- a/server/store/datastore/migration/003_repos_drop_allow_deploys_allow_tags.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2021 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var alterTableReposDropAllowDeploysAllowTags = xormigrate.Migration{ - ID: "drop-allow-push-tags-deploys-columns", - MigrateSession: func(sess *xorm.Session) error { - return dropTableColumns(sess, "repos", - "repo_allow_deploys", - "repo_allow_tags", - ) - }, -} diff --git a/server/store/datastore/migration/004_fix_pr_secret_event_name.go b/server/store/datastore/migration/004_fix_pr_secret_event_name.go deleted file mode 100644 index 66406a07b..000000000 --- a/server/store/datastore/migration/004_fix_pr_secret_event_name.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2021 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" - - "go.woodpecker-ci.org/woodpecker/v2/server/model" -) - -var fixPRSecretEventName = xormigrate.Migration{ - ID: "fix-pr-secret-event-name", - MigrateSession: func(sess *xorm.Session) error { - const batchSize = 100 - for start := 0; ; start += batchSize { - secrets := make([]*model.Secret, 0, batchSize) - if err := sess.Limit(batchSize, start).Table("secrets").Cols("secret_id", "secret_events").Where("secret_events LIKE '%pull-request%'").Find(&secrets); err != nil { - return err - } - - if len(secrets) == 0 { - break - } - - for _, secret := range secrets { - for i, event := range secret.Events { - if event == "pull-request" { - secret.Events[i] = "pull_request" - } - } - if _, err := sess.ID(secret.ID).Cols("secret_events").Update(secret); err != nil { - return err - } - } - } - return nil - }, -} diff --git a/server/store/datastore/migration/026_remove_secrets_plugin_only_col.go b/server/store/datastore/migration/004_remove_secrets_plugin_only_col.go similarity index 92% rename from server/store/datastore/migration/026_remove_secrets_plugin_only_col.go rename to server/store/datastore/migration/004_remove_secrets_plugin_only_col.go index 082744cca..8a91ffb36 100644 --- a/server/store/datastore/migration/026_remove_secrets_plugin_only_col.go +++ b/server/store/datastore/migration/004_remove_secrets_plugin_only_col.go @@ -19,7 +19,7 @@ import ( "xorm.io/xorm" ) -type oldSecret026 struct { +type oldSecret004 struct { ID int64 `json:"id" xorm:"pk autoincr 'secret_id'"` PluginsOnly bool `json:"plugins_only" xorm:"secret_plugins_only"` SkipVerify bool `json:"-" xorm:"secret_skip_verify"` @@ -27,7 +27,7 @@ type oldSecret026 struct { Images []string `json:"images" xorm:"json 'secret_images'"` } -func (oldSecret026) TableName() string { +func (oldSecret004) TableName() string { return "secrets" } @@ -35,7 +35,7 @@ var removePluginOnlyOptionFromSecretsTable = xormigrate.Migration{ ID: "remove-plugin-only-option-from-secrets-table", MigrateSession: func(sess *xorm.Session) (err error) { // make sure plugin_only column exists - if err := sess.Sync(new(oldSecret026)); err != nil { + if err := sess.Sync(new(oldSecret004)); err != nil { return err } diff --git a/server/store/datastore/migration/027_convert_to_new_pipeline_errors_format.go b/server/store/datastore/migration/005_convert_to_new_pipeline_errors_format.go similarity index 83% rename from server/store/datastore/migration/027_convert_to_new_pipeline_errors_format.go rename to server/store/datastore/migration/005_convert_to_new_pipeline_errors_format.go index a51ac1c6a..53d18c8fd 100644 --- a/server/store/datastore/migration/027_convert_to_new_pipeline_errors_format.go +++ b/server/store/datastore/migration/005_convert_to_new_pipeline_errors_format.go @@ -21,20 +21,20 @@ import ( errorTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/errors/types" ) -// perPage027 set the size of the slice to read per page. -var perPage027 = 100 +// perPage005 set the size of the slice to read per page. +var perPage005 = 100 -type pipeline027 struct { +type pipeline005 struct { ID int64 `json:"id" xorm:"pk autoincr 'pipeline_id'"` Error string `json:"error" xorm:"LONGTEXT 'pipeline_error'"` // old error format Errors []*errorTypes.PipelineError `json:"errors" xorm:"json 'pipeline_errors'"` // new error format } -func (pipeline027) TableName() string { +func (pipeline005) TableName() string { return "pipelines" } -type PipelineError027 struct { +type PipelineError005 struct { Type string `json:"type"` Message string `json:"message"` IsWarning bool `json:"is_warning"` @@ -46,23 +46,23 @@ var convertToNewPipelineErrorFormat = xormigrate.Migration{ Long: true, MigrateSession: func(sess *xorm.Session) (err error) { // make sure pipeline_error column exists - if err := sess.Sync(new(pipeline027)); err != nil { + if err := sess.Sync(new(pipeline005)); err != nil { return err } page := 0 - oldPipelines := make([]*pipeline027, 0, perPage027) + oldPipelines := make([]*pipeline005, 0, perPage005) for { oldPipelines = oldPipelines[:0] - err := sess.Limit(perPage027, page*perPage027).Cols("pipeline_id", "pipeline_error").Where("pipeline_error != ''").Find(&oldPipelines) + err := sess.Limit(perPage005, page*perPage005).Cols("pipeline_id", "pipeline_error").Where("pipeline_error != ''").Find(&oldPipelines) if err != nil { return err } for _, oldPipeline := range oldPipelines { - var newPipeline pipeline027 + var newPipeline pipeline005 newPipeline.ID = oldPipeline.ID newPipeline.Errors = []*errorTypes.PipelineError{{ Type: "generic", @@ -74,7 +74,7 @@ var convertToNewPipelineErrorFormat = xormigrate.Migration{ } } - if len(oldPipelines) < perPage027 { + if len(oldPipelines) < perPage005 { break } diff --git a/server/store/datastore/migration/005_repos_drop_repo_counter.go b/server/store/datastore/migration/005_repos_drop_repo_counter.go deleted file mode 100644 index dc1785e0c..000000000 --- a/server/store/datastore/migration/005_repos_drop_repo_counter.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2021 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var alterTableReposDropCounter = xormigrate.Migration{ - ID: "alter-table-drop-counter", - MigrateSession: func(sess *xorm.Session) error { - return dropTableColumns(sess, "repos", "repo_counter") - }, -} diff --git a/server/store/datastore/migration/006_drop_senders.go b/server/store/datastore/migration/006_drop_senders.go deleted file mode 100644 index f547bda09..000000000 --- a/server/store/datastore/migration/006_drop_senders.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var dropSenders = xormigrate.Migration{ - ID: "drop-senders", - MigrateSession: func(sess *xorm.Session) error { - return sess.DropTable("senders") - }, -} diff --git a/server/store/datastore/migration/028_link_to_url.go b/server/store/datastore/migration/006_link_to_url.go similarity index 100% rename from server/store/datastore/migration/028_link_to_url.go rename to server/store/datastore/migration/006_link_to_url.go diff --git a/server/store/datastore/migration/029_clean_registry_pipeline.go b/server/store/datastore/migration/007_clean_registry_pipeline.go similarity index 89% rename from server/store/datastore/migration/029_clean_registry_pipeline.go rename to server/store/datastore/migration/007_clean_registry_pipeline.go index 21eb4277b..a0b5dba87 100644 --- a/server/store/datastore/migration/029_clean_registry_pipeline.go +++ b/server/store/datastore/migration/007_clean_registry_pipeline.go @@ -19,17 +19,17 @@ import ( "xorm.io/xorm" ) -type oldRegistry029 struct { +type oldRegistry007 struct { ID int64 `json:"id" xorm:"pk autoincr 'registry_id'"` Token string `json:"token" xorm:"TEXT 'registry_token'"` Email string `json:"email" xorm:"varchar(500) 'registry_email'"` } -func (oldRegistry029) TableName() string { +func (oldRegistry007) TableName() string { return "registry" } -type oldPipeline029 struct { +type oldPipeline007 struct { ID int64 `json:"id" xorm:"pk autoincr 'pipeline_id'"` ConfigID int64 `json:"-" xorm:"pipeline_config_id"` Enqueued int64 `json:"enqueued_at" xorm:"pipeline_enqueued"` @@ -37,14 +37,14 @@ type oldPipeline029 struct { } // TableName return database table name for xorm. -func (oldPipeline029) TableName() string { +func (oldPipeline007) TableName() string { return "pipelines" } var cleanRegistryPipeline = xormigrate.Migration{ ID: "clean-registry-pipeline", MigrateSession: func(sess *xorm.Session) (err error) { - if err := sess.Sync(new(oldRegistry029), new(oldPipeline029)); err != nil { + if err := sess.Sync(new(oldRegistry007), new(oldPipeline007)); err != nil { return err } diff --git a/server/store/datastore/migration/007_log_data_type.go b/server/store/datastore/migration/007_log_data_type.go deleted file mode 100644 index cb70c5d2c..000000000 --- a/server/store/datastore/migration/007_log_data_type.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2023 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" - "xorm.io/xorm/schemas" -) - -var alterTableLogUpdateColumnLogDataType = xormigrate.Migration{ - ID: "alter-table-logs-update-type-of-data", - MigrateSession: func(sess *xorm.Session) (err error) { - dialect := sess.Engine().Dialect().URI().DBType - - switch dialect { - case schemas.POSTGRES: - _, err = sess.Exec("ALTER TABLE logs ALTER COLUMN log_data TYPE BYTEA") - case schemas.MYSQL: - _, err = sess.Exec("ALTER TABLE logs MODIFY COLUMN log_data LONGBLOB") - default: - // sqlite does only know BLOB in all cases - return nil - } - - return err - }, -} diff --git a/server/store/datastore/migration/008_secrets_add_user.go b/server/store/datastore/migration/008_secrets_add_user.go deleted file mode 100644 index 6755a3b62..000000000 --- a/server/store/datastore/migration/008_secrets_add_user.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -type SecretV008 struct { - Owner string `json:"-" xorm:"NOT NULL DEFAULT '' UNIQUE(s) INDEX 'secret_owner'"` - RepoID int64 `json:"-" xorm:"NOT NULL DEFAULT 0 UNIQUE(s) INDEX 'secret_repo_id'"` - Name string `json:"name" xorm:"NOT NULL UNIQUE(s) INDEX 'secret_name'"` -} - -// TableName return database table name for xorm. -func (SecretV008) TableName() string { - return "secrets" -} - -var alterTableSecretsAddUserCol = xormigrate.Migration{ - ID: "alter-table-add-secrets-user-id", - MigrateSession: func(sess *xorm.Session) error { - if err := sess.Sync(new(SecretV008)); err != nil { - return err - } - if err := alterColumnDefault(sess, "secrets", "secret_repo_id", "0"); err != nil { - return err - } - if err := alterColumnNull(sess, "secrets", "secret_repo_id", false); err != nil { - return err - } - return alterColumnNull(sess, "secrets", "secret_name", false) - }, -} diff --git a/server/store/datastore/migration/030_set_default_forge_id.go b/server/store/datastore/migration/008_set_default_forge_id.go similarity index 93% rename from server/store/datastore/migration/030_set_default_forge_id.go rename to server/store/datastore/migration/008_set_default_forge_id.go index bbd9300c3..4edc817f9 100644 --- a/server/store/datastore/migration/030_set_default_forge_id.go +++ b/server/store/datastore/migration/008_set_default_forge_id.go @@ -23,7 +23,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -type userV030 struct { +type userV008 struct { ID int64 `xorm:"pk autoincr 'user_id'"` ForgeID int64 `xorm:"forge_id"` ForgeRemoteID model.ForgeRemoteID `xorm:"forge_remote_id"` @@ -38,11 +38,11 @@ type userV030 struct { OrgID int64 `xorm:"user_org_id"` } -func (userV030) TableName() string { +func (userV008) TableName() string { return "users" } -type repoV030 struct { +type repoV008 struct { ID int64 `xorm:"pk autoincr 'repo_id'"` UserID int64 `xorm:"repo_user_id"` ForgeID int64 `xorm:"forge_id"` @@ -73,11 +73,11 @@ type repoV030 struct { NetrcOnlyTrusted bool `xorm:"NOT NULL DEFAULT true 'netrc_only_trusted'"` } -func (repoV030) TableName() string { +func (repoV008) TableName() string { return "repos" } -type forgeV030 struct { +type forgeV008 struct { ID int64 `xorm:"pk autoincr 'id'"` Type model.ForgeType `xorm:"VARCHAR(250) 'type'"` URL string `xorm:"VARCHAR(500) 'url'"` @@ -88,18 +88,18 @@ type forgeV030 struct { AdditionalOptions map[string]any `xorm:"json 'additional_options'"` } -func (forgeV030) TableName() string { +func (forgeV008) TableName() string { return "forge" } var setForgeID = xormigrate.Migration{ ID: "set-forge-id", MigrateSession: func(sess *xorm.Session) (err error) { - if err := sess.Sync(new(userV030), new(repoV030), new(forgeV030), new(model.Org)); err != nil { + if err := sess.Sync(new(userV008), new(repoV008), new(forgeV008), new(model.Org)); err != nil { return fmt.Errorf("sync new models failed: %w", err) } - _, err = sess.Exec(fmt.Sprintf("UPDATE `%s` SET forge_id=1;", userV030{}.TableName())) + _, err = sess.Exec(fmt.Sprintf("UPDATE `%s` SET forge_id=1;", userV008{}.TableName())) if err != nil { return err } @@ -109,7 +109,7 @@ var setForgeID = xormigrate.Migration{ return err } - _, err = sess.Exec(fmt.Sprintf("UPDATE `%s` SET forge_id=1;", repoV030{}.TableName())) + _, err = sess.Exec(fmt.Sprintf("UPDATE `%s` SET forge_id=1;", repoV008{}.TableName())) return err }, } diff --git a/server/store/datastore/migration/009_recreate_agents_table.go b/server/store/datastore/migration/009_recreate_agents_table.go deleted file mode 100644 index b0a8d1f60..000000000 --- a/server/store/datastore/migration/009_recreate_agents_table.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" - - "go.woodpecker-ci.org/woodpecker/v2/server/model" -) - -var recreateAgentsTable = xormigrate.Migration{ - ID: "recreate-agents-table", - MigrateSession: func(sess *xorm.Session) error { - if err := sess.DropTable("agents"); err != nil { - return err - } - return sess.Sync(new(model.Agent)) - }, -} diff --git a/server/store/datastore/migration/031_unify_columns_tables.go b/server/store/datastore/migration/009_unify_columns_tables.go similarity index 94% rename from server/store/datastore/migration/031_unify_columns_tables.go rename to server/store/datastore/migration/009_unify_columns_tables.go index 881832b0f..db8578b2f 100644 --- a/server/store/datastore/migration/031_unify_columns_tables.go +++ b/server/store/datastore/migration/009_unify_columns_tables.go @@ -24,7 +24,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -type configV031 struct { +type configV009 struct { ID int64 `xorm:"pk autoincr 'config_id'"` RepoID int64 `xorm:"UNIQUE(s) 'config_repo_id'"` Hash string `xorm:"UNIQUE(s) 'config_hash'"` @@ -32,11 +32,11 @@ type configV031 struct { Data []byte `xorm:"LONGBLOB 'config_data'"` } -func (configV031) TableName() string { +func (configV009) TableName() string { return "config" } -type cronV031 struct { +type cronV009 struct { ID int64 `xorm:"pk autoincr 'i_d'"` Name string `xorm:"name UNIQUE(s) INDEX"` RepoID int64 `xorm:"repo_id UNIQUE(s) INDEX"` @@ -47,11 +47,11 @@ type cronV031 struct { Branch string `xorm:"branch"` } -func (cronV031) TableName() string { +func (cronV009) TableName() string { return "crons" } -type permV031 struct { +type permV009 struct { UserID int64 `xorm:"UNIQUE(s) INDEX NOT NULL 'perm_user_id'"` RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL 'perm_repo_id'"` Pull bool `xorm:"perm_pull"` @@ -60,11 +60,11 @@ type permV031 struct { Synced int64 `xorm:"perm_synced"` } -func (permV031) TableName() string { +func (permV009) TableName() string { return "perms" } -type pipelineV031 struct { +type pipelineV009 struct { ID int64 `xorm:"pk autoincr 'pipeline_id'"` RepoID int64 `xorm:"UNIQUE(s) INDEX 'pipeline_repo_id'"` Number int64 `xorm:"UNIQUE(s) 'pipeline_number'"` @@ -93,19 +93,19 @@ type pipelineV031 struct { Reviewed int64 `xorm:"pipeline_reviewed"` } -func (pipelineV031) TableName() string { +func (pipelineV009) TableName() string { return "pipelines" } -type redirectionV031 struct { +type redirectionV009 struct { ID int64 `xorm:"pk autoincr 'redirection_id'"` } -func (r redirectionV031) TableName() string { +func (r redirectionV009) TableName() string { return "redirections" } -type registryV031 struct { +type registryV009 struct { ID int64 `xorm:"pk autoincr 'registry_id'"` RepoID int64 `xorm:"UNIQUE(s) INDEX 'registry_repo_id'"` Address string `xorm:"UNIQUE(s) INDEX 'registry_addr'"` @@ -113,7 +113,11 @@ type registryV031 struct { Password string `xorm:"TEXT 'registry_password'"` } -type repoV031 struct { +func (registryV009) TableName() string { + return "registry" +} + +type repoV009 struct { ID int64 `xorm:"pk autoincr 'repo_id'"` UserID int64 `xorm:"repo_user_id"` OrgID int64 `xorm:"repo_org_id"` @@ -139,11 +143,11 @@ type repoV031 struct { Hash string `xorm:"varchar(500) 'repo_hash'"` } -func (repoV031) TableName() string { +func (repoV009) TableName() string { return "repos" } -type secretV031 struct { +type secretV009 struct { ID int64 `xorm:"pk autoincr 'secret_id'"` OrgID int64 `xorm:"NOT NULL DEFAULT 0 UNIQUE(s) INDEX 'secret_org_id'"` RepoID int64 `xorm:"NOT NULL DEFAULT 0 UNIQUE(s) INDEX 'secret_repo_id'"` @@ -153,11 +157,11 @@ type secretV031 struct { Events []model.WebhookEvent `xorm:"json 'secret_events'"` } -func (secretV031) TableName() string { +func (secretV009) TableName() string { return "secrets" } -type stepV031 struct { +type stepV009 struct { ID int64 `xorm:"pk autoincr 'step_id'"` UUID string `xorm:"INDEX 'step_uuid'"` PipelineID int64 `xorm:"UNIQUE(s) INDEX 'step_pipeline_id'"` @@ -173,11 +177,11 @@ type stepV031 struct { Type model.StepType `xorm:"step_type"` } -func (stepV031) TableName() string { +func (stepV009) TableName() string { return "steps" } -type taskV031 struct { +type taskV009 struct { ID string `xorm:"PK UNIQUE 'task_id'"` Data []byte `xorm:"LONGBLOB 'task_data'"` Labels map[string]string `xorm:"json 'task_labels'"` @@ -186,11 +190,11 @@ type taskV031 struct { DepStatus map[string]model.StatusValue `xorm:"json 'task_dep_status'"` } -func (taskV031) TableName() string { +func (taskV009) TableName() string { return "tasks" } -type userV031 struct { +type userV009 struct { ID int64 `xorm:"pk autoincr 'user_id'"` Login string `xorm:"UNIQUE 'user_login'"` Token string `xorm:"TEXT 'user_token'"` @@ -203,11 +207,11 @@ type userV031 struct { OrgID int64 `xorm:"user_org_id"` } -func (userV031) TableName() string { +func (userV009) TableName() string { return "users" } -type workflowV031 struct { +type workflowV009 struct { ID int64 `xorm:"pk autoincr 'workflow_id'"` PipelineID int64 `xorm:"UNIQUE(s) INDEX 'workflow_pipeline_id'"` PID int `xorm:"UNIQUE(s) 'workflow_pid'"` @@ -222,23 +226,23 @@ type workflowV031 struct { AxisID int `xorm:"workflow_axis_id"` } -func (workflowV031) TableName() string { +func (workflowV009) TableName() string { return "workflows" } -type serverConfigV031 struct { +type serverConfigV009 struct { Key string `xorm:"pk 'key'"` Value string `xorm:"value"` } -func (serverConfigV031) TableName() string { +func (serverConfigV009) TableName() string { return "server_config" } var unifyColumnsTables = xormigrate.Migration{ ID: "unify-columns-tables", MigrateSession: func(sess *xorm.Session) (err error) { - if err := sess.Sync(new(configV031), new(cronV031), new(permV031), new(pipelineV031), new(redirectionV031), new(registryV031), new(repoV031), new(secretV031), new(stepV031), new(taskV031), new(userV031), new(workflowV031), new(serverConfigV031)); err != nil { + if err := sess.Sync(new(configV009), new(cronV009), new(permV009), new(pipelineV009), new(redirectionV009), new(registryV009), new(repoV009), new(secretV009), new(stepV009), new(taskV009), new(userV009), new(workflowV009), new(serverConfigV009)); err != nil { return fmt.Errorf("sync models failed: %w", err) } diff --git a/server/store/datastore/migration/032_registries_add_user.go b/server/store/datastore/migration/010_registries_add_user.go similarity index 100% rename from server/store/datastore/migration/032_registries_add_user.go rename to server/store/datastore/migration/010_registries_add_user.go diff --git a/server/store/datastore/migration/010_rename_builds_to_pipeline.go b/server/store/datastore/migration/010_rename_builds_to_pipeline.go deleted file mode 100644 index 9fd042425..000000000 --- a/server/store/datastore/migration/010_rename_builds_to_pipeline.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var renameBuildsToPipeline = xormigrate.Migration{ - ID: "rename-builds-to-pipeline", - MigrateSession: func(sess *xorm.Session) error { - err := renameTable(sess, "builds", "pipelines") - if err != nil { - return err - } - err = renameTable(sess, "build_config", "pipeline_config") - if err != nil { - return err - } - return nil - }, -} diff --git a/server/store/datastore/migration/011_columns_rename_builds_to_pipeline.go b/server/store/datastore/migration/011_columns_rename_builds_to_pipeline.go deleted file mode 100644 index b80a465e5..000000000 --- a/server/store/datastore/migration/011_columns_rename_builds_to_pipeline.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "strings" - - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -type oldTable struct { - table string - columns []string -} - -var renameColumnsBuildsToPipeline = xormigrate.Migration{ - ID: "rename-columns-builds-to-pipeline", - MigrateSession: func(sess *xorm.Session) error { - var oldColumns []*oldTable - - oldColumns = append(oldColumns, &oldTable{ - table: "pipelines", - columns: []string{ - "build_id", - "build_repo_id", - "build_number", - "build_author", - "build_config_id", - "build_parent", - "build_event", - "build_status", - "build_error", - "build_enqueued", - "build_created", - "build_started", - "build_finished", - "build_deploy", - "build_commit", - "build_branch", - "build_ref", - "build_refspec", - "build_remote", - "build_title", - "build_message", - "build_timestamp", - "build_sender", - "build_avatar", - "build_email", - "build_link", - "build_signed", - "build_verified", - "build_reviewer", - "build_reviewed", - }, - }, - ) - - oldColumns = append(oldColumns, &oldTable{ - table: "pipeline_config", - columns: []string{"build_id"}, - }) - - oldColumns = append(oldColumns, &oldTable{ - table: "files", - columns: []string{"file_build_id"}, - }) - - oldColumns = append(oldColumns, &oldTable{ - table: "procs", - columns: []string{"proc_build_id"}, - }) - - for _, table := range oldColumns { - for _, column := range table.columns { - err := renameColumn(sess, table.table, column, strings.Replace(column, "build_", "pipeline_", 1)) - if err != nil { - return err - } - } - } - - return nil - }, -} diff --git a/server/store/datastore/migration/033_cron_without_sec.go b/server/store/datastore/migration/011_cron_without_sec.go similarity index 100% rename from server/store/datastore/migration/033_cron_without_sec.go rename to server/store/datastore/migration/011_cron_without_sec.go diff --git a/server/store/datastore/migration/012_columns_rename_procs_to_steps.go b/server/store/datastore/migration/012_columns_rename_procs_to_steps.go deleted file mode 100644 index 015bdac4f..000000000 --- a/server/store/datastore/migration/012_columns_rename_procs_to_steps.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "strings" - - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var renameTableProcsToSteps = xormigrate.Migration{ - ID: "rename-procs-to-steps", - MigrateSession: func(sess *xorm.Session) error { - err := renameTable(sess, "procs", "steps") - if err != nil { - return err - } - - oldProcColumns := []*oldTable{ - { - table: "steps", - columns: []string{ - "proc_id", - "proc_pipeline_id", - "proc_pid", - "proc_ppid", - "proc_pgid", - "proc_name", - "proc_state", - "proc_error", - "proc_exit_code", - "proc_started", - "proc_stopped", - "proc_machine", - "proc_platform", - "proc_environ", - }, - }, - { - table: "files", - columns: []string{"file_proc_id"}, - }, - } - - for _, table := range oldProcColumns { - for _, column := range table.columns { - err := renameColumn(sess, table.table, column, strings.Replace(column, "proc_", "step_", 1)) - if err != nil { - return err - } - } - } - - oldJobColumns := []*oldTable{ - { - table: "logs", - columns: []string{ - "log_job_id", - }, - }, - } - - for _, table := range oldJobColumns { - for _, column := range table.columns { - err := renameColumn(sess, table.table, column, strings.Replace(column, "job_", "step_", 1)) - if err != nil { - return err - } - } - } - - return nil - }, -} diff --git a/server/store/datastore/migration/034_rename_start_end_time.go b/server/store/datastore/migration/012_rename_start_end_time.go similarity index 86% rename from server/store/datastore/migration/034_rename_start_end_time.go rename to server/store/datastore/migration/012_rename_start_end_time.go index 2bc968f6b..e710c7f0a 100644 --- a/server/store/datastore/migration/034_rename_start_end_time.go +++ b/server/store/datastore/migration/012_rename_start_end_time.go @@ -21,26 +21,26 @@ import ( "xorm.io/xorm" ) -type stepV033 struct { +type stepV012 struct { Finished int64 `xorm:"stopped"` } -func (stepV033) TableName() string { +func (stepV012) TableName() string { return "steps" } -type workflowV033 struct { +type workflowV012 struct { Finished int64 `xorm:"stopped"` } -func (workflowV033) TableName() string { +func (workflowV012) TableName() string { return "workflows" } var renameStartEndTime = xormigrate.Migration{ ID: "rename-start-end-time", MigrateSession: func(sess *xorm.Session) (err error) { - if err := sess.Sync(new(stepV033), new(workflowV033)); err != nil { + if err := sess.Sync(new(stepV012), new(workflowV012)); err != nil { return fmt.Errorf("sync models failed: %w", err) } diff --git a/server/store/datastore/migration/009_lowercase_secret_names.go b/server/store/datastore/migration/013_fix_v31_registries.go similarity index 67% rename from server/store/datastore/migration/009_lowercase_secret_names.go rename to server/store/datastore/migration/013_fix_v31_registries.go index 5402dc36c..988e49541 100644 --- a/server/store/datastore/migration/009_lowercase_secret_names.go +++ b/server/store/datastore/migration/013_fix_v31_registries.go @@ -1,10 +1,10 @@ -// Copyright 2022 Woodpecker Authors +// Copyright 2024 Woodpecker Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -19,10 +19,17 @@ import ( "xorm.io/xorm" ) -var lowercaseSecretNames = xormigrate.Migration{ - ID: "lowercase-secret-names", +var fixV31Registries = xormigrate.Migration{ + ID: "fix-v31-registries", MigrateSession: func(sess *xorm.Session) (err error) { - _, err = sess.Exec("UPDATE secrets SET secret_name = LOWER(secret_name);") - return err + has, err := sess.IsTableExist("registry_v031") + if err != nil { + return err + } + if has { + return sess.DropTable("registry_v031") + } + + return nil }, } diff --git a/server/store/datastore/migration/013_rename_remote_to_forge.go b/server/store/datastore/migration/013_rename_remote_to_forge.go deleted file mode 100644 index 78eff328b..000000000 --- a/server/store/datastore/migration/013_rename_remote_to_forge.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -type oldRepo013 struct { - ID int64 `xorm:"pk autoincr 'repo_id'"` - RemoteID string `xorm:"remote_id"` -} - -func (oldRepo013) TableName() string { - return "repos" -} - -var renameRemoteToForge = xormigrate.Migration{ - ID: "rename-remote-to-forge", - MigrateSession: func(sess *xorm.Session) error { - if err := renameColumn(sess, "pipelines", "pipeline_remote", "pipeline_clone_url"); err != nil { - return err - } - - // make sure the column exist before rename it - if err := sess.Sync(new(oldRepo013)); err != nil { - return err - } - - return renameColumn(sess, "repos", "remote_id", "forge_id") - }, -} diff --git a/server/store/datastore/migration/014_remove_old_migrations_of_v1.go b/server/store/datastore/migration/014_remove_old_migrations_of_v1.go new file mode 100644 index 000000000..5f49d559b --- /dev/null +++ b/server/store/datastore/migration/014_remove_old_migrations_of_v1.go @@ -0,0 +1,54 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migration + +import ( + "src.techknowlogick.com/xormigrate" + "xorm.io/xorm" +) + +var removeOldMigrationsOfV1 = xormigrate.Migration{ + ID: "remove-old-migrations-of-v1", + MigrateSession: func(sess *xorm.Session) (err error) { + _, err = sess.Table(&xormigrate.Migration{}).In("id", []string{ + "xorm", + "alter-table-drop-repo-fallback", + "drop-allow-push-tags-deploys-columns", + "fix-pr-secret-event-name", + "alter-table-drop-counter", + "drop-senders", + "alter-table-logs-update-type-of-data", + "alter-table-add-secrets-user-id", + "lowercase-secret-names", + "recreate-agents-table", + "rename-builds-to-pipeline", + "rename-columns-builds-to-pipeline", + "rename-procs-to-steps", + "rename-remote-to-forge", + "rename-forge-id-to-forge-remote-id", + "remove-active-from-users", + "remove-inactive-repos", + "drop-files", + "remove-machine-col", + "drop-old-col", + "init-log_entries", + "migrate-logs-to-log_entries", + "parent-steps-to-workflows", + "add-orgs", + }).Delete() + + return err + }, +} diff --git a/server/store/datastore/migration/014_rename_forge_id_to_forge_remote_id.go b/server/store/datastore/migration/014_rename_forge_id_to_forge_remote_id.go deleted file mode 100644 index 60a58660d..000000000 --- a/server/store/datastore/migration/014_rename_forge_id_to_forge_remote_id.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var renameForgeIDToForgeRemoteID = xormigrate.Migration{ - ID: "rename-forge-id-to-forge-remote-id", - MigrateSession: func(sess *xorm.Session) error { - return renameColumn(sess, "repos", "forge_id", "forge_remote_id") - }, -} diff --git a/server/store/datastore/migration/015_remove_active_from_users.go b/server/store/datastore/migration/015_remove_active_from_users.go deleted file mode 100644 index d0eb785d1..000000000 --- a/server/store/datastore/migration/015_remove_active_from_users.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var removeActiveFromUsers = xormigrate.Migration{ - ID: "remove-active-from-users", - MigrateSession: func(sess *xorm.Session) error { - return dropTableColumns(sess, "users", "user_active") - }, -} diff --git a/server/store/datastore/migration/016_remove_inactive_repos.go b/server/store/datastore/migration/016_remove_inactive_repos.go deleted file mode 100644 index 48742f89b..000000000 --- a/server/store/datastore/migration/016_remove_inactive_repos.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var removeInactiveRepos = xormigrate.Migration{ - ID: "remove-inactive-repos", - MigrateSession: func(sess *xorm.Session) error { - // If the timeout is 0, the repo was never activated, so we remove it. - _, err := sess.Table("repos").Where("repo_active = ?", false).And("repo_timeout = ?", 0).Delete() - if err != nil { - return err - } - - return dropTableColumns(sess, "users", "user_synced") - }, -} diff --git a/server/store/datastore/migration/017_remove_files_table.go b/server/store/datastore/migration/017_remove_files_table.go deleted file mode 100644 index 8cc99b806..000000000 --- a/server/store/datastore/migration/017_remove_files_table.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -var dropFiles = xormigrate.Migration{ - ID: "drop-files", - MigrateSession: func(sess *xorm.Session) error { - return sess.DropTable("files") - }, -} diff --git a/server/store/datastore/migration/018_remove_machine_col.go b/server/store/datastore/migration/018_remove_machine_col.go deleted file mode 100644 index 791104479..000000000 --- a/server/store/datastore/migration/018_remove_machine_col.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2023 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -type oldStep018 struct { - ID int64 `xorm:"pk autoincr 'step_id'"` - Machine string `xorm:"step_machine"` -} - -func (oldStep018) TableName() string { - return "steps" -} - -var removeMachineCol = xormigrate.Migration{ - ID: "remove-machine-col", - MigrateSession: func(sess *xorm.Session) error { - // make sure step_machine column exists - if err := sess.Sync(new(oldStep018)); err != nil { - return err - } - return dropTableColumns(sess, "steps", "step_machine") - }, -} diff --git a/server/store/datastore/migration/019_drop_old_cols.go b/server/store/datastore/migration/019_drop_old_cols.go deleted file mode 100644 index f380c924e..000000000 --- a/server/store/datastore/migration/019_drop_old_cols.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2023 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" -) - -type oldPipeline019 struct { - ID int64 `xorm:"pk autoincr 'pipeline_id'"` - Signed bool `xorm:"pipeline_signed"` - Verified bool `xorm:"pipeline_verified"` -} - -func (oldPipeline019) TableName() string { - return "pipelines" -} - -var dropOldCols = xormigrate.Migration{ - ID: "drop-old-col", - MigrateSession: func(sess *xorm.Session) error { - // make sure columns on pipelines exist - if err := sess.Sync(new(oldPipeline019)); err != nil { - return err - } - if err := dropTableColumns(sess, "steps", "step_pgid"); err != nil { - return err - } - - return dropTableColumns(sess, "pipelines", "pipeline_signed", "pipeline_verified") - }, -} diff --git a/server/store/datastore/migration/020_alter_logs_table.go b/server/store/datastore/migration/020_alter_logs_table.go deleted file mode 100644 index 4f13434cb..000000000 --- a/server/store/datastore/migration/020_alter_logs_table.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2023 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "context" - "encoding/json" - "fmt" - "runtime" - - "github.com/rs/zerolog/log" - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" - - "go.woodpecker-ci.org/woodpecker/v2/shared/utils" -) - -// perPage020 sets the size of the slice to read per page. -var perPage020 = 100 - -type oldLogs020 struct { - ID int64 `xorm:"pk autoincr 'log_id'"` - StepID int64 `xorm:"UNIQUE 'log_step_id'"` - Data []byte `xorm:"LONGBLOB 'log_data'"` -} - -func (oldLogs020) TableName() string { - return "logs" -} - -type oldLogEntry020 struct { - Step string `json:"step,omitempty"` - Time int64 `json:"time,omitempty"` - Type int `json:"type,omitempty"` - Pos int `json:"pos,omitempty"` - Out string `json:"out,omitempty"` -} - -type newLogEntry020 struct { - ID int64 `xorm:"pk autoincr 'id'"` - StepID int64 `xorm:"'step_id'"` - Time int64 - Line int - Data []byte `xorm:"LONGBLOB"` - Created int64 `xorm:"created"` - Type int -} - -func (newLogEntry020) TableName() string { - return "log_entries" -} - -var initLogsEntriesTable = xormigrate.Migration{ - ID: "init-log_entries", - MigrateSession: func(sess *xorm.Session) error { - return sess.Sync(new(newLogEntry020)) - }, -} - -var migrateLogs2LogEntries = xormigrate.Migration{ - ID: "migrate-logs-to-log_entries", - Long: true, - Migrate: func(e *xorm.Engine) error { - // make sure old logs table exists - if exist, err := e.IsTableExist(new(oldLogs020)); !exist || err != nil { - return err - } - - if err := e.Sync(new(oldLogs020)); err != nil { - return err - } - - hasJSONErrors := false - - page := 0 - offset := 0 - logs := make([]*oldLogs020, 0, perPage020) - logEntries := make([]*oldLogEntry020, 0, 50) - - ctx, cancelCtx := context.WithCancelCause(context.Background()) - defer cancelCtx(nil) - sigtermCtx := utils.WithContextSigtermCallback(ctx, func() { - log.Info().Msg("ctrl+c received, stopping current migration") - }) - - for { - if sigtermCtx.Err() != nil { - return fmt.Errorf("migration 'migrate-logs-to-log_entries' gracefully aborted") - } - - sess := e.NewSession().NoCache() - defer sess.Close() - if err := sess.Begin(); err != nil { - return err - } - logs = logs[:0] - - err := sess.Limit(perPage020, offset).Find(&logs) - if err != nil { - return err - } - - log.Trace().Msgf("migrate-logs-to-log_entries: process page %d", page) - - for _, l := range logs { - logEntries = logEntries[:0] - if err := json.Unmarshal(l.Data, &logEntries); err != nil { - hasJSONErrors = true - offset++ - continue - } - - time := int64(0) - for _, logEntry := range logEntries { - - if logEntry.Time > time { - time = logEntry.Time - } - - log := &newLogEntry020{ - StepID: l.StepID, - Data: []byte(logEntry.Out), - Line: logEntry.Pos, - Time: time, - Type: logEntry.Type, - } - - if _, err := sess.Insert(log); err != nil { - return err - } - } - - if _, err := sess.Delete(l); err != nil { - return err - } - } - - if err := sess.Commit(); err != nil { - return err - } - - if len(logs) < perPage020 { - break - } - - runtime.GC() - page++ - } - - if hasJSONErrors { - return fmt.Errorf("skipped some logs as json could not be deserialized for them") - } - - return e.DropTables("logs") - }, -} diff --git a/server/store/datastore/migration/021_parent_steps_to_workflows.go b/server/store/datastore/migration/021_parent_steps_to_workflows.go deleted file mode 100644 index a97efb1d2..000000000 --- a/server/store/datastore/migration/021_parent_steps_to_workflows.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "src.techknowlogick.com/xormigrate" - "xorm.io/xorm" - - "go.woodpecker-ci.org/woodpecker/v2/server/model" -) - -type oldStep021 struct { - ID int64 `xorm:"pk autoincr 'step_id'"` - PipelineID int64 `xorm:"UNIQUE(s) INDEX 'step_pipeline_id'"` - PID int `xorm:"UNIQUE(s) 'step_pid'"` - PPID int `xorm:"step_ppid"` - Name string `xorm:"step_name"` - State model.StatusValue `xorm:"step_state"` - Error string `xorm:"TEXT 'step_error'"` - Started int64 `xorm:"step_started"` - Stopped int64 `xorm:"step_stopped"` - AgentID int64 `xorm:"step_agent_id"` - Platform string `xorm:"step_platform"` - Environ map[string]string `xorm:"json 'step_environ'"` -} - -func (oldStep021) TableName() string { - return "steps" -} - -var parentStepsToWorkflows = xormigrate.Migration{ - ID: "parent-steps-to-workflows", - MigrateSession: func(sess *xorm.Session) error { - if err := sess.Sync(new(workflowV031)); err != nil { - return err - } - // make sure the columns exist before removing them - if err := sess.Sync(new(oldStep021)); err != nil { - return err - } - - var parentSteps []*oldStep021 - err := sess.Where("step_ppid = ?", 0).Find(&parentSteps) - if err != nil { - return err - } - - for _, p := range parentSteps { - asWorkflow := &workflowV031{ - PipelineID: p.PipelineID, - PID: p.PID, - Name: p.Name, - State: p.State, - Error: p.Error, - Started: p.Started, - Stopped: p.Stopped, - AgentID: p.AgentID, - Platform: p.Platform, - Environ: p.Environ, - } - - _, err = sess.Insert(asWorkflow) - if err != nil { - return err - } - - _, err = sess.Delete(&oldStep021{ID: p.ID}) - if err != nil { - return err - } - } - - return dropTableColumns(sess, "steps", "step_agent_id", "step_platform", "step_environ") - }, -} diff --git a/server/store/datastore/migration/022_add_orgs.go b/server/store/datastore/migration/022_add_orgs.go deleted file mode 100644 index 9d93e0537..000000000 --- a/server/store/datastore/migration/022_add_orgs.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migration - -import ( - "fmt" - "strings" - - "src.techknowlogick.com/xormigrate" - "xorm.io/builder" - "xorm.io/xorm" - - "go.woodpecker-ci.org/woodpecker/v2/server/model" -) - -type oldSecret022 struct { - ID int64 `xorm:"pk autoincr 'secret_id'"` - Owner string `xorm:"'secret_owner'"` - OrgID int64 `xorm:"NOT NULL DEFAULT 0 'secret_org_id'"` - RepoID int64 `xorm:"NOT NULL DEFAULT 0 'secret_repo_id'"` - Name string `xorm:"NOT NULL INDEX 'secret_name'"` -} - -func (oldSecret022) TableName() string { - return "secrets" -} - -type syncRepo022 struct { - OrgID int64 `json:"org_id" xorm:"repo_org_id"` -} - -// TableName return database table name for xorm. -func (syncRepo022) TableName() string { - return "repos" -} - -type repo022 struct { - ID int64 `json:"id,omitempty" xorm:"pk autoincr 'repo_id'"` - OrgID int64 `json:"org_id" xorm:"repo_org_id"` - Owner string `json:"owner" xorm:"UNIQUE(name) 'repo_owner'"` -} - -// TableName return database table name for xorm. -func (repo022) TableName() string { - return "repos" -} - -var addOrgs = xormigrate.Migration{ - ID: "add-orgs", - MigrateSession: func(sess *xorm.Session) error { - if exist, err := sess.IsTableExist("orgs"); exist && err == nil { - if err := sess.DropTable("orgs"); err != nil { - return fmt.Errorf("drop old orgs table failed: %w", err) - } - } - - if err := sess.Sync(new(model.Org), new(syncRepo022), new(userV031)); err != nil { - return fmt.Errorf("sync new models failed: %w", err) - } - - // make sure the columns exist before removing them - if _, err := sess.SyncWithOptions(xorm.SyncOptions{IgnoreConstrains: true, IgnoreIndices: true}, new(oldSecret022)); err != nil { - return fmt.Errorf("sync old secrets models failed: %w", err) - } - - // get all org names from repos - var repos []*repo022 - if err := sess.Find(&repos); err != nil { - return fmt.Errorf("find all repos failed: %w", err) - } - - orgs := make(map[string]*model.Org) - users := make(map[string]bool) - for _, repo := range repos { - orgName := strings.ToLower(repo.Owner) - - // check if it's a registered user - if _, ok := users[orgName]; !ok { - exist, err := sess.Where("user_login = ?", orgName).Exist(new(userV031)) - if err != nil { - return fmt.Errorf("check if user '%s' exist failed: %w", orgName, err) - } - users[orgName] = exist - } - - // create org if not already created - if _, ok := orgs[orgName]; !ok { - org := &model.Org{ - Name: orgName, - IsUser: users[orgName], - } - if _, err := sess.Insert(org); err != nil { - return fmt.Errorf("insert org %#v failed: %w", org, err) - } - orgs[orgName] = org - - // update org secrets - var secrets []*oldSecret022 - if err := sess.Where(builder.Eq{"secret_owner": orgName, "secret_repo_id": 0}).Find(&secrets); err != nil { - return fmt.Errorf("get org secrets failed: %w", err) - } - - for _, secret := range secrets { - secret.OrgID = org.ID - if _, err := sess.ID(secret.ID).Cols("secret_org_id").Update(secret); err != nil { - return fmt.Errorf("update org secret %d failed: %w", secret.ID, err) - } - } - } - - // update the repo - repo.OrgID = orgs[orgName].ID - if _, err := sess.ID(repo.ID).Cols("repo_org_id").Update(repo); err != nil { - return fmt.Errorf("update repos failed: %w", err) - } - } - - return dropTableColumns(sess, "secrets", "secret_owner") - }, -} diff --git a/server/store/datastore/migration/common.go b/server/store/datastore/migration/common.go index d705f99be..0ba2ff0d5 100644 --- a/server/store/datastore/migration/common.go +++ b/server/store/datastore/migration/common.go @@ -214,7 +214,6 @@ func alterColumnDefault(sess *xorm.Session, table, column, defValue string) erro } } -//nolint:unparam func alterColumnNull(sess *xorm.Session, table, column string, null bool) error { val := "NULL" if !null { diff --git a/server/store/datastore/migration/migration.go b/server/store/datastore/migration/migration.go index da9b1f732..d82a3a336 100644 --- a/server/store/datastore/migration/migration.go +++ b/server/store/datastore/migration/migration.go @@ -29,30 +29,6 @@ import ( // They are executed in order and if one fails Xormigrate will try to rollback that specific one and quits. var migrationTasks = []*xormigrate.Migration{ &legacyToXormigrate, - &legacy2Xorm, - &alterTableReposDropFallback, - &alterTableReposDropAllowDeploysAllowTags, - &fixPRSecretEventName, - &alterTableReposDropCounter, - &dropSenders, - &alterTableLogUpdateColumnLogDataType, - &alterTableSecretsAddUserCol, - &recreateAgentsTable, - &lowercaseSecretNames, - &renameBuildsToPipeline, - &renameColumnsBuildsToPipeline, - &renameTableProcsToSteps, - &renameRemoteToForge, - &renameForgeIDToForgeRemoteID, - &removeActiveFromUsers, - &removeInactiveRepos, - &dropFiles, - &removeMachineCol, - &dropOldCols, - &initLogsEntriesTable, - &migrateLogs2LogEntries, - &parentStepsToWorkflows, - &addOrgs, &addOrgID, &alterTableTasksUpdateColumnTaskDataType, &alterTableConfigUpdateColumnConfigDataType, @@ -65,6 +41,8 @@ var migrationTasks = []*xormigrate.Migration{ &alterTableRegistriesFixRequiredFields, &cronWithoutSec, &renameStartEndTime, + &fixV31Registries, + &removeOldMigrationsOfV1, } var allBeans = []any{ diff --git a/server/store/datastore/migration/test-files/sqlite.db b/server/store/datastore/migration/test-files/sqlite.db index 219e5184a..731dfe5cd 100644 Binary files a/server/store/datastore/migration/test-files/sqlite.db and b/server/store/datastore/migration/test-files/sqlite.db differ diff --git a/server/store/datastore/server_config.go b/server/store/datastore/server_config.go index 2dd52f406..a18517edb 100644 --- a/server/store/datastore/server_config.go +++ b/server/store/datastore/server_config.go @@ -31,7 +31,13 @@ func (s storage) ServerConfigSet(key, value string) error { Key: key, } - count, err := s.engine.Count(config) + sess := s.engine.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + count, err := sess.Count(config) if err != nil { return err } @@ -39,12 +45,15 @@ func (s storage) ServerConfigSet(key, value string) error { config.Value = value if count == 0 { - _, err := s.engine.Insert(config) + _, err = sess.Insert(config) + } else { + _, err = sess.Where("`key` = ?", config.Key).Cols("value").Update(config) + } + if err != nil { return err } - _, err = s.engine.Where("`key` = ?", config.Key).Cols("value").Update(config) - return err + return sess.Commit() } func (s storage) ServerConfigDelete(key string) error { diff --git a/server/store/mocks/store.go b/server/store/mocks/store.go index 55d7257aa..df88e436d 100644 --- a/server/store/mocks/store.go +++ b/server/store/mocks/store.go @@ -1340,17 +1340,17 @@ func (_m *Store) HasRedirectionForRepo(_a0 int64, _a1 string) (bool, error) { return r0, r1 } -// LogAppend provides a mock function with given fields: logEntry -func (_m *Store) LogAppend(logEntry *model.LogEntry) error { - ret := _m.Called(logEntry) +// LogAppend provides a mock function with given fields: _a0, _a1 +func (_m *Store) LogAppend(_a0 *model.Step, _a1 []*model.LogEntry) error { + ret := _m.Called(_a0, _a1) if len(ret) == 0 { panic("no return value specified for LogAppend") } var r0 error - if rf, ok := ret.Get(0).(func(*model.LogEntry) error); ok { - r0 = rf(logEntry) + if rf, ok := ret.Get(0).(func(*model.Step, []*model.LogEntry) error); ok { + r0 = rf(_a0, _a1) } else { r0 = ret.Error(0) } diff --git a/server/store/store.go b/server/store/store.go index ff408f075..f1e958994 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -143,7 +143,7 @@ type Store interface { // Logs LogFind(*model.Step) ([]*model.LogEntry, error) - LogAppend(logEntry *model.LogEntry) error + LogAppend(*model.Step, []*model.LogEntry) error LogDelete(*model.Step) error // Tasks diff --git a/shared/constant/constant.go b/shared/constant/constant.go index 1d060bfeb..e6cfa63ed 100644 --- a/shared/constant/constant.go +++ b/shared/constant/constant.go @@ -14,6 +14,8 @@ package constant +import "time" + // DefaultConfigOrder represent the priority in witch woodpecker search for a pipeline config by default // folders are indicated by supplying a trailing slash. var DefaultConfigOrder = [...]string{ @@ -25,7 +27,7 @@ var DefaultConfigOrder = [...]string{ const ( // DefaultClonePlugin can be changed by 'WOODPECKER_DEFAULT_CLONE_PLUGIN' at runtime. // renovate: datasource=docker depName=woodpeckerci/plugin-git - DefaultClonePlugin = "docker.io/woodpeckerci/plugin-git:2.5.2" + DefaultClonePlugin = "docker.io/woodpeckerci/plugin-git:2.6.0" ) // TrustedClonePlugins can be changed by 'WOODPECKER_PLUGINS_TRUSTED_CLONE' at runtime. @@ -34,3 +36,6 @@ var TrustedClonePlugins = []string{ "docker.io/woodpeckerci/plugin-git", "quay.io/woodpeckerci/plugin-git", } + +// TaskTimeout is the time till a running task is counted as dead. +var TaskTimeout = time.Minute diff --git a/web/src/lib/api/types/queue.ts b/web/src/lib/api/types/queue.ts index e8752c71d..a6ff4edc7 100644 --- a/web/src/lib/api/types/queue.ts +++ b/web/src/lib/api/types/queue.ts @@ -1,6 +1,5 @@ export interface Task { id: number; - data: string; labels: { [key: string]: string }; dependencies: string[]; dep_status: { [key: string]: string }; diff --git a/web/src/views/RepoAdd.vue b/web/src/views/RepoAdd.vue index 1730aef07..5a217f6e9 100644 --- a/web/src/views/RepoAdd.vue +++ b/web/src/views/RepoAdd.vue @@ -38,6 +38,7 @@ import { useRouter } from 'vue-router'; import Badge from '~/components/atomic/Badge.vue'; import Button from '~/components/atomic/Button.vue'; +import Icon from '~/components/atomic/Icon.vue'; import ListItem from '~/components/atomic/ListItem.vue'; import Scaffold from '~/components/layout/scaffold/Scaffold.vue'; import useApiClient from '~/compositions/useApiClient'; diff --git a/web/src/views/repo/pipeline/PipelineWrapper.vue b/web/src/views/repo/pipeline/PipelineWrapper.vue index dff059e24..a0b898c4c 100644 --- a/web/src/views/repo/pipeline/PipelineWrapper.vue +++ b/web/src/views/repo/pipeline/PipelineWrapper.vue @@ -31,7 +31,7 @@ }} -