From 3786077c07b67b69df1890ef65c5b4fb398b4d10 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 26 Jan 2026 21:25:38 +0100 Subject: [PATCH] Update Architecture: move `pipeline/rpc` => `rpc` & `server/{grpc => rpc}` (#6012) --- .cspell.json | 31 ++++---- .golangci.yaml | 47 ++++++++---- .mockery.yaml | 2 +- Makefile | 2 +- {pipeline => agent}/log/line_writer.go | 2 +- {pipeline => agent}/log/line_writer_test.go | 6 +- agent/logger.go | 7 +- agent/rpc/auth_client_grpc.go | 2 +- agent/rpc/client_grpc.go | 4 +- agent/runner.go | 2 +- agent/tracer.go | 2 +- cli/exec/exec.go | 4 +- cmd/agent/core/agent.go | 2 +- cmd/server/grpc_server.go | 4 +- docs/docs/92-development/05-architecture.md | 74 +++++++++++++------ .../utils.go => utils/copy_line_by_line.go} | 2 +- .../copy_line_by_line_test.go} | 16 ++-- {pipeline/rpc => rpc}/log_entry.go | 0 {pipeline/rpc => rpc}/log_entry_test.go | 0 {pipeline/rpc => rpc}/mocks/mock_Peer.go | 2 +- {pipeline/rpc => rpc}/peer.go | 0 {pipeline/rpc => rpc}/proto/generate.go | 0 {pipeline/rpc => rpc}/proto/version.go | 0 {pipeline/rpc => rpc}/proto/woodpecker.pb.go | 2 +- {pipeline/rpc => rpc}/proto/woodpecker.proto | 2 +- .../rpc => rpc}/proto/woodpecker_grpc.pb.go | 0 server/pipeline/queue.go | 2 +- server/pipeline/step_status.go | 2 +- server/pipeline/step_status_test.go | 2 +- server/pipeline/workflow_status.go | 2 +- server/{grpc => rpc}/auth_server.go | 2 +- server/{grpc => rpc}/authorizer.go | 0 server/{grpc => rpc}/filter.go | 2 +- server/{grpc => rpc}/filter_test.go | 2 +- server/{grpc => rpc}/jwt_manager.go | 0 server/{grpc => rpc}/rpc.go | 2 +- server/{grpc => rpc}/rpc_test.go | 2 +- server/{grpc => rpc}/server.go | 4 +- 38 files changed, 145 insertions(+), 92 deletions(-) rename {pipeline => agent}/log/line_writer.go (97%) rename {pipeline => agent}/log/line_writer_test.go (90%) rename pipeline/{log/utils.go => utils/copy_line_by_line.go} (99%) rename pipeline/{log/utils_test.go => utils/copy_line_by_line_test.go} (96%) rename {pipeline/rpc => rpc}/log_entry.go (100%) rename {pipeline/rpc => rpc}/log_entry_test.go (100%) rename {pipeline/rpc => rpc}/mocks/mock_Peer.go (99%) rename {pipeline/rpc => rpc}/peer.go (100%) rename {pipeline/rpc => rpc}/proto/generate.go (100%) rename {pipeline/rpc => rpc}/proto/version.go (100%) rename {pipeline/rpc => rpc}/proto/woodpecker.pb.go (99%) rename {pipeline/rpc => rpc}/proto/woodpecker.proto (97%) rename {pipeline/rpc => rpc}/proto/woodpecker_grpc.pb.go (100%) rename server/{grpc => rpc}/auth_server.go (97%) rename server/{grpc => rpc}/authorizer.go (100%) rename server/{grpc => rpc}/filter.go (97%) rename server/{grpc => rpc}/filter_test.go (98%) rename server/{grpc => rpc}/jwt_manager.go (100%) rename server/{grpc => rpc}/rpc.go (99%) rename server/{grpc => rpc}/rpc_test.go (98%) rename server/{grpc => rpc}/server.go (98%) diff --git a/.cspell.json b/.cspell.json index c729a0ce0..5ceafba75 100644 --- a/.cspell.json +++ b/.cspell.json @@ -28,9 +28,9 @@ "Bluesky", "Boguslawski", "bradrydzewski", + "buildkit", "BUILDPLATFORM", "buildx", - "buildkit", "caddyfile", "ccmenu", "CERTDIR", @@ -120,10 +120,10 @@ "mapstructure", "markdownlint", "mdbook", - "Milestoned", "memswap", "Metas", "mhmxs", + "Milestoned", "moby", "Msgf", "mstruebing", @@ -209,8 +209,8 @@ "unsanitize", "Upsert", "urfave", - "useragent", "usecase", + "useragent", "varchar", "varz", "Vieter", @@ -239,32 +239,33 @@ "zerologger" ], "ignorePaths": [ - "*.excalidraw", - "*.svg", - "*_test.go", ".cspell.json", ".git/**/*", ".gitignore", ".golangci.yaml", ".vscode/extensions.json", + "*_test.go", + "*.excalidraw", + "*.svg", + "**/*.pb.go", + "**/fixtures/**", + "**/testdata/**", "CHANGELOG.md", - "Makefile", + "docs/versioned_docs/", "flake.nix", "go.mod", - "**/*.pb.go", + "Makefile", + "package.json", "server/store/datastore/migration/**/*", "web/components.d.ts", "web/src/assets/locales/**/*", - "**/fixtures/**", - "**/testdata/**", - "docs/versioned_docs/", - "package.json", // generated - "go.sum", - "flake.lock", - "pnpm-lock.yaml", + "**/mocks/**", "**/node_modules/**/*", "cmd/server/openapi/docs.go", + "flake.lock", + "go.sum", + "pnpm-lock.yaml", "renovate.json", // TODO: remove the following "docs/**/*.js", diff --git a/.golangci.yaml b/.golangci.yaml index be309508d..f76440bfd 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -49,10 +49,10 @@ linters: agent: list-mode: lax files: - - '**/agent/**/*.go' - '**/agent/*.go' - - '**/cmd/agent/**/*.go' + - '**/agent/**/*.go' - '**/cmd/agent/*.go' + - '**/cmd/agent/**/*.go' deny: - pkg: go.woodpecker-ci.org/woodpecker/v3/cli - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd/cli @@ -63,70 +63,89 @@ linters: cli: list-mode: lax files: - - '**/cli/**/*.go' - '**/cli/*.go' - - '**/cmd/cli/**/*.go' + - '**/cli/**/*.go' - '**/cmd/cli/*.go' + - '**/cmd/cli/**/*.go' deny: - pkg: go.woodpecker-ci.org/woodpecker/v3/agent - - pkg: go.woodpecker-ci.org/woodpecker/v3/server - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd/agent - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd/server + - pkg: go.woodpecker-ci.org/woodpecker/v3/rpc + - pkg: go.woodpecker-ci.org/woodpecker/v3/server - pkg: go.woodpecker-ci.org/woodpecker/v3/web pipeline: list-mode: lax files: - - '**/pipeline/**/*.go' - - '**/pipeline/*.go' - '!**/cli/pipeline/*.go' - '!**/cli/pipeline/**/*.go' - '!**/server/pipeline/*.go' - '!**/server/pipeline/**/*.go' + - '**/pipeline/*.go' + - '**/pipeline/**/*.go' deny: - pkg: go.woodpecker-ci.org/woodpecker/v3/agent - pkg: go.woodpecker-ci.org/woodpecker/v3/cli - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd + - pkg: go.woodpecker-ci.org/woodpecker/v3/rpc - pkg: go.woodpecker-ci.org/woodpecker/v3/server - pkg: go.woodpecker-ci.org/woodpecker/v3/web server: list-mode: lax files: - - '**/server/**/*.go' - - '**/server/*.go' - - '**/cmd/server/**/*.go' - '**/cmd/server/*.go' - - '**/web/**/*.go' + - '**/cmd/server/**/*.go' + - '**/server/*.go' + - '**/server/**/*.go' - '**/web/*.go' + - '**/web/**/*.go' deny: - pkg: go.woodpecker-ci.org/woodpecker/v3/agent - pkg: go.woodpecker-ci.org/woodpecker/v3/cli - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd/agent - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd/cli - pkg: go.woodpecker-ci.org/woodpecker/v3/woodpecker-go/woodpecker + rpc: + list-mode: lax + files: + - '!**/agent/rpc/*.go' + - '!**/agent/rpc/**/*.go' + - '!**/server/rpc/*.go' + - '!**/server/rpc/**/*.go' + - '**/rpc/*.go' + - '**/rpc/**/*.go' + deny: + - pkg: go.woodpecker-ci.org/woodpecker/v3/agent + - pkg: go.woodpecker-ci.org/woodpecker/v3/cli + - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd + - pkg: go.woodpecker-ci.org/woodpecker/v3/server + - pkg: go.woodpecker-ci.org/woodpecker/v3/web shared: list-mode: lax files: - - '**/shared/**/*.go' - - '**/shared/*.go' - '!**/pipeline/shared/*.go' - '!**/pipeline/shared/**/*.go' + - '**/shared/*.go' + - '**/shared/**/*.go' deny: - pkg: go.woodpecker-ci.org/woodpecker/v3/agent - pkg: go.woodpecker-ci.org/woodpecker/v3/cli - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd - pkg: go.woodpecker-ci.org/woodpecker/v3/pipeline + - pkg: go.woodpecker-ci.org/woodpecker/v3/rpc - pkg: go.woodpecker-ci.org/woodpecker/v3/server - pkg: go.woodpecker-ci.org/woodpecker/v3/web woodpecker-go: list-mode: lax files: - - '**/woodpecker-go/woodpecker/**/*.go' - '**/woodpecker-go/woodpecker/*.go' + - '**/woodpecker-go/woodpecker/**/*.go' deny: - pkg: go.woodpecker-ci.org/woodpecker/v3/agent - pkg: go.woodpecker-ci.org/woodpecker/v3/cli - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd - pkg: go.woodpecker-ci.org/woodpecker/v3/pipeline + - pkg: go.woodpecker-ci.org/woodpecker/v3/rpc - pkg: go.woodpecker-ci.org/woodpecker/v3/server - pkg: go.woodpecker-ci.org/woodpecker/v3/shared - pkg: go.woodpecker-ci.org/woodpecker/v3/web diff --git a/.mockery.yaml b/.mockery.yaml index 0990c2916..1f0dfe261 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -5,7 +5,7 @@ filename: mock_{{.InterfaceName}}.go pkgname: mocks recursive: true packages: - go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc: + go.woodpecker-ci.org/woodpecker/v3/rpc: config: recursive: false go.woodpecker-ci.org/woodpecker/v3/server/forge: diff --git a/Makefile b/Makefile index 6ec044a5b..c1a32a1d2 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,7 @@ generate: install-mockery generate-openapi ## Run all code generations CGO_ENABLED=0 go generate ./... generate-openapi: ## Run openapi code generation and format it - CGO_ENABLED=0 go run github.com/swaggo/swag/cmd/swag fmt --exclude pipeline/rpc/proto + CGO_ENABLED=0 go run github.com/swaggo/swag/cmd/swag fmt --exclude rpc/proto CGO_ENABLED=0 go generate cmd/server/openapi.go generate-license-header: install-addlicense diff --git a/pipeline/log/line_writer.go b/agent/log/line_writer.go similarity index 97% rename from pipeline/log/line_writer.go rename to agent/log/line_writer.go index b88cbd671..3c3227434 100644 --- a/pipeline/log/line_writer.go +++ b/agent/log/line_writer.go @@ -23,8 +23,8 @@ import ( "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v3/pipeline/shared" + "go.woodpecker-ci.org/woodpecker/v3/rpc" ) // LineWriter sends logs to the client. diff --git a/pipeline/log/line_writer_test.go b/agent/log/line_writer_test.go similarity index 90% rename from pipeline/log/line_writer_test.go rename to agent/log/line_writer_test.go index 551899a45..e080ef7c9 100644 --- a/pipeline/log/line_writer_test.go +++ b/agent/log/line_writer_test.go @@ -20,9 +20,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/log" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/mocks" + "go.woodpecker-ci.org/woodpecker/v3/agent/log" + "go.woodpecker-ci.org/woodpecker/v3/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc/mocks" ) func TestLineWriter(t *testing.T) { diff --git a/agent/logger.go b/agent/logger.go index 47413df90..32052b1e9 100644 --- a/agent/logger.go +++ b/agent/logger.go @@ -20,10 +20,11 @@ import ( "github.com/rs/zerolog" + "go.woodpecker-ci.org/woodpecker/v3/agent/log" "go.woodpecker-ci.org/woodpecker/v3/pipeline" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/log" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + pipeline_utils "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils" + "go.woodpecker-ci.org/woodpecker/v3/rpc" ) func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger { @@ -44,7 +45,7 @@ func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, w logger.Debug().Msg("log stream opened") logStream := log.NewLineWriter(r.client, step.UUID, secrets...) - if err := log.CopyLineByLine(logStream, rc, pipeline.MaxLogLineLength); err != nil { + if err := pipeline_utils.CopyLineByLine(logStream, rc, pipeline.MaxLogLineLength); err != nil { logger.Error().Err(err).Msg("copy limited logStream part") } diff --git a/agent/rpc/auth_client_grpc.go b/agent/rpc/auth_client_grpc.go index 7abc7835d..92eb1cb98 100644 --- a/agent/rpc/auth_client_grpc.go +++ b/agent/rpc/auth_client_grpc.go @@ -20,7 +20,7 @@ import ( "google.golang.org/grpc" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/proto" + "go.woodpecker-ci.org/woodpecker/v3/rpc/proto" ) const authClientTimeout = time.Second * 5 diff --git a/agent/rpc/client_grpc.go b/agent/rpc/client_grpc.go index 2b5de5b65..57d1cad9c 100644 --- a/agent/rpc/client_grpc.go +++ b/agent/rpc/client_grpc.go @@ -28,8 +28,8 @@ import ( grpcproto "google.golang.org/protobuf/proto" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/proto" + "go.woodpecker-ci.org/woodpecker/v3/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc/proto" ) const ( diff --git a/agent/runner.go b/agent/runner.go index f5640982e..4cf5b835e 100644 --- a/agent/runner.go +++ b/agent/runner.go @@ -27,7 +27,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline" backend "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/shared/constant" "go.woodpecker-ci.org/woodpecker/v3/shared/utils" ) diff --git a/agent/tracer.go b/agent/tracer.go index a3af590d3..2fc7308b4 100644 --- a/agent/tracer.go +++ b/agent/tracer.go @@ -24,7 +24,7 @@ import ( "github.com/rs/zerolog" "go.woodpecker-ci.org/woodpecker/v3/pipeline" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" ) func (r *Runner) createTracer(ctxMeta context.Context, uploads *sync.WaitGroup, logger zerolog.Logger, workflow *rpc.Workflow) pipeline.TraceFunc { diff --git a/cli/exec/exec.go b/cli/exec/exec.go index 35b60fbac..c96c5beb5 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -45,7 +45,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/matrix" - pipelineLog "go.woodpecker-ci.org/woodpecker/v3/pipeline/log" + pipeline_utils "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils" "go.woodpecker-ci.org/woodpecker/v3/shared/constant" "go.woodpecker-ci.org/woodpecker/v3/shared/utils" ) @@ -350,5 +350,5 @@ func convertPathForWindows(path string) string { var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error { logWriter := NewLineWriter(step.Name, step.UUID) - return pipelineLog.CopyLineByLine(logWriter, rc, pipeline.MaxLogLineLength) + return pipeline_utils.CopyLineByLine(logWriter, rc, pipeline.MaxLogLineLength) }) diff --git a/cmd/agent/core/agent.go b/cmd/agent/core/agent.go index 889cd2c65..0ac9e6914 100644 --- a/cmd/agent/core/agent.go +++ b/cmd/agent/core/agent.go @@ -43,7 +43,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline" "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend" "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/shared/logger" "go.woodpecker-ci.org/woodpecker/v3/shared/utils" "go.woodpecker-ci.org/woodpecker/v3/version" diff --git a/cmd/server/grpc_server.go b/cmd/server/grpc_server.go index 7e7cf2a1a..216dde8de 100644 --- a/cmd/server/grpc_server.go +++ b/cmd/server/grpc_server.go @@ -24,9 +24,9 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/keepalive" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/proto" + "go.woodpecker-ci.org/woodpecker/v3/rpc/proto" "go.woodpecker-ci.org/woodpecker/v3/server" - woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v3/server/grpc" + woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v3/server/rpc" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) diff --git a/docs/docs/92-development/05-architecture.md b/docs/docs/92-development/05-architecture.md index 535ec6a80..b5ac422e6 100644 --- a/docs/docs/92-development/05-architecture.md +++ b/docs/docs/92-development/05-architecture.md @@ -11,38 +11,70 @@ | package | meaning | imports | | ------------------ | -------------------------------------------------------------- | ------------------------------------- | | `cmd/**` | parse command-line args & environment to stat server/cli/agent | all other | -| `agent/**` | code only agent (remote worker) will need | `pipeline`, `shared` | +| `agent/**` | code only agent (remote worker) will need | `pipeline`, `rpc`, `shared` | | `cli/**` | code only cli tool does need | `pipeline`, `shared`, `woodpecker-go` | -| `server/**` | code only server will need | `pipeline`, `shared` | +| `server/**` | code only server will need | `pipeline`, `rpc`, `shared` | +| `pipeline/**` | core ci/cd engine from parsing to execution | `shared` | +| `rpc/**` | RPC interface for agent-server communication | `pipeline` | | `shared/**` | code shared for all three main tools (go help utils) | only std and external libs | | `woodpecker-go/**` | go client for server rest api | std | ### Server -| package | meaning | imports | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `server/api/**` | handle web requests from `server/router` | `pipeline`, `../badges`, `../ccmenu`, `../logging`, `../model`, `../pubsub`, `../queue`, `../forge`, `../shared`, `../store`, `shared`, (TODO: mv `server/router/middleware/session`) | -| `server/badges/**` | generate svg badges for pipelines | `../model` | -| `server/ccmenu/**` | generate xml ccmenu for pipelines | `../model` | -| `server/grpc/**` | gRPC server agents can connect to | `pipeline/rpc/**`, `../logging`, `../model`, `../pubsub`, `../queue`, `../forge`, `../pipeline`, `../store` | -| `server/logging/**` | logging lib for gPRC server to stream logs while running | std | -| `server/model/**` | structs for store (db) and api (json) | std | -| `server/plugins/**` | plugins for server | `../model`, `../forge` | -| `server/pipeline/**` | orchestrate pipelines | `pipeline`, `../model`, `../pubsub`, `../queue`, `../forge`, `../store`, `../plugins` | -| `server/pubsub/**` | pubsub lib for server to push changes to the WebUI | std | -| `server/queue/**` | queue lib for server where agents pull new pipelines from via gRPC | `server/model` | -| `server/forge/**` | forge lib for server to connect and handle forge specific stuff | `shared`, `server/model` | -| `server/router/**` | handle requests to REST API (and all middleware) and serve UI and WebUI config | `shared`, `../api`, `../model`, `../forge`, `../store`, `../web` | -| `server/store/**` | handle database | `server/model` | -| `server/shared/**` | TODO: move and split [#974](https://github.com/woodpecker-ci/woodpecker/issues/974) | | -| `server/web/**` | server SPA | | +| package | meaning | imports | +| -------------------- | ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `server/api/**` | handle web requests from `server/router` | `pipeline`, `rpc`, `../badges`, `../ccmenu`, `../logging`, `../model`, `../pubsub`, `../queue`, `../forge`, `../shared`, `../store`, `shared`, (TODO: mv `server/router/middleware/session`) | +| `server/badges/**` | generate svg badges for pipelines | `../model` | +| `server/ccmenu/**` | generate xml ccmenu for pipelines | `../model` | +| `server/rpc/**` | gRPC server agents can connect to | `rpc`, `../logging`, `../model`, `../pubsub`, `../queue`, `../forge`, `../pipeline`, `../store` | +| `server/logging/**` | logging lib for gPRC server to stream logs while running | std | +| `server/model/**` | structs for store (db) and api (json) | std | +| `server/pipeline/**` | orchestrate pipelines (TODO: parts of it should move into /pipeline) | `pipeline`, `../model`, `../pubsub`, `../queue`, `../forge`, `../store`, `../plugins` | +| `server/pubsub/**` | pubsub lib for server to push changes to the WebUI | std | +| `server/queue/**` | queue lib for server where agents pull new pipelines from via gRPC | `server/model` | +| `server/forge/**` | forge lib for server to connect and handle forge specific stuff | `shared`, `server/model` | +| `server/router/**` | handle requests to REST API (and all middleware) and serve UI and WebUI config | `shared`, `../api`, `../model`, `../forge`, `../store`, `../web` | +| `server/store/**` | handle database | `server/model` | +| `server/web/**` | server SPA | | - `../` = `server/` ### Agent -TODO +| package | meaning | imports | +| -------------- | ---------------------------------------------------- | ------------------------------------------------------ | +| `agent/**` | agent implementation that runs workflows | `pipeline`, `rpc`, `shared` | +| `agent/rpc/**` | gRPC client for agent-server communication | `rpc`, `pipeline/backend/types`, std and external libs | +| `cmd/agent/**` | CLI interface for starting and configuring the agent | `agent`, std and external libs | + +The agent is a remote worker that connects to the server via gRPC to receive pipeline execution instructions and report back execution state and logs. +The agent polls the server's queue for new work, executes pipeline steps using the pipeline engine, and streams results back to the server. + +TODO: Review cmd/agent/core to determine if any logic should be moved into the agent package for better separation of concerns. ### CLI -TODO +| package | meaning | imports | +| ------------------------ | ----------------------------------------------------------------------- | -------------------------------------------------------------------------------- | +| `cli/admin/**` | admin commands for server management (users, secrets, registries, etc.) | `../common`, `../internal`, `woodpecker-go` | +| `cli/common/**` | shared utilities and helpers used across all CLI subcommands | `../internal/config`, `../update`, `shared` | +| `cli/context/**` | manage multiple server contexts (connections to different servers) | `../common`, `../internal/config`, `../output` | +| `cli/exec/**` | execute pipelines locally without server orchestration | `pipeline`, `../common`, `../lint`, `shared` | +| `cli/info/**` | display information about the current user | `../common`, `../internal` | +| `cli/internal/**` | internal utilities for HTTP client, auth, and server communication | `../internal/config`, `woodpecker-go`, `shared` | +| `cli/internal/config/**` | configuration file management (load, store, credentials) | std and external libs | +| `cli/lint/**` | validate pipeline configuration files | `pipeline/frontend/yaml`, `pipeline/frontend/yaml/linter`, `../common`, `shared` | +| `cli/org/**` | manage organization-level resources (secrets, registries) | `../common`, `../internal`, `woodpecker-go` | +| `cli/output/**` | formatting utilities for CLI output (tables, etc.) | std and external libs | +| `cli/pipeline/**` | manage pipeline operations (start, stop, approve, logs, etc.) | `../common`, `../internal`, `../output`, `woodpecker-go`, `shared` | +| `cli/repo/**` | manage repository-level resources (repos, crons, secrets, registries) | `../common`, `../internal`, `../output`, `woodpecker-go` | +| `cli/setup/**` | interactive first-time setup wizard for CLI configuration | `../internal/config` | +| `cli/update/**` | self-updater for the CLI binary | std and external libs | +| `cmd/cli/**` | CLI entry point and command structure | `cli/**` | + +The CLI provides a command-line interface for interacting with Woodpecker servers. +Each subcommand is organized into its own package under `cli//`. + +The `cli/exec` subcommand allows local pipeline execution for testing and development by combining pipeline parsing and execution without requiring a running server or agent. + +- `../` = `cli/` diff --git a/pipeline/log/utils.go b/pipeline/utils/copy_line_by_line.go similarity index 99% rename from pipeline/log/utils.go rename to pipeline/utils/copy_line_by_line.go index 84a29e946..09cbf9ce6 100644 --- a/pipeline/log/utils.go +++ b/pipeline/utils/copy_line_by_line.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package log +package utils import ( "bufio" diff --git a/pipeline/log/utils_test.go b/pipeline/utils/copy_line_by_line_test.go similarity index 96% rename from pipeline/log/utils_test.go rename to pipeline/utils/copy_line_by_line_test.go index 8b83b67ea..bc63ae984 100644 --- a/pipeline/log/utils_test.go +++ b/pipeline/utils/copy_line_by_line_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package log_test +package utils_test import ( "io" @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/log" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/utils" ) type testWriter struct { @@ -61,7 +61,7 @@ func TestCopyLineByLine(t *testing.T) { done := make(chan struct{}) go func() { - err := log.CopyLineByLine(testWriter, r, 1024) + err := utils.CopyLineByLine(testWriter, r, 1024) assert.NoError(t, err) close(done) }() @@ -118,7 +118,7 @@ func TestCopyLineByLineSizeLimit(t *testing.T) { go func() { defer wg.Done() - err := log.CopyLineByLine(testWriter, r, 4) + err := utils.CopyLineByLine(testWriter, r, 4) assert.NoError(t, err) }() @@ -161,7 +161,7 @@ func TestStringReader(t *testing.T) { writes: make([]string, 0), } - err := log.CopyLineByLine(testWriter, r, 1024) + err := utils.CopyLineByLine(testWriter, r, 1024) assert.NoError(t, err) writes := testWriter.GetWrites() @@ -179,7 +179,7 @@ func TestCopyLineByLineNewlineCharacter(t *testing.T) { done := make(chan struct{}) go func() { - err := log.CopyLineByLine(testWriter, r, 4) + err := utils.CopyLineByLine(testWriter, r, 4) assert.NoError(t, err) close(done) }() @@ -243,7 +243,7 @@ func TestCopyLineByLineLongLine(t *testing.T) { maxSize := 10 go func() { - err := log.CopyLineByLine(testWriter, r, maxSize) + err := utils.CopyLineByLine(testWriter, r, maxSize) assert.NoError(t, err) close(done) }() @@ -296,7 +296,7 @@ func TestCopyLineByLineWriteChunks(t *testing.T) { maxSize := 8 go func() { - err := log.CopyLineByLine(testWriter, r, maxSize) + err := utils.CopyLineByLine(testWriter, r, maxSize) assert.NoError(t, err) close(done) }() diff --git a/pipeline/rpc/log_entry.go b/rpc/log_entry.go similarity index 100% rename from pipeline/rpc/log_entry.go rename to rpc/log_entry.go diff --git a/pipeline/rpc/log_entry_test.go b/rpc/log_entry_test.go similarity index 100% rename from pipeline/rpc/log_entry_test.go rename to rpc/log_entry_test.go diff --git a/pipeline/rpc/mocks/mock_Peer.go b/rpc/mocks/mock_Peer.go similarity index 99% rename from pipeline/rpc/mocks/mock_Peer.go rename to rpc/mocks/mock_Peer.go index 1205d9737..1596f5b40 100644 --- a/pipeline/rpc/mocks/mock_Peer.go +++ b/rpc/mocks/mock_Peer.go @@ -8,7 +8,7 @@ import ( "context" mock "github.com/stretchr/testify/mock" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" ) // NewMockPeer creates a new instance of MockPeer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. diff --git a/pipeline/rpc/peer.go b/rpc/peer.go similarity index 100% rename from pipeline/rpc/peer.go rename to rpc/peer.go diff --git a/pipeline/rpc/proto/generate.go b/rpc/proto/generate.go similarity index 100% rename from pipeline/rpc/proto/generate.go rename to rpc/proto/generate.go diff --git a/pipeline/rpc/proto/version.go b/rpc/proto/version.go similarity index 100% rename from pipeline/rpc/proto/version.go rename to rpc/proto/version.go diff --git a/pipeline/rpc/proto/woodpecker.pb.go b/rpc/proto/woodpecker.pb.go similarity index 99% rename from pipeline/rpc/proto/woodpecker.pb.go rename to rpc/proto/woodpecker.pb.go index a08a42f08..b13b8b3ab 100644 --- a/pipeline/rpc/proto/woodpecker.pb.go +++ b/rpc/proto/woodpecker.pb.go @@ -1238,7 +1238,7 @@ const file_woodpecker_proto_rawDesc = "" + "\x0fUnregisterAgent\x12\f.proto.Empty\x1a\f.proto.Empty\"\x00\x12:\n" + "\fReportHealth\x12\x1a.proto.ReportHealthRequest\x1a\f.proto.Empty\"\x002C\n" + "\x0eWoodpeckerAuth\x121\n" + - "\x04Auth\x12\x12.proto.AuthRequest\x1a\x13.proto.AuthResponse\"\x00B7Z5go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/protob\x06proto3" + "\x04Auth\x12\x12.proto.AuthRequest\x1a\x13.proto.AuthResponse\"\x00B.Z,go.woodpecker-ci.org/woodpecker/v3/rpc/protob\x06proto3" var ( file_woodpecker_proto_rawDescOnce sync.Once diff --git a/pipeline/rpc/proto/woodpecker.proto b/rpc/proto/woodpecker.proto similarity index 97% rename from pipeline/rpc/proto/woodpecker.proto rename to rpc/proto/woodpecker.proto index 58410f820..38cf2352f 100644 --- a/pipeline/rpc/proto/woodpecker.proto +++ b/rpc/proto/woodpecker.proto @@ -15,7 +15,7 @@ syntax = "proto3"; -option go_package = "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/proto"; +option go_package = "go.woodpecker-ci.org/woodpecker/v3/rpc/proto"; package proto; // !IMPORTANT! diff --git a/pipeline/rpc/proto/woodpecker_grpc.pb.go b/rpc/proto/woodpecker_grpc.pb.go similarity index 100% rename from pipeline/rpc/proto/woodpecker_grpc.pb.go rename to rpc/proto/woodpecker_grpc.pb.go diff --git a/server/pipeline/queue.go b/server/pipeline/queue.go index f204b4461..f921930c2 100644 --- a/server/pipeline/queue.go +++ b/server/pipeline/queue.go @@ -20,7 +20,7 @@ import ( "fmt" "maps" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/stepbuilder" diff --git a/server/pipeline/step_status.go b/server/pipeline/step_status.go index 31ac5b8dd..fabce7d86 100644 --- a/server/pipeline/step_status.go +++ b/server/pipeline/step_status.go @@ -17,7 +17,7 @@ package pipeline import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) diff --git a/server/pipeline/step_status_test.go b/server/pipeline/step_status_test.go index 0537d892b..08a122826 100644 --- a/server/pipeline/step_status_test.go +++ b/server/pipeline/step_status_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/mock" "go.woodpecker-ci.org/woodpecker/v3/pipeline" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/store" "go.woodpecker-ci.org/woodpecker/v3/server/store/mocks" diff --git a/server/pipeline/workflow_status.go b/server/pipeline/workflow_status.go index 12d2c15a7..3bf81d135 100644 --- a/server/pipeline/workflow_status.go +++ b/server/pipeline/workflow_status.go @@ -15,7 +15,7 @@ package pipeline import ( - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) diff --git a/server/grpc/auth_server.go b/server/rpc/auth_server.go similarity index 97% rename from server/grpc/auth_server.go rename to server/rpc/auth_server.go index c009b6eca..45621b113 100644 --- a/server/grpc/auth_server.go +++ b/server/rpc/auth_server.go @@ -21,7 +21,7 @@ import ( "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/proto" + "go.woodpecker-ci.org/woodpecker/v3/rpc/proto" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/store" "go.woodpecker-ci.org/woodpecker/v3/server/store/types" diff --git a/server/grpc/authorizer.go b/server/rpc/authorizer.go similarity index 100% rename from server/grpc/authorizer.go rename to server/rpc/authorizer.go diff --git a/server/grpc/filter.go b/server/rpc/filter.go similarity index 97% rename from server/grpc/filter.go rename to server/rpc/filter.go index 177499abe..3a1edc8fd 100644 --- a/server/grpc/filter.go +++ b/server/rpc/filter.go @@ -19,7 +19,7 @@ import ( "strings" pipelineConsts "go.woodpecker-ci.org/woodpecker/v3/pipeline" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/queue" ) diff --git a/server/grpc/filter_test.go b/server/rpc/filter_test.go similarity index 98% rename from server/grpc/filter_test.go rename to server/rpc/filter_test.go index 4a4b01f79..04d0e13dc 100644 --- a/server/grpc/filter_test.go +++ b/server/rpc/filter_test.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/assert" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server/model" ) diff --git a/server/grpc/jwt_manager.go b/server/rpc/jwt_manager.go similarity index 100% rename from server/grpc/jwt_manager.go rename to server/rpc/jwt_manager.go diff --git a/server/grpc/rpc.go b/server/rpc/rpc.go similarity index 99% rename from server/grpc/rpc.go rename to server/rpc/rpc.go index bef1b7fdd..c32693de1 100644 --- a/server/grpc/rpc.go +++ b/server/rpc/rpc.go @@ -29,7 +29,7 @@ import ( "github.com/rs/zerolog/log" grpcMetadata "google.golang.org/grpc/metadata" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/forge" "go.woodpecker-ci.org/woodpecker/v3/server/logging" diff --git a/server/grpc/rpc_test.go b/server/rpc/rpc_test.go similarity index 98% rename from server/grpc/rpc_test.go rename to server/rpc/rpc_test.go index 8c3775c7c..04a6a6f60 100644 --- a/server/grpc/rpc_test.go +++ b/server/rpc/rpc_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc/metadata" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server/model" store_mocks "go.woodpecker-ci.org/woodpecker/v3/server/store/mocks" ) diff --git a/server/grpc/server.go b/server/rpc/server.go similarity index 98% rename from server/grpc/server.go rename to server/rpc/server.go index 4a79ef447..b5414db62 100644 --- a/server/grpc/server.go +++ b/server/rpc/server.go @@ -22,8 +22,8 @@ import ( prometheus_auto "github.com/prometheus/client_golang/prometheus/promauto" "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc" - "go.woodpecker-ci.org/woodpecker/v3/pipeline/rpc/proto" + "go.woodpecker-ci.org/woodpecker/v3/rpc" + "go.woodpecker-ci.org/woodpecker/v3/rpc/proto" "go.woodpecker-ci.org/woodpecker/v3/server/logging" "go.woodpecker-ci.org/woodpecker/v3/server/pubsub" "go.woodpecker-ci.org/woodpecker/v3/server/queue"