From 3f70822458b435ca406fda96671289f2d846d339 Mon Sep 17 00:00:00 2001 From: silverwind Date: Tue, 18 Aug 2026 20:16:11 +0000 Subject: [PATCH] enhance: add `runner.tool_cache_mode` and default it to `none` (#1171) The current shared tools cache is not concurrency-safe, e.g. multiple jobs can write and corrupt it, for example `setup-go` with explicit go version under concurrency reliably corrupts the tool cache and fails all jobs. This adds a new `runner.tool_cache_mode` (and `--tool-cache-mode` exec option) option which defaults to unshared tools cache: - `none` mounts nothing, so a job uses what its image ships there and discards what it installs - `shared` keeps the single volume every job reuses, and warns when `runner.capacity` is above 1 Under `none` effective tool cache can only come from the image or host, which is the same as it is on GitHub Actions which ships many preinstalled tools in its fat VM images. Co-authored-by: bircni Reviewed-on: https://gitea.com/gitea/runner/pulls/1171 Reviewed-by: bircni Co-authored-by: silverwind --- README.md | 26 +++++++++++++++++++++++ act/runner/run_context.go | 19 +++++++++++++---- act/runner/run_context_test.go | 28 +++++++++++++++++++++---- act/runner/runner.go | 1 + internal/app/cmd/exec.go | 19 ++++++++++++++++- internal/app/cmd/exec_test.go | 15 +++++++++++++ internal/app/run/runner.go | 7 ++++--- internal/pkg/config/config.example.yaml | 8 ++++++- internal/pkg/config/config.go | 20 ++++++++++++++++++ internal/pkg/config/config_test.go | 17 +++++++++++++++ 10 files changed, 147 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 95e1b4a2..1dfb1f32 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,32 @@ An edit keeps the comments and the key order of the file. Indentation becomes tw `config get`, `set`, `add` and `remove` use `config.yaml` (or `config.yml`) from the working directory, then from the directory of the binary, and print their choice to stderr. `config init` writes `config.yaml` in the working directory, and refuses to overwrite an existing config without `--force`. Pass `-c` for another path. +#### Tool cache + +Setup actions like `setup-go` install tools into `RUNNER_TOOL_CACHE`, which is `/opt/hostedtoolcache` inside a job. `runner.tool_cache_mode` selects what backs it: + +| Mode | Tool cache | Trade-off | +| --- | --- | --- | +| `none` (default) | Per job, provided by the job image | A version the image lacks is downloaded in every job | +| `shared` | One volume reused by every job | Two jobs writing the same tool version at once corrupt it, so use it only with `runner.capacity: 1` | + +With `none`, tools must come from the job image. Install them into `/opt/hostedtoolcache///`, with an empty `.complete` file next to the directory: + +```dockerfile +RUN GO=$(curl -fsSL 'https://go.dev/dl/?mode=json' | grep -oP '"version": "\Kgo1\.26\.[0-9]*' | head -1); \ + DIR="/opt/hostedtoolcache/go/${GO#go}/x64" && \ + mkdir -p "$(dirname "$DIR")" && \ + curl -fsSL "https://dl.google.com/go/${GO}.linux-amd64.tar.gz" | tar -xz -C /tmp && \ + mv /tmp/go "$DIR" && \ + touch "${DIR}.complete" +``` + +A workflow requesting a minor version, `go-version: "1.26"`, resolves to the newest matching version in the cache, so a patch update in the image still hits it. + +Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavour is the one that ships tools in this layout. + +`gitea-runner exec` reads no config file and takes `--tool-cache-mode` instead, defaulting to `none`. + #### Environment variables Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below. diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 5b7c90e3..e8946160 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -229,14 +229,19 @@ func (rc *RunContext) containerDaemonSocket() string { return rc.Config.ContainerDaemonSocket } +const sharedToolCacheVolume = "act-toolcache" // mounted only when the tool cache is shared + // validVolumes returns the volumes allowed on this job's containers: the configured base // plus the volumes the runner mounts automatically. It derives a fresh slice every call and // never mutates the shared Config (see containerDaemonSocket). func (rc *RunContext) validVolumes() []string { name := rc.jobContainerName() volumes := slices.Clone(rc.Config.ValidVolumes) + if rc.Config.SharedToolCache { + volumes = append(volumes, sharedToolCacheVolume) + } // TODO: add a new configuration to control whether the docker daemon can be mounted - return append(volumes, "act-toolcache", name, name+"-env", + return append(volumes, name, name+"-env", getDockerDaemonSocketMountPath(rc.containerDaemonSocket())) } @@ -309,8 +314,10 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { if daemonSocket := rc.containerDaemonSocket(); daemonSocket != "-" && !claimed["/var/run/docker.sock"] { binds = append(binds, getDockerDaemonSocketMountPath(daemonSocket)+":/var/run/docker.sock") } - if toolCache := rc.toolCache(container.DefaultToolCache); !claimed[toolCache] { - mounts["act-toolcache"] = toolCache + if rc.Config.SharedToolCache { + if toolCache := rc.toolCache(container.DefaultToolCache); !claimed[toolCache] { + mounts[sharedToolCacheVolume] = toolCache + } } mounts[name+"-env"] = ext.GetActPath() // runner-internal, never overridable @@ -360,7 +367,11 @@ func (rc *RunContext) startHostEnvironment() common.Executor { if err := os.MkdirAll(runnerTmp, 0o777); err != nil { return err } - toolCache := rc.toolCache(filepath.Join(cacheDir, "tool_cache")) + toolCacheParent := miscpath // per job, so cleanup removes it with the job + if rc.Config.SharedToolCache { + toolCacheParent = cacheDir + } + toolCache := rc.toolCache(filepath.Join(toolCacheParent, "tool_cache")) if err := os.MkdirAll(toolCache, 0o777); err != nil { return err } diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index dcc8edc6..1e22b584 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -502,7 +502,8 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) { }, }, Config: &Config{ - BindWorkdir: false, + BindWorkdir: false, + SharedToolCache: true, // so OverridesToolCache has a mount to displace }, } rc.Run.JobID = "job1" @@ -543,25 +544,44 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) { }) } }) + + t.Run("ToolCacheMount", func(t *testing.T) { + rc := &RunContext{ + Name: "TestRCName", + Run: &model.Run{Workflow: &model.Workflow{Name: "TestWorkflowName"}}, + Config: &Config{}, + } + + _, gotmount := rc.GetBindsAndMounts() + assert.NotContains(t, gotmount, sharedToolCacheVolume) + + rc.Config.SharedToolCache = true + _, gotmount = rc.GetBindsAndMounts() + assert.Equal(t, container.DefaultToolCache, gotmount[sharedToolCacheVolume]) + }) } func TestRunContextValidVolumes(t *testing.T) { rc := &RunContext{ Name: "job", Run: &model.Run{Workflow: &model.Workflow{Name: "wf"}}, - Config: &Config{ValidVolumes: []string{"my-vol", "/host/path"}}, + Config: &Config{ValidVolumes: []string{"my-vol", "/host/path"}, SharedToolCache: true}, } name := rc.jobContainerName() got := rc.validVolumes() - // the configured volumes plus the four the runner mounts automatically - assert.Subset(t, got, []string{"my-vol", "/host/path", "act-toolcache", name, name + "-env", "/var/run/docker.sock"}) + // the configured volumes plus the ones the runner mounts automatically + assert.Subset(t, got, []string{"my-vol", "/host/path", sharedToolCacheVolume, name, name + "-env", "/var/run/docker.sock"}) // deriving the list must never mutate or grow the shared Config slice: parallel matrix // combinations share one *Config, and the previous in-place append was a data race. assert.Equal(t, []string{"my-vol", "/host/path"}, rc.Config.ValidVolumes) assert.Len(t, rc.validVolumes(), len(got), "repeated calls must be stable, not accumulate") + + // a job may mount it only while the runner does + rc.Config.SharedToolCache = false + assert.NotContains(t, rc.validVolumes(), sharedToolCacheVolume) } func TestCleanupJobResourcesCleansServicesWithoutJobContainer(t *testing.T) { diff --git a/act/runner/runner.go b/act/runner/runner.go index 5995149b..a05737f9 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -91,6 +91,7 @@ type Config struct { PlatformPicker func(labels []string) string // platform picker, it will take precedence over Platforms if isn't nil JobLoggerLevel *log.Level // the level of job logger ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers + SharedToolCache bool // one tool cache for all jobs instead of one per job InsecureSkipTLS bool // whether to skip verifying TLS certificate of the Gitea instance MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count) AllocatePTY bool // allocate a pseudo-TTY for each step's process diff --git a/internal/app/cmd/exec.go b/internal/app/cmd/exec.go index 37275c65..ae2e71b9 100644 --- a/internal/app/cmd/exec.go +++ b/internal/app/cmd/exec.go @@ -13,6 +13,7 @@ import ( "maps" "os" "path/filepath" + "slices" "strconv" "strings" "time" @@ -68,6 +69,15 @@ type executeArgs struct { cacheHandler *artifactcache.Handler network string githubInstance string + toolCacheMode string +} + +// sharedToolCache reports whether mode mounts one tool cache for every job. +func sharedToolCache(mode string) (bool, error) { + if !slices.Contains(config.ToolCacheModes, mode) { + return false, fmt.Errorf("invalid --tool-cache-mode %q: must be one of %q", mode, config.ToolCacheModes) + } + return mode == config.ToolCacheModeShared, nil } // WorkflowsPath returns path to workflow file(s) @@ -427,6 +437,11 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command proxyEnv := run.JobProxyEnv(env, env["ACTIONS_CACHE_URL"], nil) maps.Copy(env, proxyEnv) + shared, err := sharedToolCache(execArgs.toolCacheMode) + if err != nil { + return err + } + // run the plan config := &runner.Config{ Workdir: execArgs.Workdir(), @@ -467,7 +482,8 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command PlatformPicker: func(_ []string) string { return execArgs.image }, - ValidVolumes: []string{"**"}, // All volumes are allowed for `exec` command + ValidVolumes: []string{"**"}, // All volumes are allowed for `exec` command + SharedToolCache: shared, } config.Env["ACT_EXEC"] = "true" @@ -543,6 +559,7 @@ func loadExecCmd(ctx context.Context) *cobra.Command { execCmd.PersistentFlags().BoolVarP(&execArg.debug, "debug", "d", false, "enable debug log") execCmd.PersistentFlags().BoolVarP(&execArg.dryrun, "dryrun", "n", false, "dryrun mode") execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", "docker.gitea.com/runner-images:ubuntu-latest", "Docker image to use. Use \"-self-hosted\" to run directly on the host.") + execCmd.PersistentFlags().StringVarP(&execArg.toolCacheMode, "tool-cache-mode", "", config.ToolCacheModeNone, "What to mount at RUNNER_TOOL_CACHE: none, or shared to reuse one tool cache across runs") execCmd.PersistentFlags().StringVarP(&execArg.network, "network", "", "", "Specify the network to which the container will connect") execCmd.PersistentFlags().StringVarP(&execArg.githubInstance, "gitea-instance", "", "", "Gitea instance to use.") diff --git a/internal/app/cmd/exec_test.go b/internal/app/cmd/exec_test.go index 14e3ff03..ca6e3af5 100644 --- a/internal/app/cmd/exec_test.go +++ b/internal/app/cmd/exec_test.go @@ -12,6 +12,8 @@ import ( "strings" "testing" + "gitea.com/gitea/runner/internal/pkg/config" + "gitea.dev/actionslib/pkg/model" "github.com/stretchr/testify/require" "go.yaml.in/yaml/v4" @@ -28,6 +30,19 @@ func TestExecuteArgsResolve(t *testing.T) { require.Equal(t, abs, args.resolve(abs)) } +func TestSharedToolCache(t *testing.T) { + shared, err := sharedToolCache(config.ToolCacheModeShared) + require.NoError(t, err) + require.True(t, shared) + + shared, err = sharedToolCache(config.ToolCacheModeNone) + require.NoError(t, err) + require.False(t, shared) + + _, err = sharedToolCache("everyone") + require.ErrorContains(t, err, "tool-cache-mode") +} + func TestExecuteArgsPaths(t *testing.T) { workdir := t.TempDir() args := &executeArgs{ diff --git a/internal/app/run/runner.go b/internal/app/run/runner.go index 07192af8..0bfa2adf 100644 --- a/internal/app/run/runner.go +++ b/internal/app/run/runner.go @@ -159,8 +159,8 @@ func (r *Runner) OnIdle(ctx context.Context) { } // Host mode: reclaim per-job scratch dirs left behind when HostEnvironment // cleanup timed out (e.g. a delete stalled by an AV/EDR filter driver). They - // sit under the host workdir parent alongside the shared tool_cache, which - // the name match leaves untouched. No-op when no host-mode job ever ran. + // sit under the host workdir parent next to a shared tool_cache, which the name + // match leaves untouched. No-op when no host-mode job ever ran. if hostRoot := filepath.FromSlash(r.cfg.Host.WorkdirParent); hostRoot != "" { r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir) } @@ -219,7 +219,7 @@ func isTaskIDDir(name string) bool { // isHostScratchDir reports whether name is a per-job host-mode scratch dir: // hex.EncodeToString of 8 random bytes, i.e. exactly 16 lowercase hex chars // (see startHostEnvironment in act/runner/run_context.go). The narrow match -// leaves the sibling shared "tool_cache" dir and any operator data untouched. +// leaves a sibling shared "tool_cache" dir and any operator data untouched. func isHostScratchDir(name string) bool { if len(name) != 16 { return false @@ -522,6 +522,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. JobCompletedHook: r.cfg.Runner.Hooks.JobCompleted, Vars: task.Vars, ValidVolumes: r.cfg.Container.ValidVolumes, + SharedToolCache: r.cfg.Runner.ToolCacheMode == config.ToolCacheModeShared, InsecureSkipTLS: r.cfg.Runner.Insecure, RunnerName: r.name, } diff --git a/internal/pkg/config/config.example.yaml b/internal/pkg/config/config.example.yaml index dd6716e4..eab75350 100644 --- a/internal/pkg/config/config.example.yaml +++ b/internal/pkg/config/config.example.yaml @@ -93,6 +93,12 @@ runner: # terminal; tools like `docker build` emit redrawing progress frames into the captured log # when a TTY is present. #allocate_pty: false + # What to mount at RUNNER_TOOL_CACHE (/opt/hostedtoolcache), where setup actions install tools: + # none: nothing. A docker job sees what its image ships there, a host job an empty dir, and + # either way what it installs is gone when the job ends. + # shared: one tool cache every job reuses. Two jobs writing the same tool version at once + # corrupt it, so use it only with capacity 1. + #tool_cache_mode: none # Optional executable on the host, run once after each task's built-in cleanup # (post-steps, container teardown, bind-workdir removal). Additive only. # @@ -195,7 +201,7 @@ container: #privileged: false # Any other options to be used when the container is started, for example: # options: --add-host=my.gitea.url:host-gateway - # A volume declared here replaces the one the runner mounts on the same container path, so the + # A volume declared here replaces the one the runner would mount on the same container path, so the # tool cache can be kept on the host. Its source must also be allowed by valid_volumes below: # options: --volume /host/toolcache:/opt/hostedtoolcache #options: diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 0afd9c68..52e509d6 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -10,6 +10,7 @@ import ( "maps" "os" "path/filepath" + "slices" "strings" "time" @@ -60,6 +61,7 @@ type Runner struct { ActionShallowClone *bool `yaml:"action_shallow_clone"` // ActionShallowClone fetches only the requested ref of an action repository at depth 1 instead of cloning every branch's full history. It is a pointer to distinguish between false and not set; if not set, it defaults to true. SetActEnv *bool `yaml:"set_act_env"` // SetActEnv controls whether the ACT=true environment variable is injected into jobs. It is a pointer to distinguish between false and not set; if not set, it defaults to true. Set it to false so workflows gated on `if: ${{ !env.ACT }}` behave like on GitHub. AllocatePTY bool `yaml:"allocate_pty"` // AllocatePTY allocates a pseudo-TTY for each step's process. Default is false, matching GitHub's actions/runner. Enable only for jobs that need an interactive terminal; tools like docker build emit redrawing progress frames into the captured log when a TTY is present. Applies to both host and docker backends. + ToolCacheMode string `yaml:"tool_cache_mode"` // ToolCacheMode is what the runner mounts at RUNNER_TOOL_CACHE on both backends: ToolCacheModeNone or ToolCacheModeShared. PostTaskScript string `yaml:"post_task_script"` // PostTaskScript is the path to an executable script run on the host after each task's cleanup completes. Empty disables the hook. On Windows use .exe/.bat/.cmd; PowerShell (.ps1) is not supported yet as the configured path. PostTaskScriptTimeout time.Duration `yaml:"post_task_script_timeout"` // PostTaskScriptTimeout caps how long the post-task script may run. Default is 5m when post_task_script is set. Hooks RunnerHooks `yaml:"hooks"` // Hooks are scripts run inside the job environment around the job's steps. @@ -143,6 +145,14 @@ type Container struct { ServiceReadyTimeout time.Duration `yaml:"service_ready_timeout"` // ServiceReadyTimeout bounds how long a job waits for a service container that declares a healthcheck to report healthy. Negative disables waiting. } +// Values of Runner.ToolCacheMode: the runner mounts no tool cache, or one that every job reuses. +const ( + ToolCacheModeNone = "none" + ToolCacheModeShared = "shared" +) + +var ToolCacheModes = []string{ToolCacheModeNone, ToolCacheModeShared} + type ContainerNetworkCreateOptions struct { EnableIPv4 *bool `yaml:"enable_ipv4"` // Enable or disable IPv4 for the network (true for docker by default) EnableIPv6 *bool `yaml:"enable_ipv6"` // Enable or disable IPv6 for the network (false for docker by default) @@ -257,6 +267,12 @@ func LoadDefault(file string) (*Config, error) { if cfg.Container.WorkdirParent == "" { cfg.Container.WorkdirParent = "workspace" } + if cfg.Runner.ToolCacheMode == "" { + cfg.Runner.ToolCacheMode = ToolCacheModeNone + } + if !slices.Contains(ToolCacheModes, cfg.Runner.ToolCacheMode) { + return nil, fmt.Errorf("invalid runner.tool_cache_mode %q: must be one of %q", cfg.Runner.ToolCacheMode, ToolCacheModes) + } if cfg.Host.WorkdirParent == "" { home, err := os.UserHomeDir() if err != nil { @@ -314,6 +330,10 @@ func LoadDefault(file string) (*Config, error) { } // Validate and fix invalid config combinations to prevent confusing behavior. + if cfg.Runner.ToolCacheMode == ToolCacheModeShared && cfg.Runner.Capacity > 1 { + log.Warnf("runner.tool_cache_mode %q with capacity %d: two jobs writing the same tool version at once corrupt it", + ToolCacheModeShared, cfg.Runner.Capacity) + } if cfg.Runner.FetchIntervalMax < cfg.Runner.FetchInterval { log.Warnf("fetch_interval_max (%v) is less than fetch_interval (%v), setting fetch_interval_max to fetch_interval", cfg.Runner.FetchIntervalMax, cfg.Runner.FetchInterval) diff --git a/internal/pkg/config/config_test.go b/internal/pkg/config/config_test.go index 26908112..56b66ef5 100644 --- a/internal/pkg/config/config_test.go +++ b/internal/pkg/config/config_test.go @@ -42,6 +42,23 @@ cache: require.NoError(t, err) } +func TestLoadDefault_ToolCacheMode(t *testing.T) { + cfg, err := LoadDefault("") + require.NoError(t, err) + assert.Equal(t, ToolCacheModeNone, cfg.Runner.ToolCacheMode) + + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte("runner:\n tool_cache_mode: shared\n"), 0o600)) + cfg, err = LoadDefault(path) + require.NoError(t, err) + assert.Equal(t, ToolCacheModeShared, cfg.Runner.ToolCacheMode) + + require.NoError(t, os.WriteFile(path, []byte("runner:\n tool_cache_mode: everyone\n"), 0o600)) + _, err = LoadDefault(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "tool_cache_mode") +} + func TestLoadDefault_DefaultsWorkdirCleanupAge(t *testing.T) { cfg, err := LoadDefault("") require.NoError(t, err)