enhance: add runner.tool_cache_mode and default it to none (#1171)

The current shared tools cache is not concurrency-safe, e.g. multiple jobs can write and corrupt it, for example `setup-go` with explicit go version under concurrency reliably corrupts the tool cache and fails all jobs.

This adds a new `runner.tool_cache_mode` (and `--tool-cache-mode` exec option) option which defaults to unshared tools cache:

- `none` mounts nothing, so a job uses what its image ships there and discards what it installs
- `shared` keeps the single volume every job reuses, and warns when `runner.capacity` is above 1

Under `none` effective tool cache can only come from the image or host, which is the same as it is on GitHub Actions which ships many preinstalled tools in its fat VM images.

Co-authored-by: bircni <bircni@icloud.com>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1171
Reviewed-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>

README.md

@@ -158,6 +158,32 @@ An edit keeps the comments and the key order of the file. Indentation becomes tw
 
 `config get`, `set`, `add` and `remove` use `config.yaml` (or `config.yml`) from the working directory, then from the directory of the binary, and print their choice to stderr. `config init` writes `config.yaml` in the working directory, and refuses to overwrite an existing config without `--force`. Pass `-c` for another path.
 
+#### Tool cache
+
+Setup actions like `setup-go` install tools into `RUNNER_TOOL_CACHE`, which is `/opt/hostedtoolcache` inside a job. `runner.tool_cache_mode` selects what backs it:
+
+| Mode | Tool cache | Trade-off |
+| --- | --- | --- |
+| `none` (default) | Per job, provided by the job image | A version the image lacks is downloaded in every job |
+| `shared` | One volume reused by every job | Two jobs writing the same tool version at once corrupt it, so use it only with `runner.capacity: 1` |
+
+With `none`, tools must come from the job image. Install them into `/opt/hostedtoolcache/<tool>/<version>/<arch>`, with an empty `<arch>.complete` file next to the directory:
+
+```dockerfile
+RUN GO=$(curl -fsSL 'https://go.dev/dl/?mode=json' | grep -oP '"version": "\Kgo1\.26\.[0-9]*' | head -1); \
+  DIR="/opt/hostedtoolcache/go/${GO#go}/x64" && \
+  mkdir -p "$(dirname "$DIR")" && \
+  curl -fsSL "https://dl.google.com/go/${GO}.linux-amd64.tar.gz" | tar -xz -C /tmp && \
+  mv /tmp/go "$DIR" && \
+  touch "${DIR}.complete"
+```
+
+A workflow requesting a minor version, `go-version: "1.26"`, resolves to the newest matching version in the cache, so a patch update in the image still hits it.
+
+Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavour is the one that ships tools in this layout.
+
+`gitea-runner exec` reads no config file and takes `--tool-cache-mode` instead, defaulting to `none`.
+
 #### Environment variables
 
 Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.

act/runner/run_context.go

@@ -229,14 +229,19 @@ func (rc *RunContext) containerDaemonSocket() string {
 	return rc.Config.ContainerDaemonSocket
 }
 
+const sharedToolCacheVolume = "act-toolcache" // mounted only when the tool cache is shared
+
 // validVolumes returns the volumes allowed on this job's containers: the configured base
 // plus the volumes the runner mounts automatically. It derives a fresh slice every call and
 // never mutates the shared Config (see containerDaemonSocket).
 func (rc *RunContext) validVolumes() []string {
 	name := rc.jobContainerName()
 	volumes := slices.Clone(rc.Config.ValidVolumes)
+	if rc.Config.SharedToolCache {
+		volumes = append(volumes, sharedToolCacheVolume)
+	}
 	// TODO: add a new configuration to control whether the docker daemon can be mounted
-	return append(volumes, "act-toolcache", name, name+"-env",
+	return append(volumes, name, name+"-env",
 		getDockerDaemonSocketMountPath(rc.containerDaemonSocket()))
 }
 
@@ -309,8 +314,10 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) {
 	if daemonSocket := rc.containerDaemonSocket(); daemonSocket != "-" && !claimed["/var/run/docker.sock"] {
 		binds = append(binds, getDockerDaemonSocketMountPath(daemonSocket)+":/var/run/docker.sock")
 	}
-	if toolCache := rc.toolCache(container.DefaultToolCache); !claimed[toolCache] {
-		mounts["act-toolcache"] = toolCache
+	if rc.Config.SharedToolCache {
+		if toolCache := rc.toolCache(container.DefaultToolCache); !claimed[toolCache] {
+			mounts[sharedToolCacheVolume] = toolCache
+		}
 	}
 	mounts[name+"-env"] = ext.GetActPath() // runner-internal, never overridable
 
@@ -360,7 +367,11 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
 		if err := os.MkdirAll(runnerTmp, 0o777); err != nil {
 			return err
 		}
-		toolCache := rc.toolCache(filepath.Join(cacheDir, "tool_cache"))
+		toolCacheParent := miscpath // per job, so cleanup removes it with the job
+		if rc.Config.SharedToolCache {
+			toolCacheParent = cacheDir
+		}
+		toolCache := rc.toolCache(filepath.Join(toolCacheParent, "tool_cache"))
 		if err := os.MkdirAll(toolCache, 0o777); err != nil {
 			return err
 		}

act/runner/run_context_test.go

@@ -502,7 +502,8 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
 						},
 					},
 					Config: &Config{
-						BindWorkdir: false,
+						BindWorkdir:     false,
+						SharedToolCache: true, // so OverridesToolCache has a mount to displace
 					},
 				}
 				rc.Run.JobID = "job1"
@@ -543,25 +544,44 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("ToolCacheMount", func(t *testing.T) {
+		rc := &RunContext{
+			Name:   "TestRCName",
+			Run:    &model.Run{Workflow: &model.Workflow{Name: "TestWorkflowName"}},
+			Config: &Config{},
+		}
+
+		_, gotmount := rc.GetBindsAndMounts()
+		assert.NotContains(t, gotmount, sharedToolCacheVolume)
+
+		rc.Config.SharedToolCache = true
+		_, gotmount = rc.GetBindsAndMounts()
+		assert.Equal(t, container.DefaultToolCache, gotmount[sharedToolCacheVolume])
+	})
 }
 
 func TestRunContextValidVolumes(t *testing.T) {
 	rc := &RunContext{
 		Name:   "job",
 		Run:    &model.Run{Workflow: &model.Workflow{Name: "wf"}},
-		Config: &Config{ValidVolumes: []string{"my-vol", "/host/path"}},
+		Config: &Config{ValidVolumes: []string{"my-vol", "/host/path"}, SharedToolCache: true},
 	}
 	name := rc.jobContainerName()
 
 	got := rc.validVolumes()
 
-	// the configured volumes plus the four the runner mounts automatically
-	assert.Subset(t, got, []string{"my-vol", "/host/path", "act-toolcache", name, name + "-env", "/var/run/docker.sock"})
+	// the configured volumes plus the ones the runner mounts automatically
+	assert.Subset(t, got, []string{"my-vol", "/host/path", sharedToolCacheVolume, name, name + "-env", "/var/run/docker.sock"})
 
 	// deriving the list must never mutate or grow the shared Config slice: parallel matrix
 	// combinations share one *Config, and the previous in-place append was a data race.
 	assert.Equal(t, []string{"my-vol", "/host/path"}, rc.Config.ValidVolumes)
 	assert.Len(t, rc.validVolumes(), len(got), "repeated calls must be stable, not accumulate")
+
+	// a job may mount it only while the runner does
+	rc.Config.SharedToolCache = false
+	assert.NotContains(t, rc.validVolumes(), sharedToolCacheVolume)
 }
 
 func TestCleanupJobResourcesCleansServicesWithoutJobContainer(t *testing.T) {

act/runner/runner.go

@@ -91,6 +91,7 @@ type Config struct {
 	PlatformPicker                    func(labels []string) string // platform picker, it will take precedence over Platforms if isn't nil
 	JobLoggerLevel                    *log.Level                   // the level of job logger
 	ValidVolumes                      []string                     // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
+	SharedToolCache                   bool                         // one tool cache for all jobs instead of one per job
 	InsecureSkipTLS                   bool                         // whether to skip verifying TLS certificate of the Gitea instance
 	MaxParallel                       int                          // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
 	AllocatePTY                       bool                         // allocate a pseudo-TTY for each step's process

internal/app/cmd/exec.go

@@ -13,6 +13,7 @@ import (
 	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -68,6 +69,15 @@ type executeArgs struct {
 	cacheHandler          *artifactcache.Handler
 	network               string
 	githubInstance        string
+	toolCacheMode         string
+}
+
+// sharedToolCache reports whether mode mounts one tool cache for every job.
+func sharedToolCache(mode string) (bool, error) {
+	if !slices.Contains(config.ToolCacheModes, mode) {
+		return false, fmt.Errorf("invalid --tool-cache-mode %q: must be one of %q", mode, config.ToolCacheModes)
+	}
+	return mode == config.ToolCacheModeShared, nil
 }
 
 // WorkflowsPath returns path to workflow file(s)
@@ -427,6 +437,11 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command
 		proxyEnv := run.JobProxyEnv(env, env["ACTIONS_CACHE_URL"], nil)
 		maps.Copy(env, proxyEnv)
 
+		shared, err := sharedToolCache(execArgs.toolCacheMode)
+		if err != nil {
+			return err
+		}
+
 		// run the plan
 		config := &runner.Config{
 			Workdir:               execArgs.Workdir(),
@@ -467,7 +482,8 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command
 			PlatformPicker: func(_ []string) string {
 				return execArgs.image
 			},
-			ValidVolumes: []string{"**"}, // All volumes are allowed for `exec` command
+			ValidVolumes:    []string{"**"}, // All volumes are allowed for `exec` command
+			SharedToolCache: shared,
 		}
 
 		config.Env["ACT_EXEC"] = "true"
@@ -543,6 +559,7 @@ func loadExecCmd(ctx context.Context) *cobra.Command {
 	execCmd.PersistentFlags().BoolVarP(&execArg.debug, "debug", "d", false, "enable debug log")
 	execCmd.PersistentFlags().BoolVarP(&execArg.dryrun, "dryrun", "n", false, "dryrun mode")
 	execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", "docker.gitea.com/runner-images:ubuntu-latest", "Docker image to use. Use \"-self-hosted\" to run directly on the host.")
+	execCmd.PersistentFlags().StringVarP(&execArg.toolCacheMode, "tool-cache-mode", "", config.ToolCacheModeNone, "What to mount at RUNNER_TOOL_CACHE: none, or shared to reuse one tool cache across runs")
 	execCmd.PersistentFlags().StringVarP(&execArg.network, "network", "", "", "Specify the network to which the container will connect")
 	execCmd.PersistentFlags().StringVarP(&execArg.githubInstance, "gitea-instance", "", "", "Gitea instance to use.")
 

internal/app/cmd/exec_test.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"testing"
 
+	"gitea.com/gitea/runner/internal/pkg/config"
+
 	"gitea.dev/actionslib/pkg/model"
 	"github.com/stretchr/testify/require"
 	"go.yaml.in/yaml/v4"
@@ -28,6 +30,19 @@ func TestExecuteArgsResolve(t *testing.T) {
 	require.Equal(t, abs, args.resolve(abs))
 }
 
+func TestSharedToolCache(t *testing.T) {
+	shared, err := sharedToolCache(config.ToolCacheModeShared)
+	require.NoError(t, err)
+	require.True(t, shared)
+
+	shared, err = sharedToolCache(config.ToolCacheModeNone)
+	require.NoError(t, err)
+	require.False(t, shared)
+
+	_, err = sharedToolCache("everyone")
+	require.ErrorContains(t, err, "tool-cache-mode")
+}
+
 func TestExecuteArgsPaths(t *testing.T) {
 	workdir := t.TempDir()
 	args := &executeArgs{

internal/app/run/runner.go

@@ -159,8 +159,8 @@ func (r *Runner) OnIdle(ctx context.Context) {
 	}
 	// Host mode: reclaim per-job scratch dirs left behind when HostEnvironment
 	// cleanup timed out (e.g. a delete stalled by an AV/EDR filter driver). They
-	// sit under the host workdir parent alongside the shared tool_cache, which
-	// the name match leaves untouched. No-op when no host-mode job ever ran.
+	// sit under the host workdir parent next to a shared tool_cache, which the name
+	// match leaves untouched. No-op when no host-mode job ever ran.
 	if hostRoot := filepath.FromSlash(r.cfg.Host.WorkdirParent); hostRoot != "" {
 		r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir)
 	}
@@ -219,7 +219,7 @@ func isTaskIDDir(name string) bool {
 // isHostScratchDir reports whether name is a per-job host-mode scratch dir:
 // hex.EncodeToString of 8 random bytes, i.e. exactly 16 lowercase hex chars
 // (see startHostEnvironment in act/runner/run_context.go). The narrow match
-// leaves the sibling shared "tool_cache" dir and any operator data untouched.
+// leaves a sibling shared "tool_cache" dir and any operator data untouched.
 func isHostScratchDir(name string) bool {
 	if len(name) != 16 {
 		return false
@@ -522,6 +522,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		JobCompletedHook:                  r.cfg.Runner.Hooks.JobCompleted,
 		Vars:                              task.Vars,
 		ValidVolumes:                      r.cfg.Container.ValidVolumes,
+		SharedToolCache:                   r.cfg.Runner.ToolCacheMode == config.ToolCacheModeShared,
 		InsecureSkipTLS:                   r.cfg.Runner.Insecure,
 		RunnerName:                        r.name,
 	}

internal/pkg/config/config.example.yaml

@@ -93,6 +93,12 @@ runner:
   # terminal; tools like `docker build` emit redrawing progress frames into the captured log
   # when a TTY is present.
   #allocate_pty: false
+  # What to mount at RUNNER_TOOL_CACHE (/opt/hostedtoolcache), where setup actions install tools:
+  #   none:   nothing. A docker job sees what its image ships there, a host job an empty dir, and
+  #           either way what it installs is gone when the job ends.
+  #   shared: one tool cache every job reuses. Two jobs writing the same tool version at once
+  #           corrupt it, so use it only with capacity 1.
+  #tool_cache_mode: none
   # Optional executable on the host, run once after each task's built-in cleanup
   # (post-steps, container teardown, bind-workdir removal). Additive only.
   #
@@ -195,7 +201,7 @@ container:
   #privileged: false
   # Any other options to be used when the container is started, for example:
   #   options: --add-host=my.gitea.url:host-gateway
-  # A volume declared here replaces the one the runner mounts on the same container path, so the
+  # A volume declared here replaces the one the runner would mount on the same container path, so the
   # tool cache can be kept on the host. Its source must also be allowed by valid_volumes below:
   #   options: --volume /host/toolcache:/opt/hostedtoolcache
   #options:

internal/pkg/config/config.go

@@ -10,6 +10,7 @@ import (
 	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 
@@ -60,6 +61,7 @@ type Runner struct {
 	ActionShallowClone    *bool             `yaml:"action_shallow_clone"`     // ActionShallowClone fetches only the requested ref of an action repository at depth 1 instead of cloning every branch's full history. It is a pointer to distinguish between false and not set; if not set, it defaults to true.
 	SetActEnv             *bool             `yaml:"set_act_env"`              // SetActEnv controls whether the ACT=true environment variable is injected into jobs. It is a pointer to distinguish between false and not set; if not set, it defaults to true. Set it to false so workflows gated on `if: ${{ !env.ACT }}` behave like on GitHub.
 	AllocatePTY           bool              `yaml:"allocate_pty"`             // AllocatePTY allocates a pseudo-TTY for each step's process. Default is false, matching GitHub's actions/runner. Enable only for jobs that need an interactive terminal; tools like docker build emit redrawing progress frames into the captured log when a TTY is present. Applies to both host and docker backends.
+	ToolCacheMode         string            `yaml:"tool_cache_mode"`          // ToolCacheMode is what the runner mounts at RUNNER_TOOL_CACHE on both backends: ToolCacheModeNone or ToolCacheModeShared.
 	PostTaskScript        string            `yaml:"post_task_script"`         // PostTaskScript is the path to an executable script run on the host after each task's cleanup completes. Empty disables the hook. On Windows use .exe/.bat/.cmd; PowerShell (.ps1) is not supported yet as the configured path.
 	PostTaskScriptTimeout time.Duration     `yaml:"post_task_script_timeout"` // PostTaskScriptTimeout caps how long the post-task script may run. Default is 5m when post_task_script is set.
 	Hooks                 RunnerHooks       `yaml:"hooks"`                    // Hooks are scripts run inside the job environment around the job's steps.
@@ -143,6 +145,14 @@ type Container struct {
 	ServiceReadyTimeout  time.Duration                 `yaml:"service_ready_timeout"`  // ServiceReadyTimeout bounds how long a job waits for a service container that declares a healthcheck to report healthy. Negative disables waiting.
 }
 
+// Values of Runner.ToolCacheMode: the runner mounts no tool cache, or one that every job reuses.
+const (
+	ToolCacheModeNone   = "none"
+	ToolCacheModeShared = "shared"
+)
+
+var ToolCacheModes = []string{ToolCacheModeNone, ToolCacheModeShared}
+
 type ContainerNetworkCreateOptions struct {
 	EnableIPv4 *bool `yaml:"enable_ipv4"` // Enable or disable IPv4 for the network (true for docker by default)
 	EnableIPv6 *bool `yaml:"enable_ipv6"` // Enable or disable IPv6 for the network (false for docker by default)
@@ -257,6 +267,12 @@ func LoadDefault(file string) (*Config, error) {
 	if cfg.Container.WorkdirParent == "" {
 		cfg.Container.WorkdirParent = "workspace"
 	}
+	if cfg.Runner.ToolCacheMode == "" {
+		cfg.Runner.ToolCacheMode = ToolCacheModeNone
+	}
+	if !slices.Contains(ToolCacheModes, cfg.Runner.ToolCacheMode) {
+		return nil, fmt.Errorf("invalid runner.tool_cache_mode %q: must be one of %q", cfg.Runner.ToolCacheMode, ToolCacheModes)
+	}
 	if cfg.Host.WorkdirParent == "" {
 		home, err := os.UserHomeDir()
 		if err != nil {
@@ -314,6 +330,10 @@ func LoadDefault(file string) (*Config, error) {
 	}
 
 	// Validate and fix invalid config combinations to prevent confusing behavior.
+	if cfg.Runner.ToolCacheMode == ToolCacheModeShared && cfg.Runner.Capacity > 1 {
+		log.Warnf("runner.tool_cache_mode %q with capacity %d: two jobs writing the same tool version at once corrupt it",
+			ToolCacheModeShared, cfg.Runner.Capacity)
+	}
 	if cfg.Runner.FetchIntervalMax < cfg.Runner.FetchInterval {
 		log.Warnf("fetch_interval_max (%v) is less than fetch_interval (%v), setting fetch_interval_max to fetch_interval",
 			cfg.Runner.FetchIntervalMax, cfg.Runner.FetchInterval)

internal/pkg/config/config_test.go

@@ -42,6 +42,23 @@ cache:
 	require.NoError(t, err)
 }
 
+func TestLoadDefault_ToolCacheMode(t *testing.T) {
+	cfg, err := LoadDefault("")
+	require.NoError(t, err)
+	assert.Equal(t, ToolCacheModeNone, cfg.Runner.ToolCacheMode)
+
+	path := filepath.Join(t.TempDir(), "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("runner:\n  tool_cache_mode: shared\n"), 0o600))
+	cfg, err = LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, ToolCacheModeShared, cfg.Runner.ToolCacheMode)
+
+	require.NoError(t, os.WriteFile(path, []byte("runner:\n  tool_cache_mode: everyone\n"), 0o600))
+	_, err = LoadDefault(path)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "tool_cache_mode")
+}
+
 func TestLoadDefault_DefaultsWorkdirCleanupAge(t *testing.T) {
 	cfg, err := LoadDefault("")
 	require.NoError(t, err)