From 612473e08ac44d992939eb083c8373ebda36c150 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Wed, 5 Aug 2026 16:34:41 -0400 Subject: [PATCH] feat: async pulls (#711) Signed-off-by: Adam Martin --- cmd/hauler/cli/cli.go | 2 +- cmd/hauler/cli/store.go | 34 + cmd/hauler/cli/store/add.go | 1092 ++++++-- cmd/hauler/cli/store/add_test.go | 2093 ++++++++++++++- cmd/hauler/cli/store/copy.go | 6 +- cmd/hauler/cli/store/copy_test.go | 123 +- cmd/hauler/cli/store/extract_test.go | 10 +- cmd/hauler/cli/store/info_test.go | 20 +- cmd/hauler/cli/store/lifecycle_test.go | 4 +- cmd/hauler/cli/store/remove_test.go | 2 +- cmd/hauler/cli/store/save_test.go | 12 +- cmd/hauler/cli/store/sync.go | 955 +++++-- cmd/hauler/cli/store/sync_test.go | 2239 +++++++++++++++++ cmd/hauler/cli/store/testhelpers_test.go | 197 ++ cmd/hauler/cli/store_info_check_test.go | 4 +- go.mod | 8 +- internal/flags/add.go | 6 + internal/flags/cli.go | 2 +- internal/flags/concurrency.go | 104 + internal/flags/concurrency_test.go | 298 +++ internal/flags/ignore_errors.go | 24 + internal/flags/ignore_errors_test.go | 72 + internal/flags/no_progress_test.go | 82 + internal/flags/store.go | 30 +- internal/flags/store_blob_concurrency_test.go | 133 + internal/flags/sync.go | 6 + pkg/artifacts/file/cache.go | 89 + pkg/artifacts/file/cache_test.go | 218 ++ pkg/artifacts/file/context_test.go | 92 + pkg/artifacts/file/file.go | 57 +- pkg/artifacts/file/options.go | 11 + pkg/audit/audit.go | 11 +- pkg/audit/audit_test.go | 20 + pkg/consts/consts.go | 27 +- pkg/content/chart/chart.go | 18 - pkg/content/oci.go | 681 ++++- pkg/content/oci_test.go | 1423 +++++++++++ pkg/content/stats.go | 84 + pkg/content/stats_test.go | 148 ++ pkg/content/types.go | 20 +- pkg/content/types_test.go | 94 + pkg/cosign/cosign.go | 58 - pkg/cosign/verifier.go | 381 +++ pkg/cosign/verifier_test.go | 672 +++++ pkg/getter/https.go | 6 +- pkg/getter/https_test.go | 62 + pkg/layer/layer.go | 40 +- pkg/layer/layer_test.go | 43 + pkg/log/context.go | 28 + pkg/log/context_test.go | 66 + pkg/log/gating.go | 63 + pkg/log/gating_test.go | 198 ++ pkg/log/log.go | 18 +- pkg/log/log_test.go | 41 + pkg/log/logcapture.go | 15 + pkg/log/logcapture_test.go | 58 + pkg/log/progress.go | 322 +++ pkg/log/progress_test.go | 473 ++++ pkg/retry/retry.go | 26 +- pkg/retry/retry_test.go | 96 +- pkg/store/check_test.go | 2 +- pkg/store/stats.go | 36 + pkg/store/stats_test.go | 109 + pkg/store/store.go | 200 +- pkg/store/store_blob_concurrency_test.go | 115 + pkg/store/store_concurrency_test.go | 406 +++ pkg/store/store_test.go | 98 +- 67 files changed, 13386 insertions(+), 797 deletions(-) create mode 100644 internal/flags/concurrency.go create mode 100644 internal/flags/concurrency_test.go create mode 100644 internal/flags/ignore_errors.go create mode 100644 internal/flags/ignore_errors_test.go create mode 100644 internal/flags/no_progress_test.go create mode 100644 internal/flags/store_blob_concurrency_test.go create mode 100644 pkg/artifacts/file/cache.go create mode 100644 pkg/artifacts/file/cache_test.go create mode 100644 pkg/artifacts/file/context_test.go create mode 100644 pkg/content/stats.go create mode 100644 pkg/content/stats_test.go create mode 100644 pkg/content/types_test.go delete mode 100644 pkg/cosign/cosign.go create mode 100644 pkg/cosign/verifier.go create mode 100644 pkg/cosign/verifier_test.go create mode 100644 pkg/getter/https_test.go create mode 100644 pkg/layer/layer_test.go create mode 100644 pkg/log/context.go create mode 100644 pkg/log/context_test.go create mode 100644 pkg/log/gating.go create mode 100644 pkg/log/gating_test.go create mode 100644 pkg/log/log_test.go create mode 100644 pkg/log/logcapture_test.go create mode 100644 pkg/log/progress.go create mode 100644 pkg/log/progress_test.go create mode 100644 pkg/store/stats.go create mode 100644 pkg/store/stats_test.go create mode 100644 pkg/store/store_blob_concurrency_test.go create mode 100644 pkg/store/store_concurrency_test.go diff --git a/cmd/hauler/cli/cli.go b/cmd/hauler/cli/cli.go index 3830500..a57f279 100644 --- a/cmd/hauler/cli/cli.go +++ b/cmd/hauler/cli/cli.go @@ -18,7 +18,7 @@ func New(ctx context.Context, ro *flags.CliRootOpts) *cobra.Command { cmd := &cobra.Command{ Use: "hauler", Short: "Airgap Swiss Army Knife", - Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + " | " + consts.HaulerLogLevel + " | " + consts.HaulerAuditLevel + "\n Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.", + Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + " | " + consts.HaulerLogLevel + " | " + consts.HaulerAuditLevel + " | " + consts.HaulerConcurrency + " | " + consts.HaulerBlobConcurrency + "\n Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // check for log level env variable or flag if ro.LogLevel == "" { diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 8555243..3482999 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -94,6 +94,22 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman o.FileName = []string{} } } + + n, err := flags.ResolveConcurrency(cmd.Flags().Changed("concurrency"), o.Concurrency) + if err != nil { + return err + } + o.Concurrency = n + + // Must not be an unconditional assignment: rso.BlobConcurrency + // may already hold an explicit --blob-concurrency value, which + // has to win over anything derived from --concurrency. + bc, err := flags.SyncBlobConcurrency(rso.BlobConcurrency, n) + if err != nil { + return err + } + rso.BlobConcurrency = bc + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -415,6 +431,24 @@ func addStoreAddChart(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co # fetch remote helm chart and rewrite path hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --rewrite custom-path/hauler-chart:latest`, Args: cobra.ExactArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + n, err := flags.ResolveConcurrency(cmd.Flags().Changed("concurrency"), o.Concurrency) + if err != nil { + return err + } + o.Concurrency = n + + // Must not be an unconditional assignment: rso.BlobConcurrency + // may already hold an explicit --blob-concurrency value, which + // has to win over anything derived from --concurrency. + bc, err := flags.SyncBlobConcurrency(rso.BlobConcurrency, n) + if err != nil { + return err + } + rso.BlobConcurrency = bc + + return nil + }, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 7dabd92..e18cda8 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -3,15 +3,21 @@ package store import ( "bytes" "context" + "errors" "fmt" "os" "path/filepath" "regexp" "slices" "strings" + "sync" + "time" + "github.com/dustin/go-humanize" "github.com/google/go-containerregistry/pkg/name" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" + "helm.sh/helm/v4/pkg/action" "helm.sh/helm/v4/pkg/chart/common" commonutil "helm.sh/helm/v4/pkg/chart/common/util" helmchart "helm.sh/helm/v4/pkg/chart/v2" @@ -35,32 +41,81 @@ import ( ) func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string, ro *flags.CliRootOpts) error { + l := log.FromContext(ctx) + + // content.OCI's per-artifact index.json write only fsyncs once per + // indexCheckpointInterval (30s); the rest land in page cache. A single + // `store add file` can therefore return success without its index entry + // being durable, e.g. if a prior add against this store checkpointed + // recently enough that this run's write falls inside the window. Force + // one full fsync at the end so every `store add` subcommand leaves the + // store durable, matching SyncCmd's trailing SaveIndex(). + defer func() { + if err := s.OCI.SaveIndex(); err != nil { + l.Warnf("failed to save index durably after adding file: %v", err) + } + }() + cfg := v1.File{ Path: reference, } if len(o.Name) > 0 { cfg.Name = o.Name } + + l.Infof("adding file [%s] to the store", reference) + return storeFile(ctx, s, cfg, ro, o.StoreRootOpts) } func storeFile(ctx context.Context, s *store.Layout, fi v1.File, ro *flags.CliRootOpts, rso *flags.StoreRootOpts) error { l := log.FromContext(ctx) + start := time.Now() + ignoreErrors := flags.ShouldIgnoreErrors(ro) + + if err := ctx.Err(); err != nil { + log.BaseFromContext(ctx).Debugf("skipping file [%s]: %v", fi.Path, err) + return err + } + copts := getter.ClientOptions{ NameOverride: fi.Name, } - f := file.NewFile(fi.Path, file.WithClient(getter.NewClient(copts))) + f := file.NewFile(fi.Path, file.WithClient(getter.NewClient(copts)), file.WithContext(ctx)) ref, err := reference.NewTagged(f.Name(fi.Path), consts.DefaultTag) if err != nil { + if ignoreErrors { + log.BaseFromContext(ctx).Warnf("unable to derive a store reference for file [%s]: %v... skipping...", fi.Path, err) + return nil + } + log.BaseFromContext(ctx).Errorf("unable to derive a store reference for file [%s]: %v", fi.Path, err) return err } - l.Infof("adding file [%s] to the store as [%s]", fi.Path, ref.Name()) - desc, err := s.AddArtifact(ctx, f, ref.Name()) + log.BaseFromContext(ctx).Debugf("adding file [%s] to the store as [%s]", fi.Path, ref.Name()) + + var desc ocispec.Descriptor + err = retry.Operation(ctx, rso, ro, func() error { + var addErr error + desc, addErr = s.AddArtifact(ctx, f, ref.Name()) + return addErr + }) if err != nil { - return err + if ignoreErrors { + log.BaseFromContext(ctx).Warnf("unable to add file [%s] to store: %v... skipping...", fi.Path, err) + return nil + } else if errors.Is(err, context.Canceled) { + // Under errgroup.WithContext fail-fast (runFileJobs), one real + // failure cancels every other in-flight file's context -- see + // storeImage's identical branch for the full rationale. + log.BaseFromContext(ctx).Debugf("unable to add file [%s] to store: %v", fi.Path, err) + return err + } else { + log.BaseFromContext(ctx).Errorf("unable to add file [%s] to store: %v", fi.Path, err) + return err + } } resolvedPath := fi.Path @@ -97,7 +152,18 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1.File, ro *flags.CliRo l.Debugf("generated audit id of [none]") } - l.Infof("successfully added file [%s]", ref.Name()) + // stats.Layers is always 1 here: File.Layers() always returns exactly + // one layer (pkg/artifacts/file/file.go). f.Size() costs nothing extra + // on the success path -- compute() already ran (and memoized its result) + // inside the AddArtifact call above. + var stats *store.ImageStats + if size, sizeErr := f.Size(); sizeErr == nil { + stats = &store.ImageStats{} + stats.Layers.Store(1) + stats.Bytes.Store(size) + } + + log.BaseFromContext(ctx).Infof("%s", formatAddedLine(ref.Name(), stats, time.Since(start))) return nil } @@ -105,6 +171,20 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1.File, ro *flags.CliRo func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, reference string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) + // AddImage's per-artifact AddIndex calls (base image, then one per + // discovered cosign signature/attestation/SBOM/referrer) funnel through + // content.OCI's 30-second durable checkpoint, so only the very first + // call on a fresh store is fsync'd "for free" -- every call after that, + // including the final one, may land only in page cache. Force one full + // fsync at the end so `store add image` always ends durable, matching + // SyncCmd's trailing SaveIndex(). Blobs are unaffected: writeBlobOnce + // always fsyncs; only index.json entries are at risk. + defer func() { + if err := s.OCI.SaveIndex(); err != nil { + l.Warnf("failed to save index durably after adding image: %v", err) + } + }() + cfg := v1.Image{ Name: reference, Key: o.Key, @@ -127,48 +207,184 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re if o.Platform != "" { l.Warnf("--platform is ignored when --local is set: the Docker daemon stores only the host platform image") } + l.Infof("adding image [%s] from local Docker daemon to the store", cfg.Name) return storeLocalImage(ctx, s, cfg, rso, ro, o.Rewrite) } - // Check if the user provided a key. - if o.Key != "" { - // verify signature using the provided key. - err := cosign.VerifySignature(ctx, o.Key, o.Tlog, cfg.Name, rso, ro) - if err != nil { + pinnedDigest, err := verifyAddImage(ctx, o, cfg.Name, rso, ro) + if err != nil { + // Semantics and log shape mirror store sync's per-job rule; see + // logVerifyFailure's doc. + if propagate := logVerifyFailure(l, cfg.Name, err, flags.ShouldIgnoreErrors(ro)); propagate { return err } - l.Infof("signature verified for image [%s]", cfg.Name) - } else if o.CertIdentityRegexp != "" || o.CertIdentity != "" { - // verify signature using keyless details. - // Keyless (Fulcio) certificates expire after ~10 minutes, so the transparency - // log is always required to prove the cert was valid at signing time — ignore - // --use-tlog-verify for this path and always check tlog. - l.Infof("verifying keyless signature for [%s]", cfg.Name) - err := cosign.VerifyKeylessSignature(ctx, o.CertIdentity, o.CertIdentityRegexp, o.CertOidcIssuer, o.CertOidcIssuerRegexp, o.CertGithubWorkflowRepository, cfg.Name, rso, ro) - if err != nil { - return err - } - l.Infof("keyless signature verified for image [%s]", cfg.Name) } - return storeImage(ctx, s, cfg, o.Platform, o.ExcludeExtras, rso, ro, o.Rewrite) + l.Infof("adding image [%s] to the store", cfg.Name) + + // verified is true only when verification was both requested and + // succeeded: err is nil on success and non-nil on a failure that fell + // through to here under --ignore-errors, so err == nil already rules out + // the failed-but-stored-anyway case without checking ignoreErrors again. + verified := err == nil && !addImageVerifyConfig(o).Empty() + return storeImage(ctx, s, cfg, o.Platform, o.ExcludeExtras, rso, ro, o.Rewrite, pinnedDigest, verified) +} + +// addImageVerifyConfig collapses o's verification flags into a cosign.Config, +// which is empty exactly when the invocation asked for no verification. +// +// The branch mirrors the key-then-keyless precedence `store add image` has +// always applied rather than forwarding whatever flags happen to be set: a key +// wins and the identity flags are ignored. cosign.Config.validate rejects that +// pairing outright, so building the Config from the raw flags would turn an +// invocation that works today into a hard error. imageJob.verifyConfig makes +// the same choice for `store sync`. +func addImageVerifyConfig(o *flags.AddImageOpts) cosign.Config { + switch { + case o.Key != "": + return cosign.Config{Key: o.Key, Tlog: o.Tlog} + case o.CertIdentityRegexp != "" || o.CertIdentity != "": + return cosign.Config{ + CertIdentity: o.CertIdentity, + CertIdentityRegexp: o.CertIdentityRegexp, + CertOidcIssuer: o.CertOidcIssuer, + CertOidcIssuerRegexp: o.CertOidcIssuerRegexp, + CertGithubWorkflowRepository: o.CertGithubWorkflowRepository, + } + default: + return cosign.Config{} + } +} + +// verifyAddImage pins ref to a digest and verifies that exact digest, returning +// the digest for storeImage to fetch. An invocation that asked for no +// verification returns the empty digest, which leaves storeImage resolving the +// tag as before -- there is no window to close when nothing is checked, and an +// unconditional HEAD would add a round trip to the common unsigned case. +// +// Verifying the digest rather than the tag is the point: a tag verified and +// then re-resolved by storeImage can move between the two calls, so the bytes +// stored need not be the bytes checked. resolveAndVerify closes the same window +// for `store sync`. +// +// The two post-pin failure branches (cosign.NewVerifier, v.Verify) return the +// pinned digest alongside the error rather than "": under --ignore-errors +// AddImageCmd stores the image anyway, and it must store the exact bytes that +// were checked even though the check failed. The pre-pin branches (a bad +// reference, or the pin itself failing) have no digest to give back. +// +// One Verifier per call, built directly rather than through a cosign.Cache: +// there is exactly one image to check, so there is nothing to share it with. +// +// Every error it returns is a *verifyError, with the same stage strings +// resolveAndVerify uses for its equivalent branches, so logVerifyFailure +// reports add.go and sync.go failures in an identical shape. +func verifyAddImage(ctx context.Context, o *flags.AddImageOpts, ref string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) (string, error) { + l := log.FromContext(ctx) + + cfg := addImageVerifyConfig(o) + if cfg.Empty() { + return "", nil + } + + r, err := name.ParseReference(ref) + if err != nil { + return "", &verifyError{stage: "unable to parse image reference", err: err} + } + + pinned, err := pinDigest(ctx, r, rso, ro) + if err != nil { + return "", &verifyError{stage: "unable to resolve image digest", err: err} + } + + if cfg.Keyless() { + l.Infof("verifying keyless signature for [%s]", ref) + } + + v, err := cosign.NewVerifier(ctx, cfg, rso, ro) + if err != nil { + return pinned, &verifyError{stage: "unable to configure signature verification", err: err} + } + defer v.Close() + + if err := v.Verify(ctx, r.Context().Digest(pinned).Name()); err != nil { + return pinned, &verifyError{stage: "signature verification failed", err: err} + } + + if cfg.Keyless() { + l.Infof("✓ keyless signature verified for image [%s]", ref) + } else { + l.Infof("✓ signature verified for image [%s]", ref) + } + return pinned, nil +} + +func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, chartName string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { + l := log.FromContext(ctx) + + // Nothing in the chart path forces an fsync: every descriptor it adds -- + // the chart, its dependencies, and each discovered image -- lands through + // content.OCI's 30-second durable checkpoint, so a whole run can return + // success with index.json still only in page cache. This trailing + // SaveIndex is what makes the run durable, matching SyncCmd's. + defer func() { + if err := s.OCI.SaveIndex(); err != nil { + l.Warnf("failed to save index durably after adding chart: %v", err) + } + }() + + l.Infof("adding chart [%s] to the store", chartName) + + // The job owns its *action.ChartPathOptions rather than the caller's, so + // the invariant every chartJob holds -- no two jobs share a pointee -- + // is true of dependency-derived jobs and this one alike. + chartOpts := *o.ChartOpts + opts := *o + opts.ChartOpts = &chartOpts + + job := chartJob{ + cfg: v1.Chart{ + Name: chartName, + RepoURL: o.ChartOpts.RepoURL, + Version: o.ChartOpts.Version, + }, + opts: opts, + rewrite: o.Rewrite, + } + + return runChartJobs(ctx, s, []chartJob{job}, o.Concurrency, rso, ro, newProgressRenderer(o.NoProgress, ro.LogLevel)) +} + +// formatAddedLine formats the completion line logged after an artifact +// (image or file) is added to the store. When stats has at least one layer +// recorded, it includes layer count and human-readable total blob size; +// otherwise it falls back to an elapsed-only line and must never print +// "0 layers" (stats == nil covers storeLocalImage, whose s.AddLocalImage +// path never populates ImageStats). +func formatAddedLine(ref string, stats *store.ImageStats, elapsed time.Duration) string { + if stats != nil { + if layers := stats.Layers.Load(); layers > 0 { + unit := "layer" + if layers != 1 { + unit = "layers" + } + return fmt.Sprintf("✓ added %s (%d %s, %s, %.1fs)", ref, layers, unit, humanize.Bytes(uint64(stats.Bytes.Load())), elapsed.Seconds()) + } + } + return fmt.Sprintf("✓ added %s (%.1fs)", ref, elapsed.Seconds()) } func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error { l := log.FromContext(ctx) - if !ro.IgnoreErrors { - envVar := os.Getenv(consts.HaulerIgnoreErrors) - if envVar == "true" { - ro.IgnoreErrors = true - } - } + start := time.Now() + ignoreErrors := flags.ShouldIgnoreErrors(ro) - l.Infof("adding image [%s] from local Docker daemon to the store", i.Name) + l.Debugf("adding image [%s] from local Docker daemon to the store", i.Name) r, err := name.ParseReference(i.Name) if err != nil { - if ro.IgnoreErrors { + if ignoreErrors { l.Warnf("unable to parse image [%s]: %v... skipping...", i.Name, err) return nil } @@ -178,7 +394,7 @@ func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags. localDigest, err := s.AddLocalImage(ctx, r.Name()) if err != nil { - if ro.IgnoreErrors { + if ignoreErrors { l.Warnf("unable to add image [%s] from Docker daemon to store: %v... skipping...", r.Name(), err) return nil } @@ -234,46 +450,71 @@ func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags. l.Debugf("generated audit id of [none]") } - l.Infof("successfully added image [%s] from local Docker daemon", r.Name()) + l.Infof("%s", formatAddedLine(r.Name()+" from local Docker daemon", nil, time.Since(start))) return nil } -func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, excludeExtras bool, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error { +// storeImage fetches and stores image i. verified records whether +// verification was both requested and actually succeeded for the exact bytes +// being stored (pinnedDigest) -- callers compute it themselves rather than +// storeImage re-deriving it from i's verification fields, since under +// --ignore-errors those fields stay set even after a failed check and i alone +// can no longer distinguish "verified" from "verification requested." +func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, excludeExtras bool, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string, pinnedDigest string, verified bool) error { l := log.FromContext(ctx) - if !ro.IgnoreErrors { - envVar := os.Getenv(consts.HaulerIgnoreErrors) - if envVar == "true" { - ro.IgnoreErrors = true - } + start := time.Now() + ignoreErrors := flags.ShouldIgnoreErrors(ro) + + if err := ctx.Err(); err != nil { + log.BaseFromContext(ctx).Debugf("skipping image [%s]: %v", i.Name, err) + return err } - l.Infof("adding image [%s] to the store", i.Name) + log.BaseFromContext(ctx).Debugf("adding image [%s] to the store", i.Name) r, err := name.ParseReference(i.Name) if err != nil { - if ro.IgnoreErrors { - l.Warnf("unable to parse image [%s]: %v... skipping...", i.Name, err) + if ignoreErrors { + log.BaseFromContext(ctx).Warnf("unable to parse image [%s]: %v... skipping...", i.Name, err) return nil } else { - l.Errorf("unable to parse image [%s]: %v", i.Name, err) + log.BaseFromContext(ctx).Errorf("unable to parse image [%s]: %v", i.Name, err) return err } } - // fetch image along with any associated signatures and attestations + // fetch image along with any associated signatures and attestations. + // A fresh store.ImageStats is built inside the closure on every attempt, + // not once outside it, so a failed attempt's partial layer/byte counts + // aren't left for a retry to accumulate on top of. Only a successful + // attempt publishes its stats pointer to the outer variable, so + // formatAddedLine below reports the attempt that actually succeeded. var imageDigest string + var stats *store.ImageStats err = retry.Operation(ctx, rso, ro, func() error { + attemptStats := &store.ImageStats{} var addErr error - imageDigest, addErr = s.AddImage(ctx, r.Name(), platform, excludeExtras) + imageDigest, addErr = s.AddImage(store.WithImageStats(ctx, attemptStats), r.Name(), platform, excludeExtras, pinnedDigest) + if addErr == nil { + stats = attemptStats + } return addErr }) if err != nil { - if ro.IgnoreErrors { - l.Warnf("unable to add image [%s] to store: %v... skipping...", r.Name(), err) + if ignoreErrors { + log.BaseFromContext(ctx).Warnf("unable to add image [%s] to store: %v... skipping...", r.Name(), err) return nil + } else if errors.Is(err, context.Canceled) { + // Under errgroup.WithContext fail-fast (runImageJobs), one real + // failure cancels every other in-flight image's context. Logging + // this at ERROR would produce N-1 alarming lines for something + // that isn't the actual failure -- the real error is reported by + // whichever job's storeImage call hit it first. + log.BaseFromContext(ctx).Debugf("unable to add image [%s] to store: %v", r.Name(), err) + return err } else { - l.Errorf("unable to add image [%s] to store: %v", r.Name(), err) + log.BaseFromContext(ctx).Errorf("unable to add image [%s] to store: %v", r.Name(), err) return err } } @@ -298,7 +539,6 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin } } - verified := i.Key != "" || i.CertIdentity != "" || i.CertIdentityRegexp != "" if auditLevel(ro) != "none" { e := audit.Entry{ StoreID: s.StoreID, @@ -336,17 +576,11 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin l.Debugf("generated audit id of [none]") } - l.Infof("successfully added image [%s]", r.Name()) + log.BaseFromContext(ctx).Infof("%s", formatAddedLine(r.Name(), stats, time.Since(start))) return nil } func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference, rawRewrite string) error { - l := log.FromContext(ctx) - - if err := s.OCI.LoadIndex(); err != nil { - return fmt.Errorf("failed to load index: %w", err) - } - //TODO: improve string manipulation oldRefContext := oldRef.Context() newRefContext := newRef.Context() @@ -378,46 +612,29 @@ func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Referenc oldTotalReg := oldRegistry + "/" + oldTotal newTotalReg := newRegistry + "/" + newTotal - l.Infof("rewriting [%s] to [%s]", oldTotalReg, newTotalReg) + log.BaseFromContext(ctx).Infof("rewriting [%s] to [%s]", oldTotalReg, newTotalReg) //find and update reference - found := false - if err := s.OCI.Walk(func(k string, d ocispec.Descriptor) error { - if d.Annotations[ocispec.AnnotationRefName] == oldTotal && d.Annotations[consts.ContainerdImageNameKey] == oldTotalReg { - d.Annotations[ocispec.AnnotationRefName] = newTotal - d.Annotations[consts.ContainerdImageNameKey] = newTotalReg - found = true - } - return nil - }); err != nil { + matched, err := s.OCI.UpdateAnnotations( + func(d ocispec.Descriptor) bool { + return d.Annotations[ocispec.AnnotationRefName] == oldTotal && d.Annotations[consts.ContainerdImageNameKey] == oldTotalReg + }, + func(a map[string]string) { + a[ocispec.AnnotationRefName] = newTotal + a[consts.ContainerdImageNameKey] = newTotalReg + }, + ) + if err != nil { return err } - if !found { + if matched == 0 { return fmt.Errorf("could not find image [%s] in store", oldRef.Name()) } - return s.OCI.SaveIndex() - + return nil } -func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, chartName string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { - cfg := v1.Chart{ - Name: chartName, - RepoURL: o.ChartOpts.RepoURL, - Version: o.ChartOpts.Version, - } - - rewrite := "" - if o.Rewrite != "" { - rewrite = o.Rewrite - } - return storeChart(ctx, s, cfg, o, rso, ro, rewrite) -} - -// unexported type for the context key to avoid collisions -type isSubchartKey struct{} - // imageregex parses image references starting with "image:" and with optional spaces or optional quotes var imageRegex = regexp.MustCompile(`(?m)^[ \t-]*image:[ \t]*['"]?([^\s'"#]+)`) @@ -526,47 +743,413 @@ func applyDefaultRegistry(img string, defaultRegistry string) (string, error) { return newRef.Name(), nil } -func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags.AddChartOpts, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error { +// chartJob is the fully-resolved set of inputs needed to fetch and store a +// single chart; see resolveChartJobs. +type chartJob struct { + cfg v1.Chart + opts flags.AddChartOpts // held by value; ChartOpts is allocated per job + rewrite string + parent string // "" for a top-level chart, else the parent chart's ref + depth int +} + +// resolveChartJobs produces one top-level chartJob per entry in charts, +// applying the Charts precedence rules against the manifest's annotations and +// the sync flags. manifestDir is the directory holding the manifest, which +// each chart's relative valuesFiles paths resolve against. It reads chart +// credentials from the environment via resolveChartCreds, so it can fail +// before any chart is fetched if a chart's usernameEnv/passwordEnv pair is +// misconfigured. +// +// The three precedence rules are not uniform. registry is CLI > annotation. +// excludeExtras is a one-way switch that any of the three sources can flip on +// and none can flip off. platform is per-chart > CLI > annotation: an explicit +// --platform is run-time intent and outranks manifest metadata. That last rule +// must stay identical to resolveImageJobs's, or a single `hauler store sync` +// run would pull a chart's discovered images for a different platform than the +// manifest's own Images section. +// +// Every job allocates its own *action.ChartPathOptions. flags.AddChartOpts +// holds that as a pointer, so copying the struct alone would leave sibling +// jobs sharing one pointee, and a per-chart RepoURL/Version write would land +// in every other chart's options. +func resolveChartJobs(o *flags.SyncOpts, annotations map[string]string, manifestDir string, charts []v1.Chart) ([]chartJob, error) { + registry := o.Registry + if registry == "" { + registry = annotations[consts.ImageAnnotationRegistry] + } + + jobs := make([]chartJob, 0, len(charts)) + for _, ch := range charts { + excludeExtras := o.ExcludeExtras + if !o.ExcludeExtras && annotations[consts.ImageAnnotationExcludeExtras] == "true" { + excludeExtras = true + } + if ch.ExcludeExtras { + excludeExtras = ch.ExcludeExtras + } + + platform := o.Platform + if o.Platform == "" && annotations[consts.ImageAnnotationPlatform] != "" { + platform = annotations[consts.ImageAnnotationPlatform] + } + if ch.Platform != "" { + platform = ch.Platform + } + + var valuesFiles []string + for _, path := range ch.ValuesFiles { + valuesFiles = append(valuesFiles, filepath.Join(manifestDir, path)) + } + + chartUsername, chartPassword, err := resolveChartCreds(ch) + if err != nil { + return nil, err + } + + jobs = append(jobs, chartJob{ + cfg: ch, + opts: flags.AddChartOpts{ + ChartOpts: &action.ChartPathOptions{ + RepoURL: ch.RepoURL, + Version: ch.Version, + Verify: ch.Verify, + Keyring: ch.Keyring, + Username: chartUsername, + Password: chartPassword, + PassCredentialsAll: ch.PassCredentialsAll, + CertFile: ch.CertFile, + KeyFile: ch.KeyFile, + CaFile: ch.CaFile, + InsecureSkipTLSVerify: ch.InsecureSkipTLSVerify, + PlainHTTP: ch.PlainHTTP, + }, + AddImages: ch.AddImages, + AddDependencies: ch.AddDependencies, + ExcludeExtras: excludeExtras, + Registry: registry, + Platform: platform, + ValuesFiles: valuesFiles, + }, + rewrite: ch.Rewrite, + }) + } + + return jobs, nil +} + +// dedupeImageJobs collapses repeat pulls out of a chart tree's discovered +// images, keeping each (name, platform, excludeExtras) triple's first +// occurrence and the order it was first seen in. Platform and excludeExtras +// belong in the key because they change what storeImage actually fetches, not +// just how it is recorded. +// +// Precondition: every job is chart-discovered -- local is false and no +// verification field (needsPubKey, key, needsKeyless, certIdentity*, tlog) is +// set. Those fields are outside the key, so passing manifest-derived jobs here +// would silently drop a local Docker-daemon pull in favor of a same-named +// remote one, or let an unverified job displace one that would have had its +// signature checked. Chart image discovery has no syntax for either, which is +// why the key stays narrow. +func dedupeImageJobs(jobs []imageJob) []imageJob { + type key struct { + name string + platform string + excludeExtras bool + } + + seen := make(map[key]struct{}, len(jobs)) + out := make([]imageJob, 0, len(jobs)) + for _, j := range jobs { + k := key{name: j.img.Name, platform: j.platform, excludeExtras: j.excludeExtras} + if _, dup := seen[k]; dup { + continue + } + seen[k] = struct{}{} + out = append(out, j) + } + + return out +} + +// maxChartDepth bounds how many dependency levels a chart tree is walked. +// The seen set already terminates repo-based cycles; this cap is what stops +// a graph whose every node is a *distinct* name|repo|version -- which the +// seen set cannot detect -- from walking forever. 10 is far past anything +// real: helm's own dependency trees bottom out within two or three levels. +const maxChartDepth = 10 + +// chartFetcher fetches one chart, returning the images and dependency charts +// it discovered. traverseChartLevels calls it concurrently. +type chartFetcher func(ctx context.Context, j chartJob) ([]imageJob, []chartJob, error) + +// traverseChartLevels walks a chart dependency graph breadth-first, fetching +// each level concurrently (bounded by concurrency) before deriving the next, +// and returns every image discovered along the way plus the number of charts +// fetched. +// +// Level-by-level rather than depth-first because a level is the widest set of +// charts provably independent of each other: a dependency's inputs are not +// known until its parent has been fetched and expanded. It is also what makes +// runChartJobs' single shared temp root a requirement rather than a +// convenience -- see its doc comment. +// +// Cycles are cut by a seen set keyed on name|repoURL|version. A chart already +// scheduled is never scheduled again, which also collapses the common case of +// several siblings depending on the same subchart. +// +// Error semantics match runRemoteImageJobsWith: each level is one +// errgroup.WithContext with SetLimit(concurrency), so under --ignore-errors +// fetch returns nil and the walk continues, and otherwise the first failure +// cancels its level's derived context and surfaces verbatim. +func traverseChartLevels(ctx context.Context, jobs []chartJob, concurrency int, fetch chartFetcher) ([]imageJob, int, error) { + if concurrency < 1 { + concurrency = 1 + } + + var ( + mu sync.Mutex + images []imageJob + deps []chartJob + fetched int + level = jobs + seen = make(map[string]bool, len(jobs)) + depth int + ) + + for _, j := range level { + seen[chartJobKey(j)] = true + } + + for ; len(level) > 0 && depth < maxChartDepth; depth++ { + deps = nil + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + for _, j := range level { + g.Go(func() error { + gotImages, gotDeps, err := fetch(gctx, j) + if err != nil { + return err + } + mu.Lock() + images = append(images, gotImages...) + deps = append(deps, gotDeps...) + fetched++ + mu.Unlock() + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, 0, err + } + + next := make([]chartJob, 0, len(deps)) + for _, d := range deps { + k := chartJobKey(d) + if seen[k] { + continue + } + seen[k] = true + next = append(next, d) + } + level = next + } + + // Truncation is silent otherwise: the charts still queued in level are + // simply dropped, and nothing downstream would show them missing. + if depth == maxChartDepth && len(level) > 0 { + log.FromContext(ctx).Warnf("stopping chart dependency traversal at depth [%d]... [%d] dependent chart(s) not fetched", maxChartDepth, len(level)) + } + + return images, fetched, nil +} + +// chartJobKey identifies a chart for the traversal's seen set. Name alone is +// not enough: the same chart name at two versions, or from two repositories, +// is two genuinely different pulls. +func chartJobKey(j chartJob) string { + return j.cfg.Name + "|" + j.cfg.RepoURL + "|" + j.cfg.Version +} + +// runChartJobs stores every chart in jobs and everything the tree below them +// references: dependency charts are walked breadth-first and concurrently by +// traverseChartLevels, then every image discovered along the way is pulled in +// one deduplicated pass by the same runner `store sync`'s Images documents +// use. Bounded by concurrency in both phases. +// +// One temp root serves the whole call, and its lifetime is a hard constraint +// rather than a convenience. A file:// dependency is named by a path *inside* +// its parent's expanded directory, and BFS fetches it only after the parent's +// entire level has returned -- so a per-chart temp dir removed when fetchChart +// returns would delete that path out from under the child. Charts are capped +// at ~1MB, so holding every expansion until the call ends is cheap. +// +// Only the chart traversal runs inside log.CaptureOutput. Helm's downloader can +// still print to stdout from a transitive dependency, and debug=true routes +// that to DEBUG -- silent at the default level, which is what the per-call +// os.Stdout swap deleted from pkg/content/chart achieved. The image phase is +// outside it because the capture is scoped to what Helm prints; hauler's own +// log lines never reach it either way, since log.NewLogger binds its writer at +// construction (pkg/log/log.go). +func runChartJobs(ctx context.Context, s *store.Layout, jobs []chartJob, concurrency int, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, progress *log.Renderer) error { + if len(jobs) == 0 { + return nil + } + if concurrency < 1 { + concurrency = 1 + } + l := log.FromContext(ctx) - // subchart logging prefix - isSubchart := ctx.Value(isSubchartKey{}) == true - prefix := "" - if isSubchart { - prefix = " ↳ " + tempOverride := rso.TempOverride + if tempOverride == "" { + tempOverride = os.Getenv(consts.HaulerTempDir) } - - // normalize chart name for logging - displayName := cfg.Name - if strings.Contains(cfg.Name, string(os.PathSeparator)) { - displayName = filepath.Base(cfg.Name) - } - l.Infof("%sadding chart [%s] to the store", prefix, displayName) - - opts.ChartOpts.RepoURL = cfg.RepoURL - opts.ChartOpts.Version = cfg.Version - - chrt, err := chart.NewChart(cfg.Name, opts.ChartOpts) + tempRoot, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName) if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tempRoot) + + // baseLogger is the logger every job's per-chart logger is derived from -- + // see runImageJobs's identical baseLogger for the full rationale. It is + // built before CaptureOutput swaps os.Stdout so NewLogger's color decision + // still sees the real terminal, and the Renderer holds the real *os.File + // either way, so progress rows keep reaching the terminal rather than the + // capture pipe. + baseLogger := l + if progress != nil { + baseLogger = log.NewLogger(progress) + progress.Start() + defer progress.Stop() + } + + fetch := func(fctx context.Context, j chartJob) ([]imageJob, []chartJob, error) { + name := chartDisplayName(j.cfg.Name) + // Invoked from inside traverseChartLevels' errgroup goroutine, so + // Began lands after semaphore acquisition -- see runRemoteImageJobsWith. + if progress != nil { + progress.Began(name) + } + fields := log.Fields{"chart": name} + if j.parent != "" { + fields["parent"] = j.parent + } + jctx := baseLogger.With(fields).WithContext(fctx) + jctx = log.WithBaseLogger(jctx, baseLogger) + images, deps, err := fetchChart(jctx, s, j, tempRoot, rso, ro) + if progress != nil { + progress.Finished(name) + } + return images, deps, err + } + + // CaptureOutput wraps a non-nil fn error as "function execution failed: + // %w". Carrying the walk's error out in a variable keeps the one real + // failure reaching the caller verbatim, as runImageJobs' does. + var ( + discovered []imageJob + charts int + walkErr error + ) + if err := log.CaptureOutput(baseLogger, true, func() error { + discovered, charts, walkErr = traverseChartLevels(baseLogger.WithContext(ctx), jobs, concurrency, fetch) + return nil + }); err != nil { return err } + if walkErr != nil { + return walkErr + } + + images := dedupeImageJobs(discovered) + if len(images) == 0 { + return nil + } + + baseLogger.Infof("identified %d unique image(s) across %d chart(s)", len(images), charts) + + // Chart-discovered images are always remote and never carry verification + // inputs, so they go straight to the remote runner -- no local Docker pass + // and no verify pass to run first. + return runRemoteImageJobsWith(ctx, s, images, concurrency, rso, ro, progress, baseLogger) +} + +// chartDisplayName is the short name used for a chart's progress row and +// "chart=" log field. A file:// dependency's job is named by a filesystem +// path into its parent's expansion, which is neither short nor stable across +// runs, so those collapse to the basename. +func chartDisplayName(name string) string { + if strings.Contains(name, string(os.PathSeparator)) { + return filepath.Base(name) + } + return name +} + +// fetchChart stores one chart and reports what it references: the images +// discovered inside it when --add-images, and the dependency charts to walk +// next when --add-dependencies. It does not recurse -- traverseChartLevels +// owns the walk -- and it never writes through j.opts.ChartOpts, which is what +// lets sibling jobs run concurrently. +// +// tempRoot is runChartJobs' shared root. The chart expands into a fresh +// subdirectory of it, and both dependency branches resolve against that +// subdirectory, so it must outlive this call; see runChartJobs. +func fetchChart(ctx context.Context, s *store.Layout, j chartJob, tempRoot string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) ([]imageJob, []chartJob, error) { + l := log.FromContext(ctx) + + start := time.Now() + ignoreErrors := flags.ShouldIgnoreErrors(ro) + displayName := chartDisplayName(j.cfg.Name) + + if err := ctx.Err(); err != nil { + log.BaseFromContext(ctx).Debugf("skipping chart [%s]: %v", displayName, err) + return nil, nil, err + } + + log.BaseFromContext(ctx).Debugf("adding chart [%s] to the store", displayName) + + chrt, err := chart.NewChart(j.cfg.Name, j.opts.ChartOpts) + if err != nil { + return nil, nil, err + } c, err := chrt.Load() if err != nil { - return err + return nil, nil, err } ref, err := reference.NewTagged(c.Name(), c.Metadata.Version) if err != nil { - return err + return nil, nil, err } - chartDesc, err := s.AddArtifact(ctx, chrt, ref.Name()) + var chartDesc ocispec.Descriptor + err = retry.Operation(ctx, rso, ro, func() error { + var addErr error + chartDesc, addErr = s.AddArtifact(ctx, chrt, ref.Name()) + return addErr + }) if err != nil { - return err + if ignoreErrors { + log.BaseFromContext(ctx).Warnf("unable to add chart [%s] to store: %v... skipping...", ref.Name(), err) + return nil, nil, nil + } else if errors.Is(err, context.Canceled) { + // Under traverseChartLevels' fail-fast errgroup, one real failure + // cancels every other in-flight chart's context -- see + // storeImage's identical branch for the full rationale. + log.BaseFromContext(ctx).Debugf("unable to add chart [%s] to store: %v", ref.Name(), err) + return nil, nil, err + } + log.BaseFromContext(ctx).Errorf("unable to add chart [%s] to store: %v", ref.Name(), err) + return nil, nil, err } - if err := s.OCI.SaveIndex(); err != nil { - return err + + if j.rewrite != "" { + if err := rewriteChartReference(ctx, s, ref, j.rewrite); err != nil { + return nil, nil, err + } } if auditLevel(ro) != "none" { @@ -585,21 +1168,21 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. e.System = &sys e.Global = &g e.Flags = map[string]any{ - "repo": audit.SanitizeURL(cfg.RepoURL), + "repo": audit.SanitizeURL(j.cfg.RepoURL), "version": c.Metadata.Version, - "rewrite": rewrite, - "add-images": opts.AddImages, - "add-dependencies": opts.AddDependencies, - "exclude-extras": opts.ExcludeExtras, - "values": opts.ValuesFiles, - "platform": opts.Platform, - "registry": opts.Registry, - "kube-version": opts.KubeVersion, - "verify": opts.ChartOpts.Verify, - "insecure-skip-tls-verify": opts.ChartOpts.InsecureSkipTLSVerify, - "ca-file": opts.ChartOpts.CaFile, - "cert-file": opts.ChartOpts.CertFile, - "key-file": opts.ChartOpts.KeyFile, + "rewrite": j.rewrite, + "add-images": j.opts.AddImages, + "add-dependencies": j.opts.AddDependencies, + "exclude-extras": j.opts.ExcludeExtras, + "values": j.opts.ValuesFiles, + "platform": j.opts.Platform, + "registry": j.opts.Registry, + "kube-version": j.opts.KubeVersion, + "verify": j.opts.ChartOpts.Verify, + "insecure-skip-tls-verify": j.opts.ChartOpts.InsecureSkipTLSVerify, + "ca-file": j.opts.ChartOpts.CaFile, + "cert-file": j.opts.ChartOpts.CertFile, + "key-file": j.opts.ChartOpts.KeyFile, } } if err := audit.Append(ro.HaulerDir, e); err != nil { @@ -610,52 +1193,47 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. l.Debugf("generated audit id of [none]") } - l.Infof("%ssuccessfully added chart [%s:%s]", prefix, c.Name(), c.Metadata.Version) - - tempOverride := rso.TempOverride - if tempOverride == "" { - tempOverride = os.Getenv(consts.HaulerTempDir) - } - tempDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName) - if err != nil { - return fmt.Errorf("failed to create temp dir: %w", err) - } - defer os.RemoveAll(tempDir) - chartPath := chrt.Path() chartPathInfo, err := os.Stat(chartPath) if err != nil { - return fmt.Errorf("failed to stat chart path '%s': %w", chartPath, err) + return nil, nil, fmt.Errorf("failed to stat chart path '%s': %w", chartPath, err) } if !chartPathInfo.IsDir() { - l.Debugf("%sextracting chart archive [%s]", prefix, filepath.Base(chartPath)) - if err := util.ExpandFile(tempDir, chartPath); err != nil { - return fmt.Errorf("failed to extract chart: %w", err) + // A subdirectory per job, not /: two concurrent + // jobs can be the same chart at different versions, and both expand + // into a directory named for the chart. + expandDir, err := os.MkdirTemp(tempRoot, "chart-") + if err != nil { + return nil, nil, fmt.Errorf("failed to create temp dir: %w", err) + } + l.Debugf("extracting chart archive [%s]", filepath.Base(chartPath)) + if err := util.ExpandFile(expandDir, chartPath); err != nil { + return nil, nil, fmt.Errorf("failed to extract chart: %w", err) } // expanded chart should be in a directory matching the chart name - expectedChartDir := filepath.Join(tempDir, c.Name()) + expectedChartDir := filepath.Join(expandDir, c.Name()) if _, err := os.Stat(expectedChartDir); err != nil { - return fmt.Errorf("chart archive did not expand into expected directory '%s': %w", c.Name(), err) + return nil, nil, fmt.Errorf("chart archive did not expand into expected directory '%s': %w", c.Name(), err) } chartPath = expectedChartDir } - // add-images - if opts.AddImages { + var imageJobs []imageJob + if j.opts.AddImages { userValues := map[string]any{} - for _, valuesFile := range opts.ValuesFiles { + for _, valuesFile := range j.opts.ValuesFiles { l.Debugf("loading values for chart [%s]", valuesFile) valuesContent, err := os.ReadFile(valuesFile) if err != nil { - return fmt.Errorf("failed to read values file [%s]: %w", valuesFile, err) + return nil, nil, fmt.Errorf("failed to read values file [%s]: %w", valuesFile, err) } vals, err := loader.LoadValues(bytes.NewReader(valuesContent)) if err != nil { - return fmt.Errorf("failed to read helm values file [%s]: %w", valuesFile, err) + return nil, nil, fmt.Errorf("failed to read helm values file [%s]: %w", valuesFile, err) } userValues = loader.MergeMaps(userValues, vals) @@ -665,10 +1243,10 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. caps := common.DefaultCapabilities.Copy() // only parse and override if provided kube version - if opts.KubeVersion != "" { - kubeVersion, err := common.ParseKubeVersion(opts.KubeVersion) + if j.opts.KubeVersion != "" { + kubeVersion, err := common.ParseKubeVersion(j.opts.KubeVersion) if err != nil { - l.Warnf("%sinvalid kube-version [%s], using default kubernetes version", prefix, opts.KubeVersion) + l.Warnf("invalid kube-version [%s], using default kubernetes version", j.opts.KubeVersion) } else { caps.KubeVersion = *kubeVersion } @@ -679,18 +1257,18 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. // used later by --add-dependencies. renderChart, err := loader.Load(chrt.Path()) if err != nil { - return fmt.Errorf("failed to reload chart for image discovery: %w", err) + return nil, nil, fmt.Errorf("failed to reload chart for image discovery: %w", err) } // Match helm install/template: coalesce parent values into subcharts // (including dependency aliases) and honor conditions before rendering. if err := util.ProcessDependencies(renderChart, userValues); err != nil { - return fmt.Errorf("failed to process chart dependencies for image discovery: %w", err) + return nil, nil, fmt.Errorf("failed to process chart dependencies for image discovery: %w", err) } values, err := commonutil.ToRenderValues(renderChart, userValues, common.ReleaseOptions{Namespace: "hauler"}, caps) if err != nil { - return err + return nil, nil, err } // helper for normalization and deduping slices @@ -716,7 +1294,7 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. rendered, err := engine.Render(renderChart, values) if err != nil { // charts may fail due to values so still try helm chart annotations and lock - l.Warnf("%sfailed to render chart [%s]: %v", prefix, c.Name(), err) + l.Warnf("failed to render chart [%s]: %v", c.Name(), err) rendered = map[string]string{} } @@ -732,14 +1310,14 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. // parse helm chart annotations for images annotationImages, err = imagesFromChartAnnotations(c) if err != nil { - l.Warnf("%sfailed to parse helm chart annotation for [%s:%s]: %v", prefix, c.Name(), c.Metadata.Version, err) + l.Warnf("failed to parse helm chart annotation for [%s:%s]: %v", c.Name(), c.Metadata.Version, err) annotationImages = nil } // parse images lock files for images lockImages, err = imagesFromImagesLock(chartPath) if err != nil { - l.Warnf("%sfailed to parse images lock: %v", prefix, err) + l.Warnf("failed to parse images lock: %v", err) lockImages = nil } @@ -752,147 +1330,147 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. images := append(append(templateImages, annotationImages...), lockImages...) images = normalizeUniq(images) - l.Debugf("%simage references identified for helm template: [%d] image(s)", prefix, len(templateImages)) - - l.Debugf("%simage references identified for helm chart annotations: [%d] image(s)", prefix, len(annotationImages)) - - l.Debugf("%simage references identified for helm image lock file: [%d] image(s)", prefix, len(lockImages)) - l.Debugf("%ssuccessfully parsed and deduped image references: [%d] image(s)", prefix, len(images)) - - l.Debugf("%ssuccessfully parsed image references %v", prefix, images) + l.Debugf("image references identified for helm template: [%d] image(s)", len(templateImages)) + l.Debugf("image references identified for helm chart annotations: [%d] image(s)", len(annotationImages)) + l.Debugf("image references identified for helm image lock file: [%d] image(s)", len(lockImages)) + l.Debugf("successfully parsed and deduped image references: [%d] image(s)", len(images)) + l.Debugf("successfully parsed image references %v", images) if len(images) > 0 { - l.Infof("%s ↳ identified [%d] image(s) in [%s:%s]", prefix, len(images), c.Name(), c.Metadata.Version) + log.BaseFromContext(ctx).Infof("identified [%d] image(s) in [%s:%s]", len(images), c.Name(), c.Metadata.Version) } for _, image := range images { - image, err := applyDefaultRegistry(image, opts.Registry) + relocated, err := applyDefaultRegistry(image, j.opts.Registry) if err != nil { - if ro.IgnoreErrors { - l.Warnf("%s ↳ unable to apply registry to image [%s]: %v... skipping...", prefix, image, err) + if ignoreErrors { + l.Warnf("unable to apply registry to image [%s]: %v... skipping...", image, err) continue } - return fmt.Errorf("unable to apply registry to image [%s]: %w", image, err) + return nil, nil, fmt.Errorf("unable to apply registry to image [%s]: %w", image, err) } - imgCfg := v1.Image{Name: image} - if err := storeImage(ctx, s, imgCfg, opts.Platform, opts.ExcludeExtras, rso, ro, ""); err != nil { - if ro.IgnoreErrors { - l.Warnf("%s ↳ failed to store image [%s]: %v... skipping...", prefix, image, err) - continue - } - return fmt.Errorf("failed to store image [%s]: %w", image, err) - } - if err := s.OCI.LoadIndex(); err != nil { - return err - } - if err := s.OCI.SaveIndex(); err != nil { - return err - } + imageJobs = append(imageJobs, imageJob{ + img: v1.Image{Name: relocated}, + platform: j.opts.Platform, + excludeExtras: j.opts.ExcludeExtras, + }) } } - // add-dependencies - if opts.AddDependencies && len(c.Metadata.Dependencies) > 0 { + var deps []chartJob + if j.opts.AddDependencies { for _, dep := range c.Metadata.Dependencies { - l.Infof("%sadding dependent chart [%s:%s]", prefix, dep.Name, dep.Version) + l.Infof("adding dependent chart [%s:%s]", dep.Name, dep.Version) - depOpts := *opts + depOpts := j.opts depOpts.AddDependencies = true // Do not rediscover images on dependency charts in isolation. // Parent --add-images already renders the full tree (with alias // overrides and conditions) after ProcessDependencies. depOpts.AddImages = false - subCtx := context.WithValue(ctx, isSubchartKey{}, true) + + // depOpts is a struct copy, so it still points at the parent's + // *action.ChartPathOptions; the RepoURL/Version writes below would + // otherwise land in the parent's options -- a data race against + // whatever sibling job is reading them. Copying the value carries + // the parent's auth/TLS settings over without the aliasing. + depChartOpts := *j.opts.ChartOpts + depOpts.ChartOpts = &depChartOpts var depCfg v1.Chart - var err error - if strings.HasPrefix(dep.Repository, "file://") || dep.Repository == "" { + // A file:// subchart is already unpacked inside the parent's + // expansion, so it is addressed by path with nothing left to + // resolve from a repository. subchartPath := filepath.Join(chartPath, "charts", dep.Name) - depCfg = v1.Chart{Name: subchartPath, RepoURL: "", Version: ""} - depOpts.ChartOpts.RepoURL = "" - depOpts.ChartOpts.Version = "" - - err = storeChart(subCtx, s, depCfg, &depOpts, rso, ro, "") + depCfg = v1.Chart{Name: subchartPath} + depChartOpts.RepoURL = "" + depChartOpts.Version = "" } else { depCfg = v1.Chart{Name: dep.Name, RepoURL: dep.Repository, Version: dep.Version} - depOpts.ChartOpts.RepoURL = dep.Repository - depOpts.ChartOpts.Version = dep.Version - - err = storeChart(subCtx, s, depCfg, &depOpts, rso, ro, "") + depChartOpts.RepoURL = dep.Repository + depChartOpts.Version = dep.Version } - if err != nil { - if ro.IgnoreErrors { - l.Warnf("%s ↳ failed to add dependent chart [%s]: %v... skipping...", prefix, dep.Name, err) - } else { - l.Errorf("%s ↳ failed to add dependent chart [%s]: %v", prefix, dep.Name, err) - return err - } - } + deps = append(deps, chartJob{ + cfg: depCfg, + opts: depOpts, + parent: ref.Name(), + depth: j.depth + 1, + }) } } - // chart rewrite functionality - if rewrite != "" { - rewrite = strings.TrimPrefix(rewrite, "/") - newRef, err := name.ParseReference(rewrite) + // Chart.Layers() always returns exactly one layer, the chart archive + // itself. Re-deriving it costs a re-read (and, for an already-expanded + // directory chart, a re-tar) of at most ~1MB; anything unexpected falls + // back to nil stats and formatAddedLine's elapsed-only form. + var stats *store.ImageStats + if layers, layersErr := chrt.Layers(); layersErr == nil && len(layers) == 1 { + if size, sizeErr := layers[0].Size(); sizeErr == nil { + stats = &store.ImageStats{} + stats.Layers.Store(1) + stats.Bytes.Store(size) + } + } + + log.BaseFromContext(ctx).Infof("%s", formatAddedLine(ref.Name(), stats, time.Since(start))) + + return imageJobs, deps, nil +} + +// rewriteChartReference retags a stored chart's index entry from ref to +// rewrite. A rewrite that omits a tag inherits ref's. +func rewriteChartReference(ctx context.Context, s *store.Layout, ref name.Reference, rewrite string) error { + rewrite = strings.TrimPrefix(rewrite, "/") + newRef, err := name.ParseReference(rewrite) + if err != nil { + // error... don't continue with a bad reference + return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err) + } + + // if rewrite omits a tag... keep the existing tag + oldTag := ref.Identifier() + if tag, ok := ref.(name.Tag); ok { + oldTag = tag.TagStr() + } + if !strings.Contains(rewrite, ":") { + rewrite = strings.Join([]string{rewrite, oldTag}, ":") + newRef, err = name.ParseReference(rewrite) if err != nil { - // error... don't continue with a bad reference return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err) } + } - // if rewrite omits a tag... keep the existing tag - oldTag := ref.Identifier() - if tag, ok := ref.(name.Tag); ok { - oldTag = tag.TagStr() - } - if !strings.Contains(rewrite, ":") { - rewrite = strings.Join([]string{rewrite, oldTag}, ":") - newRef, err = name.ParseReference(rewrite) - if err != nil { - return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err) - } - } + // rename chart name in store + oldRepo := ref.Context().RepositoryStr() + newRepo := newRef.Context().RepositoryStr() + newTag := newRef.Identifier() + if tag, ok := newRef.(name.Tag); ok { + newTag = tag.TagStr() + } - // rename chart name in store - if err := s.OCI.LoadIndex(); err != nil { - return err - } + oldTotal := oldRepo + ":" + oldTag + newTotal := newRepo + ":" + newTag - oldRefContext := ref.Context() - newRefContext := newRef.Context() + log.BaseFromContext(ctx).Debugf("rewriting [%s] to [%s]", oldTotal, newTotal) - oldRepo := oldRefContext.RepositoryStr() - newRepo := newRefContext.RepositoryStr() - newTag := newRef.Identifier() - if tag, ok := newRef.(name.Tag); ok { - newTag = tag.TagStr() - } + matched, err := s.OCI.UpdateAnnotations( + func(d ocispec.Descriptor) bool { + return d.Annotations[ocispec.AnnotationRefName] == oldTotal + }, + func(a map[string]string) { + a[ocispec.AnnotationRefName] = newTotal + }, + ) + if err != nil { + return err + } - oldTotal := oldRepo + ":" + oldTag - newTotal := newRepo + ":" + newTag - - found := false - if err := s.OCI.Walk(func(k string, d ocispec.Descriptor) error { - if d.Annotations[ocispec.AnnotationRefName] == oldTotal { - d.Annotations[ocispec.AnnotationRefName] = newTotal - found = true - } - return nil - }); err != nil { - return err - } - - if !found { - return fmt.Errorf("could not find chart [%s] in store", ref.Name()) - } - - if err := s.OCI.SaveIndex(); err != nil { - return err - } + if matched == 0 { + return fmt.Errorf("could not find chart [%s] in store", ref.Name()) } return nil diff --git a/cmd/hauler/cli/store/add_test.go b/cmd/hauler/cli/store/add_test.go index 1917703..b7c2770 100644 --- a/cmd/hauler/cli/store/add_test.go +++ b/cmd/hauler/cli/store/add_test.go @@ -1,24 +1,39 @@ package store import ( + "bytes" + "context" + "errors" + "fmt" + "io" "net" + "net/http" "net/http/httptest" "os" "path/filepath" "reflect" + "regexp" "strings" + "sync" + "sync/atomic" "testing" + "time" + "github.com/dustin/go-humanize" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" "github.com/google/go-containerregistry/pkg/v1/remote" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/rs/zerolog" + "helm.sh/helm/v4/pkg/action" helmchart "helm.sh/helm/v4/pkg/chart/v2" "helm.sh/helm/v4/pkg/chart/v2/util" "hauler.dev/go/hauler/v2/internal/flags" v1 "hauler.dev/go/hauler/v2/pkg/apis/hauler.cattle.io/v1" "hauler.dev/go/hauler/v2/pkg/consts" + "hauler.dev/go/hauler/v2/pkg/log" + "hauler.dev/go/hauler/v2/pkg/store" ) // newLocalhostRegistry creates an in-memory OCI registry server listening on @@ -42,7 +57,7 @@ func newLocalhostRegistry(t *testing.T) (host string, remoteOpts []remote.Option } // chartTestdataDir is the relative path from cmd/hauler/cli/store/ to the -// top-level testdata directory, matching the convention in chart_test.go. +// top-level testdata directory, matching the convention in add_test.go. // It must remain relative so that url.ParseRequestURI rejects it (an absolute // path would be mistakenly treated as a URL by chart.NewChart's isUrl check). const chartTestdataDir = "../../../../testdata" @@ -237,7 +252,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -339,7 +354,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -420,7 +435,11 @@ func TestAddFileCmd(t *testing.T) { tmp.WriteString("raw content") //nolint:errcheck tmp.Close() - o := &flags.AddFileOpts{Name: "renamed.txt"} + // StoreRootOpts must be non-nil: storeFile now wraps its store write in + // retry.Operation, which dereferences rso.Retries. In production this is + // always set (see addStoreAddFile in cmd/hauler/cli/store.go); this test + // previously left it nil since storeFile never touched rso before. + o := &flags.AddFileOpts{Name: "renamed.txt", StoreRootOpts: defaultRootOpts(s.Root)} if err := AddFileCmd(ctx, o, s, tmp.Name(), defaultCliOpts()); err != nil { t.Fatalf("AddFileCmd: %v", err) } @@ -468,7 +487,7 @@ func TestStoreImage(t *testing.T) { ro := defaultCliOpts() ro.IgnoreErrors = tc.ignoreErrors - err := storeImage(ctx, s, v1.Image{Name: tc.imageName}, "", false, rso, ro, "") + err := storeImage(ctx, s, v1.Image{Name: tc.imageName}, "", false, rso, ro, "", "", false) if (err != nil) != tc.wantErr { t.Fatalf("error = %v, wantErr %v", err, tc.wantErr) } @@ -477,6 +496,22 @@ func TestStoreImage(t *testing.T) { } }) } + + t.Run("nonexistent image with HAULER_IGNORE_ERRORS env var returns nil and does not mutate ro", func(t *testing.T) { + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + t.Setenv(consts.HaulerIgnoreErrors, "true") + + err := storeImage(ctx, s, v1.Image{Name: host + "/nonexistent/image:missing"}, "", false, rso, ro, "", "", false) + if err != nil { + t.Fatalf("expected nil with HAULER_IGNORE_ERRORS=true, got: %v", err) + } + if ro.IgnoreErrors { + t.Fatal("expected ro.IgnoreErrors to remain false: storeImage must not mutate ro") + } + }) } func TestStoreImage_Rewrite(t *testing.T) { @@ -489,7 +524,7 @@ func TestStoreImage_Rewrite(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v1"}, "", false, rso, ro, "newrepo/img:v2") + err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v1"}, "", false, rso, ro, "newrepo/img:v2", "", false) if err != nil { t.Fatalf("storeImage with rewrite: %v", err) } @@ -502,7 +537,7 @@ func TestStoreImage_Rewrite(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v3"}, "", false, rso, ro, "newrepo/img") + err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v3"}, "", false, rso, ro, "newrepo/img", "", false) if err != nil { t.Fatalf("storeImage with tagless rewrite: %v", err) } @@ -522,7 +557,7 @@ func TestStoreImage_Rewrite(t *testing.T) { ro := defaultCliOpts() digestRef := host + "/src/repo@" + h.String() - err = storeImage(ctx, s, v1.Image{Name: digestRef}, "", false, rso, ro, "newrepo/img") + err = storeImage(ctx, s, v1.Image{Name: digestRef}, "", false, rso, ro, "newrepo/img", "", false) if err == nil { t.Fatal("expected error for digest ref rewrite without explicit tag, got nil") } @@ -541,7 +576,7 @@ func TestStoreImage_MultiArch(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage multi-arch index: %v", err) } // Full index (both platforms) must be stored as an index, not a single image. @@ -557,7 +592,7 @@ func TestStoreImage_PlatformFilter(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v2"}, "linux/amd64", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v2"}, "linux/amd64", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage with platform filter: %v", err) } // Platform filter resolves a single manifest from the index → stored as a single image. @@ -575,7 +610,7 @@ func TestStoreImage_CosignV2Artifacts(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage: %v", err) } assertArtifactKindInStore(t, s, "test/signed:v1", consts.KindAnnotationSigs) @@ -594,7 +629,7 @@ func TestStoreImage_CosignV3Referrer(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/image:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/image:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage: %v", err) } assertReferrerInStore(t, s, "test/image:v1") @@ -613,7 +648,7 @@ func TestStoreImage_ExcludeExtras(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v1"}, "", true, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v1"}, "", true, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage with excludeExtras: %v", err) } @@ -651,7 +686,7 @@ func TestStoreImage_ExcludeExtras(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/image:v1"}, "", true, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/image:v1"}, "", true, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage with excludeExtras: %v", err) } @@ -686,7 +721,7 @@ func TestStoreImage_ExcludeExtras(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v2"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v2"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage without excludeExtras: %v", err) } @@ -704,6 +739,7 @@ func TestAddChartCmd_LocalTgz(t *testing.T) { ro := defaultCliOpts() o := newAddChartOpts(chartTestdataDir, "") + o.Concurrency = consts.DefaultConcurrency if err := AddChartCmd(ctx, o, s, "rancher-cluster-templates-0.5.2.tgz", rso, ro); err != nil { t.Fatalf("AddChartCmd: %v", err) } @@ -721,12 +757,47 @@ func TestAddChartCmd_WithFileDep(t *testing.T) { ro := defaultCliOpts() o := newAddChartOpts(chartTestdataDir, "") + o.Concurrency = consts.DefaultConcurrency if err := AddChartCmd(ctx, o, s, "chart-with-file-dependency-chart-1.0.0.tgz", rso, ro); err != nil { t.Fatalf("AddChartCmd: %v", err) } assertArtifactInStore(t, s, "chart-with-file-dependency-chart") } +// TestAddChartCmd_DependencyTreeAtVaryingConcurrency proves AddChartCmd +// resolves a chart's dependency tree correctly regardless of o.Concurrency. +// It reuses the file-dependency fixture (a parent with two dependencies +// resolved BFS-style) at both concurrency=1 -- forcing runChartJobs' +// single-worker path -- and a higher value, and checks the store ends up +// identical either way, under -race. It does NOT prove concurrency is +// "honored": nothing here observes actual fan-out or parallelism, so a +// hardcoded consts.DefaultConcurrency that silently ignored o.Concurrency +// would still pass both subtests unchanged. +// TestTraverseChartLevels_BoundedFanOut is the test that actually asserts +// on the concurrency bound. +func TestAddChartCmd_DependencyTreeAtVaryingConcurrency(t *testing.T) { + for _, concurrency := range []int{1, 4} { + t.Run(fmt.Sprintf("concurrency=%d", concurrency), func(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + o := newAddChartOpts(chartTestdataDir, "") + o.AddDependencies = true + o.Concurrency = concurrency + + if err := AddChartCmd(ctx, o, s, "chart-with-file-dependency-chart-1.0.0.tgz", rso, ro); err != nil { + t.Fatalf("AddChartCmd concurrency=%d: %v", concurrency, err) + } + + assertArtifactInStore(t, s, "chart-with-file-dependency-chart:1.0.0") + assertArtifactInStore(t, s, "child:2.0.0") + assertArtifactInStore(t, s, "crds:0.0.1") + }) + } +} + func TestStoreChart_Rewrite(t *testing.T) { ctx := newTestContext(t) s := newTestStore(t) @@ -774,7 +845,7 @@ func seedChartWithImages(t *testing.T, dir string, images []string) string { return saved } -func TestStoreChart_AddImages_ExcludeExtras(t *testing.T) { +func TestRunChartJobs_AddImages_ExcludeExtras(t *testing.T) { ctx := newTestContext(t) host, rOpts := newLocalhostRegistry(t) @@ -792,13 +863,16 @@ func TestStoreChart_AddImages_ExcludeExtras(t *testing.T) { ro := defaultCliOpts() t.Run("excludeExtras=true suppresses sigs/atts/sboms for chart-discovered images", func(t *testing.T) { - o := &flags.AddChartOpts{ - ChartOpts: newAddChartOpts("", "").ChartOpts, - AddImages: true, - ExcludeExtras: true, + job := chartJob{ + cfg: v1.Chart{Name: tgzPath}, + opts: flags.AddChartOpts{ + ChartOpts: newAddChartOpts("", "").ChartOpts, + AddImages: true, + ExcludeExtras: true, + }, } - if err := storeChart(ctx, s, v1.Chart{Name: tgzPath}, o, rso, ro, ""); err != nil { - t.Fatalf("storeChart with ExcludeExtras: %v", err) + if err := runChartJobs(ctx, s, []chartJob{job}, 1, rso, ro, nil); err != nil { + t.Fatalf("runChartJobs with ExcludeExtras: %v", err) } // The chart itself is stored as an OCI image artifact. @@ -824,7 +898,7 @@ func TestStoreChart_AddImages_ExcludeExtras(t *testing.T) { }) } -func TestStoreChart_AddImages_IncludeExtras(t *testing.T) { +func TestRunChartJobs_AddImages_IncludeExtras(t *testing.T) { ctx := newTestContext(t) host, rOpts := newLocalhostRegistry(t) @@ -841,13 +915,16 @@ func TestStoreChart_AddImages_IncludeExtras(t *testing.T) { ro := defaultCliOpts() t.Run("excludeExtras=false includes sigs/atts/sboms for chart-discovered images", func(t *testing.T) { - o := &flags.AddChartOpts{ - ChartOpts: newAddChartOpts("", "").ChartOpts, - AddImages: true, - ExcludeExtras: false, + job := chartJob{ + cfg: v1.Chart{Name: tgzPath}, + opts: flags.AddChartOpts{ + ChartOpts: newAddChartOpts("", "").ChartOpts, + AddImages: true, + ExcludeExtras: false, + }, } - if err := storeChart(ctx, s, v1.Chart{Name: tgzPath}, o, rso, ro, ""); err != nil { - t.Fatalf("storeChart without ExcludeExtras: %v", err) + if err := runChartJobs(ctx, s, []chartJob{job}, 1, rso, ro, nil); err != nil { + t.Fatalf("runChartJobs without ExcludeExtras: %v", err) } assertArtifactKindInStore(t, s, "test/chart-image:v2", consts.KindAnnotationSigs) @@ -860,6 +937,74 @@ func TestStoreChart_AddImages_IncludeExtras(t *testing.T) { // --local flag validation tests // -------------------------------------------------------------------------- +// TestRunChartJobs_AddImages_IgnoreErrors_EnvVar verifies that a +// chart-discovered image failure is swallowed via HAULER_IGNORE_ERRORS alone, +// without --ignore-errors, and that the chart itself is still stored. +// Regression test: retry.Operation and storeImage used to mutate the shared +// ro.IgnoreErrors from the env var as a side effect, so every ro.IgnoreErrors +// read after the first one saw the env var. Now that storeImage and +// retry.Operation are pure reads via flags.ShouldIgnoreErrors, fetchChart's +// image-discovery loop and the image phase must each honor the env var on +// their own. +func TestRunChartJobs_AddImages_IgnoreErrors_EnvVar(t *testing.T) { + ctx := newTestContext(t) + + t.Run("applyDefaultRegistry failure via env var", func(t *testing.T) { + chartDir := t.TempDir() + // Uppercase letters and spaces are rejected by reference.Parse, so this + // only fails once opts.Registry is non-empty and applyDefaultRegistry + // actually parses the ref. + tgzPath := seedChartWithImages(t, chartDir, []string{"INVALID IMAGE REF !! ##"}) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + t.Setenv(consts.HaulerIgnoreErrors, "true") + + job := chartJob{ + cfg: v1.Chart{Name: tgzPath}, + opts: flags.AddChartOpts{ + ChartOpts: newAddChartOpts("", "").ChartOpts, + AddImages: true, + Registry: "registry.example.com", + }, + } + if err := runChartJobs(ctx, s, []chartJob{job}, 1, rso, ro, nil); err != nil { + t.Fatalf("expected nil with HAULER_IGNORE_ERRORS=true, got: %v", err) + } + if ro.IgnoreErrors { + t.Fatal("expected ro.IgnoreErrors to remain false: the chart path must not mutate ro") + } + assertArtifactInStore(t, s, "test-chart") + }) + + t.Run("storeImage failure via env var", func(t *testing.T) { + chartDir := t.TempDir() + // localhost:1 is a port that is never listening. + tgzPath := seedChartWithImages(t, chartDir, []string{"localhost:1/nonexistent/image:missing"}) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + t.Setenv(consts.HaulerIgnoreErrors, "true") + + job := chartJob{ + cfg: v1.Chart{Name: tgzPath}, + opts: flags.AddChartOpts{ + ChartOpts: newAddChartOpts("", "").ChartOpts, + AddImages: true, + }, + } + if err := runChartJobs(ctx, s, []chartJob{job}, 1, rso, ro, nil); err != nil { + t.Fatalf("expected nil with HAULER_IGNORE_ERRORS=true, got: %v", err) + } + if ro.IgnoreErrors { + t.Fatal("expected ro.IgnoreErrors to remain false: the chart path must not mutate ro") + } + assertArtifactInStore(t, s, "test-chart") + }) +} + func TestAddImageCmd_LocalFlagValidation(t *testing.T) { ctx := newTestContext(t) @@ -907,6 +1052,250 @@ func TestAddImageCmd_LocalFlagValidation(t *testing.T) { } } +// -------------------------------------------------------------------------- +// AddImageCmd verification tests +// -------------------------------------------------------------------------- + +// `store add image` must store the digest it verified. Verifying the tag and +// then letting storeImage resolve it a second time leaves a window in which the +// tag can move, so the bytes stored are not the bytes checked -- the window +// `store sync` closes in resolveAndVerify. +// +// The request log is what distinguishes the two: pinning first means the tag is +// resolved exactly once, by the command's own pin. Comparing the stored digest +// alone would pass either way, since nothing moves the tag mid-test. +func TestAddImageCmd_StoresTheDigestItVerified(t *testing.T) { + host, remoteOpts, rec := newRecordingRegistry(t) + img, keyPath := seedSignedImage(t, host, "signed", "v1", remoteOpts...) + want, err := img.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + rec.reset() // seeding itself HEADs the tag + + s := newTestStore(t) + o := &flags.AddImageOpts{Key: keyPath, ExcludeExtras: true} + if err := AddImageCmd(newTestContext(t), o, s, host+"/signed:v1", defaultRootOpts(s.Root), defaultCliOpts()); err != nil { + t.Fatalf("AddImageCmd: %v", err) + } + + got := storedDigest(t, s, "signed:v1") + if got == "" { + t.Fatalf("a validly signed image was not stored; requests:\n%v", rec.snapshot()) + } + if got != want.String() { + t.Fatalf("stored digest %s, want the verified digest %s", got, want) + } + if n := rec.countContaining("manifests/v1"); n != 1 { + t.Fatalf("the tag was resolved %d times, want exactly 1 (the command's own pin); verification and the pull are each resolving it\nrequests:\n%v", n, rec.snapshot()) + } +} + +// A signature that does not check out fails the command. `store sync` drops one +// image and carries on; `store add image` has only the one image, and returning +// success would leave the user believing an image had been verified when it was +// not stored at all. +func TestAddImageCmd_VerificationFailureFailsTheCommand(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + img := seedImage(t, host, "badsig", "v1", remoteOpts...) + // The signature manifest exists but carries no usable signature. + seedCosignV2Artifacts(t, host, "badsig", img, remoteOpts...) + + s := newTestStore(t) + o := &flags.AddImageOpts{Key: writeTestPubKey(t), ExcludeExtras: true} + if err := AddImageCmd(newTestContext(t), o, s, host+"/badsig:v1", defaultRootOpts(s.Root), defaultCliOpts()); err == nil { + t.Fatal("AddImageCmd returned nil for an image whose only signature is unusable") + } + if got := countArtifactsInStore(t, s); got != 0 { + t.Fatalf("store holds %d artifacts, want 0; the image that failed verification was stored anyway", got) + } +} + +// verifyAddImage must hand back the digest it pinned on every failure after +// the pin itself succeeded (verifier setup, signature check), so the caller +// can store exactly the bytes that were checked even though the check failed. +// A failure at or before the pin (malformed reference, unresolvable digest) +// has no digest to hand back and must return "". +func TestVerifyAddImage_ReturnsPinnedDigestOnPostPinFailureOnly(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + badsig := seedImage(t, host, "badsig", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "badsig", badsig, remoteOpts...) + badsigDigest, err := badsig.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + + tests := []struct { + name string + ref string + o *flags.AddImageOpts + wantPinned string // "" means the failure happened at or before the pin + }{ + { + name: "malformed reference", + ref: "NOT A REF", + o: &flags.AddImageOpts{Key: writeTestPubKey(t)}, + wantPinned: "", + }, + { + name: "unreachable registry", + ref: "127.0.0.1:1/absent/image:v1", + o: &flags.AddImageOpts{Key: writeTestPubKey(t)}, + wantPinned: "", + }, + { + name: "unreadable key", + ref: host + "/badsig:v1", + o: &flags.AddImageOpts{Key: filepath.Join(t.TempDir(), "missing.pub")}, + wantPinned: badsigDigest.String(), + }, + { + name: "signature that does not check out", + ref: host + "/badsig:v1", + o: &flags.AddImageOpts{Key: writeTestPubKey(t)}, + wantPinned: badsigDigest.String(), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rso, ro := defaultRootOpts(t.TempDir()), defaultCliOpts() + pinned, err := verifyAddImage(newTestContext(t), tt.o, tt.ref, rso, ro) + if err == nil { + t.Fatal("verifyAddImage succeeded") + } + if pinned != tt.wantPinned { + t.Fatalf("pinned digest = %q, want %q", pinned, tt.wantPinned) + } + }) + } +} + +// Without --ignore-errors, AddImageCmd's behavior is unchanged by this +// feature -- see TestAddImageCmd_VerificationFailureFailsTheCommand. With it, +// the command must log the failure at WARN and store the image anyway, +// unverified, using whatever digest verifyAddImage already pinned -- the same +// tradeoff `store sync` makes for one image out of many, now available to the +// single-image command too. +func TestAddImageCmd_IgnoreErrors_StoresUnverifiedImage(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + img := seedImage(t, host, "badsig", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "badsig", img, remoteOpts...) + ref := host + "/badsig:v1" + + s := newTestStore(t) + o := &flags.AddImageOpts{Key: writeTestPubKey(t), ExcludeExtras: true} + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + ro.IgnoreErrors = true + + var buf bytes.Buffer + l := log.NewLogger(&buf) + ctx := l.WithContext(context.Background()) + + if err := AddImageCmd(ctx, o, s, ref, rso, ro); err != nil { + t.Fatalf("AddImageCmd: %v", err) + } + + // The assertion that matters most: an image that failed verification is + // in the store anyway, unverified. + assertArtifactInStore(t, s, "badsig:v1") + + out := buf.String() + if !strings.Contains(out, "WRN") || !strings.Contains(out, ref) { + t.Fatalf("expected a WARN line naming %q, got:\n%s", ref, out) + } +} + +// AddImageCmd's audit entry must report whether verification actually +// succeeded, not merely whether it was requested -- see +// TestRunImageJobs_AuditVerifiedFlag for the sync.go counterpart and the bug +// this guards. +func TestAddImageCmd_AuditVerifiedFlag(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + tests := []struct { + name string + buildOpts func(t *testing.T) (*flags.AddImageOpts, string) + ignoreErrors bool + want bool + }{ + { + name: "not requested", + buildOpts: func(t *testing.T) (*flags.AddImageOpts, string) { + seedImage(t, host, "test/addaudit-unrequested", "v1", remoteOpts...) + return &flags.AddImageOpts{ExcludeExtras: true}, host + "/test/addaudit-unrequested:v1" + }, + want: false, + }, + { + name: "requested and passed", + buildOpts: func(t *testing.T) (*flags.AddImageOpts, string) { + _, keyPath := seedSignedImage(t, host, "test/addaudit-passed", "v1", remoteOpts...) + return &flags.AddImageOpts{Key: keyPath, ExcludeExtras: true}, host + "/test/addaudit-passed:v1" + }, + want: true, + }, + { + name: "requested and failed, stored anyway under --ignore-errors", + buildOpts: func(t *testing.T) (*flags.AddImageOpts, string) { + bad := seedImage(t, host, "test/addaudit-failed", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "test/addaudit-failed", bad, remoteOpts...) + return &flags.AddImageOpts{Key: writeTestPubKey(t), ExcludeExtras: true}, host + "/test/addaudit-failed:v1" + }, + ignoreErrors: true, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o, ref := tc.buildOpts(t) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + ro.AuditLevel = "verbose" + ro.IgnoreErrors = tc.ignoreErrors + ro.HaulerDir = t.TempDir() + + if err := AddImageCmd(newTestContext(t), o, s, ref, rso, ro); err != nil { + t.Fatalf("AddImageCmd: %v", err) + } + + flags := lastAuditEntryFlags(t, ro.HaulerDir) + got, ok := flags["verified"].(bool) + if !ok { + t.Fatalf("audit entry's flags[\"verified\"] is %v (%T), want a bool", flags["verified"], flags["verified"]) + } + if got != tc.want { + t.Errorf("flags[\"verified\"] = %v, want %v", got, tc.want) + } + }) + } +} + +// A key supplied alongside identity flags verifies against the key alone, as it +// always has. cosign.Config.validate rejects that pairing outright, so building +// the Config from the raw flags instead of from the branch this command selects +// would turn a working invocation into a hard error. +func TestAddImageCmd_KeyOutranksIdentityFlags(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + _, keyPath := seedSignedImage(t, host, "signed", "v1", remoteOpts...) + + s := newTestStore(t) + o := &flags.AddImageOpts{ + Key: keyPath, + CertIdentity: "someone@example.com", + CertIdentityRegexp: ".*", + ExcludeExtras: true, + } + if err := AddImageCmd(newTestContext(t), o, s, host+"/signed:v1", defaultRootOpts(s.Root), defaultCliOpts()); err != nil { + t.Fatalf("AddImageCmd rejected a key paired with identity flags: %v", err) + } + assertArtifactInStore(t, s, "signed:v1") +} + // -------------------------------------------------------------------------- // storeLocalImage unit tests // -------------------------------------------------------------------------- @@ -936,4 +1325,1650 @@ func TestStoreLocalImage_InvalidReference(t *testing.T) { t.Fatalf("expected nil with IgnoreErrors=true, got: %v", err) } }) + + t.Run("malformed reference with HAULER_IGNORE_ERRORS env var returns nil and does not mutate ro", func(t *testing.T) { + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + t.Setenv(consts.HaulerIgnoreErrors, "true") + + err := storeLocalImage(ctx, s, v1.Image{Name: "INVALID:::ref"}, rso, ro, "") + if err != nil { + t.Fatalf("expected nil with HAULER_IGNORE_ERRORS=true, got: %v", err) + } + if ro.IgnoreErrors { + t.Fatal("expected ro.IgnoreErrors to remain false: storeLocalImage must not mutate ro") + } + }) +} + +// -------------------------------------------------------------------------- +// Durable index save tests +// -------------------------------------------------------------------------- + +// add_durable_index_test.go covers the durability gap identified in the +// final cross-task review of the I/O tuning workstream: index.json's +// per-artifact fsync was replaced with a 30-second durable checkpoint +// (content.OCI.saveIndexCheckpointLocked), so nothing below the command +// entry points forces an fsync any more. Ending a run durably is now the +// job of each entry point's deferred SaveIndex() -- AddFileCmd, AddImageCmd, +// and AddChartCmd each have one. Without it a completed `hauler store add +// image`/`add file`/`add chart` could return success with its final +// index.json state sitting only in page cache, not fsynced -- a +// lost-index-entry risk on crash/power-loss (blobs are unaffected; +// writeBlobOnce always fsyncs). +// +// These tests observe durability via content.OCI.Stats().Snapshot()'s +// IndexDurableWrites counter rather than by inspecting the filesystem +// directly, since "was this fsync'd" isn't otherwise observable from +// outside the package. + +// TestAddImageCmd_EndsWithDurableIndexSave reproduces the gap for +// AddImageCmd. store.Layout.AddImage calls content.OCI.AddIndex once for +// the base image, then once more per discovered cosign signature, +// attestation, and SBOM (see saveRelatedArtifacts) -- all against the same +// store instance, well within the 30s checkpoint window. Because +// content.OCI's lastDurableSave starts at its zero value, Since(zero) is +// always >= 30s, so the very first AddIndex call of a fresh store is +// durable "for free" -- but every subsequent call, including the last one +// that actually reflects the complete set of discovered artifacts, is not. +// A fix must add a trailing durable SaveIndex() so the run always ends +// durable regardless of how many AddIndex calls happened inside it. +func TestAddImageCmd_EndsWithDurableIndexSave(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + + host, remoteOpts := newLocalhostRegistry(t) + img := seedImage(t, host, "myorg/durable", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "myorg/durable", img, remoteOpts...) + + o := &flags.AddImageOpts{} + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + ref := fmt.Sprintf("%s/myorg/durable:v1", host) + if err := AddImageCmd(ctx, o, s, ref, rso, ro); err != nil { + t.Fatalf("AddImageCmd: %v", err) + } + + snap := s.OCI.Stats().Snapshot() + // Without the fix this is 1: only the very first AddIndex call (image + // itself) lands durably, courtesy of lastDurableSave's zero value. The + // sig/att/sbom AddIndex calls that follow -- and the run's final state + // -- are not durable until a trailing SaveIndex() is added. + if snap.IndexDurableWrites < 2 { + t.Fatalf("expected at least 2 durable index writes (initial AddIndex + trailing checkpoint), got %d (total index writes=%d)", snap.IndexDurableWrites, snap.IndexWrites) + } +} + +// TestAddFileCmd_EndsWithDurableIndexSave verifies AddFileCmd also ends its +// run with a durable index save. A single AddFileCmd call only performs one +// AddIndex call, which (per TestAddImageCmd_EndsWithDurableIndexSave's +// rationale) is already durable "for free" on a brand new store -- so this +// test seeds an unrelated file first to consume that free durability and +// set lastDurableSave to "now", then immediately adds a second file so its +// AddIndex call falls inside the 30s checkpoint window and would be +// non-durable without the fix. +func TestAddFileCmd_EndsWithDurableIndexSave(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + + warmup, err := os.CreateTemp(t.TempDir(), "warmup-*.txt") + if err != nil { + t.Fatal(err) + } + warmup.WriteString("warmup content") //nolint:errcheck + warmup.Close() + + warmupOpts := &flags.AddFileOpts{StoreRootOpts: defaultRootOpts(s.Root)} + if err := AddFileCmd(ctx, warmupOpts, s, warmup.Name(), defaultCliOpts()); err != nil { + t.Fatalf("AddFileCmd warmup: %v", err) + } + + tmp, err := os.CreateTemp(t.TempDir(), "durable-*.txt") + if err != nil { + t.Fatal(err) + } + tmp.WriteString("durable content") //nolint:errcheck + tmp.Close() + + before := s.OCI.Stats().Snapshot().IndexDurableWrites + + o := &flags.AddFileOpts{StoreRootOpts: defaultRootOpts(s.Root)} + if err := AddFileCmd(ctx, o, s, tmp.Name(), defaultCliOpts()); err != nil { + t.Fatalf("AddFileCmd: %v", err) + } + + after := s.OCI.Stats().Snapshot().IndexDurableWrites + if after <= before { + t.Fatalf("expected a trailing durable index save after AddFileCmd, durable writes went from %d to %d", before, after) + } +} + +// -------------------------------------------------------------------------- +// File retry tests +// -------------------------------------------------------------------------- + +// add_file_retry_test.go covers storeFile's (cmd/hauler/cli/store/add.go) +// retry, --ignore-errors, and cancellation behavior -- extending it to match +// storeImage's existing shape (see add_retry_stats_test.go for the analogous +// image-side retry test) as part of bringing Files up to parity with Images +// for `hauler store sync`'s --concurrency support. + +// TestStoreFile_RetriesOnTransientFailure proves storeFile retries a failed +// fetch via retry.Operation rather than aborting the whole sync on one +// transient HTTP blip -- storeFile previously had no retry wrapping at all. +func TestStoreFile_RetriesOnTransientFailure(t *testing.T) { + if testing.Short() { + t.Skip("skipping: requires one RetriesInterval sleep (5s)") + } + + var gets int32 + mux := http.NewServeMux() + mux.HandleFunc("/flaky.sh", func(w http.ResponseWriter, r *http.Request) { + // storeFile's Client.Name(fi.Path) call (used to derive the stored + // ref, before retry.Operation ever starts) issues an unconditional + // HEAD request -- see getter.Http.Name -- that must not count + // against the GET-failure budget below, or the "failure" would be + // silently consumed before AddArtifact's first real attempt. + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusOK) + return + } + if atomic.AddInt32(&gets, 1) == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + io.WriteString(w, "#!/bin/sh\necho ok") //nolint:errcheck + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + ctx := newTestContext(t) + s := newTestStore(t) + + rso := defaultRootOpts(s.Root) + rso.Retries = 2 // one failed attempt + one successful retry + ro := defaultCliOpts() + + if err := storeFile(ctx, s, v1.File{Path: srv.URL + "/flaky.sh"}, ro, rso); err != nil { + t.Fatalf("storeFile: %v", err) + } + assertArtifactInStore(t, s, "flaky.sh") + // The first GET fails; layer.FromOpener opens once per successful + // LayerFrom call (it derives diffID from the already-computed digest + // instead of a second read), so a successful retry attempt adds 1 more + // -- 2 total. + if got := atomic.LoadInt32(&gets); got < 2 { + t.Errorf("expected at least 2 GET attempts (1 failure + 1 for the succeeding retry), got %d", got) + } +} + +// TestStoreFile_IgnoreErrors_WarnsAndReturnsNil proves storeFile absorbs a +// failure into a warning and returns nil when --ignore-errors is set, +// matching storeImage's existing behavior -- previously storeFile always +// propagated the error regardless of ignore-errors. +func TestStoreFile_IgnoreErrors_WarnsAndReturnsNil(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + + ro := defaultCliOpts() + ro.IgnoreErrors = true + rso := defaultRootOpts(s.Root) + + err := storeFile(ctx, s, v1.File{Path: "/nonexistent/path/missing-file.txt"}, ro, rso) + if err != nil { + t.Fatalf("expected nil error with --ignore-errors, got: %v", err) + } + if n := countArtifactsInStore(t, s); n != 0 { + t.Errorf("expected 0 artifacts after an ignored failure, got %d", n) + } +} + +// TestStoreFile_ContextAlreadyCancelled_ReturnsPromptly is the regression +// test for File.compute()'s context.TODO() bug (pkg/artifacts/file/file.go): +// storeFile must check ctx and bail out before ever attempting to fetch, +// matching storeImage's early ctx.Err() check. +func TestStoreFile_ContextAlreadyCancelled_ReturnsPromptly(t *testing.T) { + s := newTestStore(t) + + zl := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(zl.WithContext(context.Background())) + cancel() + + ro := defaultCliOpts() + rso := defaultRootOpts(s.Root) + + err := storeFile(ctx, s, v1.File{Path: "https://example.invalid/never-fetched.sh"}, ro, rso) + if err == nil { + t.Fatal("expected an error for an already-cancelled context, got nil") + } + if n := countArtifactsInStore(t, s); n != 0 { + t.Errorf("expected 0 artifacts, got %d", n) + } +} + +// TestStoreFile_CompletionLine_Format proves storeFile logs a "✓ added" +// completion line (matching storeImage's formatAddedLine convention) rather +// than the old plain "successfully added file" line, and that the old +// "adding file" line is demoted to debug (absent at the default/error log +// level defaultCliOpts() uses). +func TestStoreFile_CompletionLine_Format(t *testing.T) { + url := seedFileInHTTPServer(t, "completion.sh", "#!/bin/sh\necho done") + + s := newTestStore(t) + var buf bytes.Buffer + zl := zerolog.New(&buf).Level(zerolog.InfoLevel) + ctx := zl.WithContext(context.Background()) + + ro := defaultCliOpts() + ro.LogLevel = "info" + rso := defaultRootOpts(s.Root) + + if err := storeFile(ctx, s, v1.File{Path: url}, ro, rso); err != nil { + t.Fatalf("storeFile: %v", err) + } + + out := buf.String() + if !strings.Contains(out, "✓ added") { + t.Errorf("expected a \"✓ added\" completion line, got:\n%s", out) + } + if strings.Contains(out, "successfully added file") { + t.Errorf("expected the old \"successfully added file\" line to be gone, got:\n%s", out) + } +} + +// -------------------------------------------------------------------------- +// Image retry stats tests +// -------------------------------------------------------------------------- + +// add_retry_stats_test.go covers a single narrow bug in storeImage +// (cmd/hauler/cli/store/add.go): a retried s.AddImage attempt must not +// accumulate layer count / byte totals onto the same store.ImageStats +// pointer used by a prior, failed attempt -- otherwise the "✓ added ..." +// completion line double (or N-x) counts layers/bytes across retries. + +// failOnceForDigestHandler wraps an http.Handler and fails the very first GET +// request for one specific blob digest with a 403, then lets every other +// request -- including every later request for that same digest -- through +// unmodified. Targeting a single, specific digest (rather than "the first +// request for whichever digest shows up first") is deliberate: writeImageBlobs +// fetches an image's layers concurrently and its config sequentially +// afterward, so a naive "fail every digest's first-ever request" approach +// makes the *retry* attempt also fail (on whichever blob it reaches first +// that the earlier, aborted attempt never got around to requesting at all). +// Failing exactly one predetermined digest, exactly once, guarantees the +// first AddImage attempt fails (on that blob) while the second, retried +// attempt succeeds outright (every blob, including the previously-failing +// one, now passes through). +// +// 403 (rather than 500/502/503/504/408/429) is deliberate: go-containerregistry's +// remote transport has its own built-in retry for a fixed set of "temporary" +// status codes (see remote.defaultRetryStatusCodes) and would silently absorb +// a transient 500 within a single AddImage call (up to 3 internal attempts). +// 403 isn't in that set, so it surfaces as a hard error from that AddImage +// call and exercises this package's own retry.Operation-driven retry instead. +type failOnceForDigestHandler struct { + next http.Handler + target string // e.g. "sha256:abcd..." + + mu sync.Mutex + failed bool +} + +func (f *failOnceForDigestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/blobs/"+f.target) { + f.mu.Lock() + alreadyFailed := f.failed + f.failed = true + f.mu.Unlock() + + if !alreadyFailed { + w.WriteHeader(http.StatusForbidden) + return + } + } + f.next.ServeHTTP(w, r) +} + +// newFailOnceRegistry starts an in-memory OCI registry, listening on +// "localhost:0" (so go-containerregistry auto-selects plain HTTP, matching +// newLocalhostRegistry in add_test.go), wrapped in a failOnceForDigestHandler. +// The returned handler's target field must be set (to the digest that should +// fail exactly once) before the read that should fail; that happens after +// seeding, once the seeded image's layer digest is known, and before the +// caller reads the image back via AddImage/storeImage. +func newFailOnceRegistry(t *testing.T) (host string, remoteOpts []remote.Option, handler *failOnceForDigestHandler) { + t.Helper() + handler = &failOnceForDigestHandler{next: registry.New()} + + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("newFailOnceRegistry listen: %v", err) + } + srv := httptest.NewUnstartedServer(handler) + srv.Listener = l + srv.Start() + t.Cleanup(srv.Close) + host = strings.TrimPrefix(srv.URL, "http://") + remoteOpts = []remote.Option{remote.WithTransport(srv.Client().Transport)} + return host, remoteOpts, handler +} + +// addedLineLayerCount extracts the layer count from the last "✓ added ... +// (N layer(s), ...)" line in out. Fails the test if no such line is found. +func addedLineLayerCount(t *testing.T, out string) int { + t.Helper() + re := regexp.MustCompile(`✓ added .*\((\d+) layers?,`) + matches := re.FindAllStringSubmatch(out, -1) + if len(matches) == 0 { + t.Fatalf("no \"✓ added ... (N layer(s), ...)\" line found in output:\n%s", out) + } + last := matches[len(matches)-1] + n := 0 + for _, c := range last[1] { + n = n*10 + int(c-'0') + } + return n +} + +// TestStoreImage_RetryDoesNotDoubleCountStats is the regression test for the +// ImageStats double-counting bug: storeImage built a single store.ImageStats +// pointer outside retry.Operation's closure, so a failed first attempt that +// had already accumulated layer/byte counts into it left those counts in +// place for a subsequent, successful retry attempt to accumulate on top of -- +// inflating the final completion line's layer/byte totals. +// +// seedImage (testhelpers_test.go) always creates a 2-layer image. A single, +// uncontested successful AddImage call reports "2 layers"; the bug would +// report "4 layers" after exactly one failed attempt followed by one +// successful retry. +func TestStoreImage_RetryDoesNotDoubleCountStats(t *testing.T) { + if testing.Short() { + t.Skip("skipping: requires one RetriesInterval sleep (5s)") + } + + host, remoteOpts, handler := newFailOnceRegistry(t) + seededImg := seedImage(t, host, "test/retry-stats", "v1", remoteOpts...) + + layers, err := seededImg.Layers() + if err != nil { + t.Fatalf("seededImg.Layers: %v", err) + } + if len(layers) == 0 { + t.Fatal("seeded image has no layers") + } + targetDigest, err := layers[0].Digest() + if err != nil { + t.Fatalf("layers[0].Digest: %v", err) + } + handler.target = targetDigest.String() + + s := newTestStore(t) + var buf bytes.Buffer + zl := zerolog.New(&buf) + ctx := zl.WithContext(context.Background()) + + rso := defaultRootOpts(s.Root) + rso.Retries = 2 // one failed attempt + one successful retry + ro := defaultCliOpts() + + cfg := v1.Image{Name: host + "/test/retry-stats:v1"} + if err := storeImage(ctx, s, cfg, "", true /* excludeExtras: keep this to just the image's own layers */, rso, ro, "", "", false); err != nil { + t.Fatalf("storeImage: %v", err) + } + + out := buf.String() + if got := addedLineLayerCount(t, out); got != 2 { + t.Errorf("reported layer count = %d, want 2 (seedImage's fixed layer count); a retried attempt must not double-count onto the prior failed attempt's stats\nfull output:\n%s", got, out) + } +} + +// -------------------------------------------------------------------------- +// resolveChartJobs tests +// +// These exercise the Charts precedence rules directly, without a store or +// network access. Credential resolution (UsernameEnv/PasswordEnv and the +// field passthrough onto ChartOpts) is covered separately by +// TestResolveChartJobs_CredentialFields, TestResolveChartJobs_CredentialEnv, +// TestResolveChartJobs_CredentialEnvMismatch, and +// TestResolveChartJobs_CredentialIsolation below. +// -------------------------------------------------------------------------- + +func TestResolveChartJobs_Registry(t *testing.T) { + tests := []struct { + name string + cliRegistry string + annotation string + want string + }{ + { + name: "CLI flag wins over annotation", + cliRegistry: "cli-registry.io", + annotation: "annotation-registry.io", + want: "cli-registry.io", + }, + { + name: "annotation used when no CLI flag", + annotation: "annotation-registry.io", + want: "annotation-registry.io", + }, + { + name: "CLI flag used when no annotation", + cliRegistry: "cli-registry.io", + want: "cli-registry.io", + }, + { + name: "neither set leaves registry empty", + want: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{Registry: tc.cliRegistry} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationRegistry] = tc.annotation + } + + jobs, err := resolveChartJobs(o, a, "/manifests", []v1.Chart{{Name: "rancher"}}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if got := jobs[0].opts.Registry; got != tc.want { + t.Errorf("registry = %q, want %q", got, tc.want) + } + }) + } +} + +func TestResolveChartJobs_ExcludeExtras(t *testing.T) { + tests := []struct { + name string + cli bool + annotation string + perChart bool + want bool + }{ + {name: "nothing set", want: false}, + {name: "CLI flag alone", cli: true, want: true}, + {name: "annotation alone", annotation: "true", want: true}, + {name: "per-chart alone", perChart: true, want: true}, + { + name: "annotation only honored when set to the literal true", + annotation: "yes", + want: false, + }, + { + // Neither the annotation nor the per-chart field can turn a CLI + // --exclude-extras back off; both are one-way switches. + name: "CLI flag survives an annotation that is not true", + cli: true, + annotation: "false", + want: true, + }, + { + name: "CLI flag survives a false per-chart field", + cli: true, + perChart: false, + want: true, + }, + { + name: "annotation survives a false per-chart field", + annotation: "true", + perChart: false, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{ExcludeExtras: tc.cli} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationExcludeExtras] = tc.annotation + } + + jobs, err := resolveChartJobs(o, a, "/manifests", []v1.Chart{{Name: "rancher", ExcludeExtras: tc.perChart}}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if got := jobs[0].opts.ExcludeExtras; got != tc.want { + t.Errorf("excludeExtras = %v, want %v", got, tc.want) + } + }) + } +} + +func TestResolveChartJobs_Platform(t *testing.T) { + tests := []struct { + name string + cli string + annotation string + perChart string + want string + }{ + {name: "nothing set", want: ""}, + {name: "CLI flag alone", cli: "linux/amd64", want: "linux/amd64"}, + {name: "annotation alone", annotation: "linux/arm64", want: "linux/arm64"}, + {name: "per-chart alone", perChart: "linux/s390x", want: "linux/s390x"}, + { + // An explicit --platform is run-time intent and outranks manifest + // metadata, matching resolveImageJobs and matching registry's + // CLI-over-annotation rule in this same resolver. + name: "CLI flag wins over annotation", + cli: "linux/amd64", + annotation: "linux/arm64", + want: "linux/amd64", + }, + { + name: "per-chart wins over both", + cli: "linux/amd64", + annotation: "linux/arm64", + perChart: "linux/s390x", + want: "linux/s390x", + }, + { + name: "empty annotation leaves the CLI flag intact", + cli: "linux/amd64", + annotation: "", + want: "linux/amd64", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{Platform: tc.cli} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationPlatform] = tc.annotation + } + + jobs, err := resolveChartJobs(o, a, "/manifests", []v1.Chart{{Name: "rancher", Platform: tc.perChart}}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if got := jobs[0].opts.Platform; got != tc.want { + t.Errorf("platform = %v, want %v", got, tc.want) + } + }) + } +} + +// TestResolveChartJobs_PlatformMatchesImagePath pins the two resolvers +// together: a manifest carrying a platform annotation must select the same +// platform for its Charts and its Images given the same flags. The two +// resolvers hold separate copies of this precedence chain, and drift between +// them is exactly the bug this pins against. +func TestResolveChartJobs_PlatformMatchesImagePath(t *testing.T) { + tests := []struct { + name string + cli string + annotation string + perEntry string + }{ + {name: "nothing set"}, + {name: "CLI flag alone", cli: "linux/amd64"}, + {name: "annotation alone", annotation: "linux/arm64"}, + {name: "per-entry alone", perEntry: "linux/s390x"}, + {name: "CLI flag and annotation", cli: "linux/amd64", annotation: "linux/arm64"}, + {name: "per-entry and annotation", annotation: "linux/arm64", perEntry: "linux/s390x"}, + {name: "per-entry and CLI flag", cli: "linux/amd64", perEntry: "linux/s390x"}, + {name: "all three", cli: "linux/amd64", annotation: "linux/arm64", perEntry: "linux/s390x"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{Platform: tc.cli} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationPlatform] = tc.annotation + } + + chartJobs, err := resolveChartJobs(o, a, "/manifests", []v1.Chart{{Name: "rancher", Platform: tc.perEntry}}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(chartJobs) != 1 { + t.Fatalf("expected 1 chart job, got %d", len(chartJobs)) + } + + imageJobs, err := resolveImageJobs(o, a, []v1.Image{{Name: "rancher/rancher:v2.9", Platform: tc.perEntry}}) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(imageJobs) != 1 { + t.Fatalf("expected 1 image job, got %d", len(imageJobs)) + } + + if chartJobs[0].opts.Platform != imageJobs[0].platform { + t.Errorf("chart platform = %q, image platform = %q, want them equal", + chartJobs[0].opts.Platform, imageJobs[0].platform) + } + }) + } +} + +func TestResolveChartJobs_NilAnnotations(t *testing.T) { + o := &flags.SyncOpts{Registry: "cli-registry.io", Platform: "linux/amd64", ExcludeExtras: true} + + jobs, err := resolveChartJobs(o, nil, "/manifests", []v1.Chart{{Name: "rancher"}}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if got := jobs[0].opts.Registry; got != "cli-registry.io" { + t.Errorf("registry = %q, want cli-registry.io", got) + } + if got := jobs[0].opts.Platform; got != "linux/amd64" { + t.Errorf("platform = %q, want linux/amd64", got) + } + if !jobs[0].opts.ExcludeExtras { + t.Error("excludeExtras = false, want true") + } +} + +func TestResolveChartJobs_ValuesFiles(t *testing.T) { + tests := []struct { + name string + manifestDir string + valuesFiles []string + want []string + }{ + { + name: "relative paths join against the manifest directory", + manifestDir: "/manifests", + valuesFiles: []string{"values.yaml", "overrides/extra.yaml"}, + want: []string{"/manifests/values.yaml", "/manifests/overrides/extra.yaml"}, + }, + { + name: "parent-relative paths are cleaned", + manifestDir: "/manifests/prod", + valuesFiles: []string{"../shared/values.yaml"}, + want: []string{"/manifests/shared/values.yaml"}, + }, + { + name: "manifest in the current directory", + manifestDir: ".", + valuesFiles: []string{"values.yaml"}, + want: []string{"values.yaml"}, + }, + { + name: "no values files yields none", + manifestDir: "/manifests", + valuesFiles: nil, + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, tc.manifestDir, + []v1.Chart{{Name: "rancher", ValuesFiles: tc.valuesFiles}}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + + got := jobs[0].opts.ValuesFiles + if len(got) != len(tc.want) { + t.Fatalf("valuesFiles = %v, want %v", got, tc.want) + } + for i := range tc.want { + if got[i] != tc.want[i] { + t.Errorf("valuesFiles[%d] = %q, want %q", i, got[i], tc.want[i]) + } + } + }) + } +} + +func TestResolveChartJobs_PerChartFields(t *testing.T) { + charts := []v1.Chart{ + { + Name: "rancher", + RepoURL: "https://releases.rancher.com/server-charts/stable", + Version: "2.9.0", + Rewrite: "mirror/rancher", + AddImages: true, + AddDependencies: true, + }, + { + Name: "cert-manager", + RepoURL: "https://charts.jetstack.io", + Version: "1.15.0", + }, + } + + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, "/manifests", charts) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 2 { + t.Fatalf("expected 2 jobs, got %d", len(jobs)) + } + + if jobs[0].cfg.Name != "rancher" || jobs[1].cfg.Name != "cert-manager" { + t.Fatalf("jobs are out of manifest order: %q, %q", jobs[0].cfg.Name, jobs[1].cfg.Name) + } + + if jobs[0].rewrite != "mirror/rancher" { + t.Errorf("jobs[0].rewrite = %q, want mirror/rancher", jobs[0].rewrite) + } + if jobs[1].rewrite != "" { + t.Errorf("jobs[1].rewrite = %q, want empty", jobs[1].rewrite) + } + if !jobs[0].opts.AddImages || !jobs[0].opts.AddDependencies { + t.Errorf("jobs[0] add-images/add-dependencies = %v/%v, want true/true", jobs[0].opts.AddImages, jobs[0].opts.AddDependencies) + } + if jobs[1].opts.AddImages || jobs[1].opts.AddDependencies { + t.Errorf("jobs[1] add-images/add-dependencies = %v/%v, want false/false", jobs[1].opts.AddImages, jobs[1].opts.AddDependencies) + } + + for i, want := range charts { + if got := jobs[i].opts.ChartOpts.RepoURL; got != want.RepoURL { + t.Errorf("jobs[%d].opts.ChartOpts.RepoURL = %q, want %q", i, got, want.RepoURL) + } + if got := jobs[i].opts.ChartOpts.Version; got != want.Version { + t.Errorf("jobs[%d].opts.ChartOpts.Version = %q, want %q", i, got, want.Version) + } + if jobs[i].parent != "" { + t.Errorf("jobs[%d].parent = %q, want empty for a top-level chart", i, jobs[i].parent) + } + if jobs[i].depth != 0 { + t.Errorf("jobs[%d].depth = %d, want 0 for a top-level chart", i, jobs[i].depth) + } + } +} + +// TestResolveChartJobs_ChartOptsNotShared is the regression test for the +// shallow AddChartOpts copy: every job must own its *action.ChartPathOptions, +// so mutating one job's RepoURL/Version cannot leak into a sibling's. +func TestResolveChartJobs_ChartOptsNotShared(t *testing.T) { + charts := []v1.Chart{ + {Name: "rancher", RepoURL: "https://releases.rancher.com/server-charts/stable", Version: "2.9.0"}, + {Name: "cert-manager", RepoURL: "https://charts.jetstack.io", Version: "1.15.0"}, + {Name: "longhorn", RepoURL: "https://charts.longhorn.io", Version: "1.7.0"}, + } + + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, "/manifests", charts) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != len(charts) { + t.Fatalf("expected %d jobs, got %d", len(charts), len(jobs)) + } + + for i := range jobs { + for k := i + 1; k < len(jobs); k++ { + if jobs[i].opts.ChartOpts == jobs[k].opts.ChartOpts { + t.Fatalf("jobs[%d] and jobs[%d] share one *action.ChartPathOptions pointee", i, k) + } + } + } + + jobs[0].opts.ChartOpts.RepoURL = "file:///tmp/subchart" + jobs[0].opts.ChartOpts.Version = "0.0.0-mutated" + + for i := 1; i < len(jobs); i++ { + if got := jobs[i].opts.ChartOpts.RepoURL; got != charts[i].RepoURL { + t.Errorf("jobs[%d].opts.ChartOpts.RepoURL = %q, want %q", i, got, charts[i].RepoURL) + } + if got := jobs[i].opts.ChartOpts.Version; got != charts[i].Version { + t.Errorf("jobs[%d].opts.ChartOpts.Version = %q, want %q", i, got, charts[i].Version) + } + } +} + +func TestResolveChartJobs_NoCharts(t *testing.T) { + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, "/manifests", nil) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 0 { + t.Fatalf("expected no jobs, got %d", len(jobs)) + } +} + +// TestResolveChartJobs_CredentialFields pins that every TLS/verification +// field on v1.Chart reaches the job's ChartOpts unchanged. +func TestResolveChartJobs_CredentialFields(t *testing.T) { + ch := v1.Chart{ + Name: "rancher", + Verify: true, + Keyring: "/keys/pubring.gpg", + PassCredentialsAll: true, + CertFile: "/certs/client.crt", + KeyFile: "/certs/client.key", + CaFile: "/certs/ca.crt", + InsecureSkipTLSVerify: true, + PlainHTTP: true, + } + + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, "/manifests", []v1.Chart{ch}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + + opts := jobs[0].opts.ChartOpts + if opts.Verify != ch.Verify { + t.Errorf("Verify = %v, want %v", opts.Verify, ch.Verify) + } + if opts.Keyring != ch.Keyring { + t.Errorf("Keyring = %q, want %q", opts.Keyring, ch.Keyring) + } + if opts.PassCredentialsAll != ch.PassCredentialsAll { + t.Errorf("PassCredentialsAll = %v, want %v", opts.PassCredentialsAll, ch.PassCredentialsAll) + } + if opts.CertFile != ch.CertFile { + t.Errorf("CertFile = %q, want %q", opts.CertFile, ch.CertFile) + } + if opts.KeyFile != ch.KeyFile { + t.Errorf("KeyFile = %q, want %q", opts.KeyFile, ch.KeyFile) + } + if opts.CaFile != ch.CaFile { + t.Errorf("CaFile = %q, want %q", opts.CaFile, ch.CaFile) + } + if opts.InsecureSkipTLSVerify != ch.InsecureSkipTLSVerify { + t.Errorf("InsecureSkipTLSVerify = %v, want %v", opts.InsecureSkipTLSVerify, ch.InsecureSkipTLSVerify) + } + if opts.PlainHTTP != ch.PlainHTTP { + t.Errorf("PlainHTTP = %v, want %v", opts.PlainHTTP, ch.PlainHTTP) + } +} + +// TestResolveChartJobs_CredentialEnv pins that UsernameEnv/PasswordEnv are +// resolved into ChartOpts.Username/Password via resolveChartCreds. +func TestResolveChartJobs_CredentialEnv(t *testing.T) { + t.Setenv("CHART_USER", "alice") + t.Setenv("CHART_PASS", "s3cret") + + ch := v1.Chart{Name: "rancher", UsernameEnv: "CHART_USER", PasswordEnv: "CHART_PASS"} + + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, "/manifests", []v1.Chart{ch}) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if got := jobs[0].opts.ChartOpts.Username; got != "alice" { + t.Errorf("Username = %q, want alice", got) + } + if got := jobs[0].opts.ChartOpts.Password; got != "s3cret" { + t.Errorf("Password = %q, want s3cret", got) + } +} + +// TestResolveChartJobs_CredentialEnvMismatch pins the fail-fast contract: a +// chart with only one of UsernameEnv/PasswordEnv set must abort the whole +// resolve with no jobs returned, not just skip credentials for that chart. +func TestResolveChartJobs_CredentialEnvMismatch(t *testing.T) { + ch := v1.Chart{Name: "rancher", UsernameEnv: "CHART_USER"} + + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, "/manifests", []v1.Chart{ch}) + if err == nil { + t.Fatal("expected an error for a chart with usernameEnv set but passwordEnv empty") + } + if jobs != nil { + t.Errorf("expected nil jobs on error, got %d", len(jobs)) + } +} + +// TestResolveChartJobs_CredentialIsolation is the credential analogue of +// TestResolveChartJobs_ChartOptsNotShared: each job's Username/Password must +// come from that chart's own env vars, not a sibling's. +func TestResolveChartJobs_CredentialIsolation(t *testing.T) { + t.Setenv("RANCHER_USER", "rancher-user") + t.Setenv("RANCHER_PASS", "rancher-pass") + t.Setenv("LONGHORN_USER", "longhorn-user") + t.Setenv("LONGHORN_PASS", "longhorn-pass") + + charts := []v1.Chart{ + {Name: "rancher", UsernameEnv: "RANCHER_USER", PasswordEnv: "RANCHER_PASS"}, + {Name: "longhorn", UsernameEnv: "LONGHORN_USER", PasswordEnv: "LONGHORN_PASS"}, + } + + jobs, err := resolveChartJobs(&flags.SyncOpts{}, nil, "/manifests", charts) + if err != nil { + t.Fatalf("resolveChartJobs: %v", err) + } + if len(jobs) != 2 { + t.Fatalf("expected 2 jobs, got %d", len(jobs)) + } + + if got := jobs[0].opts.ChartOpts.Username; got != "rancher-user" { + t.Errorf("jobs[0].opts.ChartOpts.Username = %q, want rancher-user", got) + } + if got := jobs[0].opts.ChartOpts.Password; got != "rancher-pass" { + t.Errorf("jobs[0].opts.ChartOpts.Password = %q, want rancher-pass", got) + } + if got := jobs[1].opts.ChartOpts.Username; got != "longhorn-user" { + t.Errorf("jobs[1].opts.ChartOpts.Username = %q, want longhorn-user", got) + } + if got := jobs[1].opts.ChartOpts.Password; got != "longhorn-pass" { + t.Errorf("jobs[1].opts.ChartOpts.Password = %q, want longhorn-pass", got) + } +} + +// -------------------------------------------------------------------------- +// dedupeImageJobs tests +// -------------------------------------------------------------------------- + +// dedupeImageJobs key is (name, platform, excludeExtras); these tests pin +// which of those differences are real pulls and which are duplicates. + +func img(name, platform string, excludeExtras bool) imageJob { + return imageJob{ + img: v1.Image{Name: name}, + platform: platform, + excludeExtras: excludeExtras, + } +} + +func TestDedupeImageJobs(t *testing.T) { + tests := []struct { + name string + in []imageJob + want []imageJob + }{ + { + name: "no jobs", + in: nil, + want: nil, + }, + { + name: "exact duplicates collapse to the first", + in: []imageJob{ + img("rancher/rancher:v2.9", "linux/amd64", false), + img("rancher/rancher:v2.9", "linux/amd64", false), + }, + want: []imageJob{img("rancher/rancher:v2.9", "linux/amd64", false)}, + }, + { + name: "same name, different platform stays separate", + in: []imageJob{ + img("rancher/rancher:v2.9", "linux/amd64", false), + img("rancher/rancher:v2.9", "linux/arm64", false), + }, + want: []imageJob{ + img("rancher/rancher:v2.9", "linux/amd64", false), + img("rancher/rancher:v2.9", "linux/arm64", false), + }, + }, + { + name: "same name, different excludeExtras stays separate", + in: []imageJob{ + img("rancher/rancher:v2.9", "linux/amd64", false), + img("rancher/rancher:v2.9", "linux/amd64", true), + }, + want: []imageJob{ + img("rancher/rancher:v2.9", "linux/amd64", false), + img("rancher/rancher:v2.9", "linux/amd64", true), + }, + }, + { + name: "different names are never merged", + in: []imageJob{ + img("rancher/rancher:v2.9", "", false), + img("rancher/rancher-agent:v2.9", "", false), + }, + want: []imageJob{ + img("rancher/rancher:v2.9", "", false), + img("rancher/rancher-agent:v2.9", "", false), + }, + }, + { + name: "same repository, different tag stays separate", + in: []imageJob{ + img("rancher/rancher:v2.9", "", false), + img("rancher/rancher:v2.10", "", false), + }, + want: []imageJob{ + img("rancher/rancher:v2.9", "", false), + img("rancher/rancher:v2.10", "", false), + }, + }, + { + name: "first-seen order survives interleaved duplicates", + in: []imageJob{ + img("c:1", "", false), + img("a:1", "", false), + img("c:1", "", false), + img("b:1", "", false), + img("a:1", "", false), + }, + want: []imageJob{ + img("c:1", "", false), + img("a:1", "", false), + img("b:1", "", false), + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := dedupeImageJobs(tc.in) + if len(got) != len(tc.want) { + t.Fatalf("got %d jobs, want %d", len(got), len(tc.want)) + } + for i := range tc.want { + if got[i].img.Name != tc.want[i].img.Name || + got[i].platform != tc.want[i].platform || + got[i].excludeExtras != tc.want[i].excludeExtras { + t.Errorf("job[%d] = (%q, %q, %v), want (%q, %q, %v)", + i, got[i].img.Name, got[i].platform, got[i].excludeExtras, + tc.want[i].img.Name, tc.want[i].platform, tc.want[i].excludeExtras) + } + } + }) + } +} + +// TestDedupeImageJobs_KeepsFirstOccurrenceFields pins "first occurrence wins" +// on the whole job, not just its key: a later duplicate's other fields (here, +// rewrite) must not overwrite the retained job's. +func TestDedupeImageJobs_KeepsFirstOccurrenceFields(t *testing.T) { + first := img("rancher/rancher:v2.9", "linux/amd64", false) + first.rewrite = "mirror/rancher" + + second := img("rancher/rancher:v2.9", "linux/amd64", false) + second.rewrite = "other/rancher" + + got := dedupeImageJobs([]imageJob{first, second}) + if len(got) != 1 { + t.Fatalf("expected 1 job, got %d", len(got)) + } + if got[0].rewrite != "mirror/rancher" { + t.Errorf("rewrite = %q, want mirror/rancher", got[0].rewrite) + } +} + +// TestDedupeImageJobs_DoesNotMutateInput pins that callers keep a usable +// slice: deduping returns a new one rather than compacting in place. +func TestDedupeImageJobs_DoesNotMutateInput(t *testing.T) { + in := []imageJob{ + img("a:1", "", false), + img("b:1", "", false), + img("a:1", "", false), + } + + _ = dedupeImageJobs(in) + + if len(in) != 3 { + t.Fatalf("input length changed to %d", len(in)) + } + for i, want := range []string{"a:1", "b:1", "a:1"} { + if in[i].img.Name != want { + t.Errorf("in[%d] = %q, want %q", i, in[i].img.Name, want) + } + } +} + +// -------------------------------------------------------------------------- +// traverseChartLevels graph tests +// +// These cover the traversal in isolation: the seen-set cycle guard, the +// maxChartDepth cap, per-level dependency deduplication, and error +// propagation. Every test injects a chartFetcher, so none of them touch helm, +// the store, or the network. +// -------------------------------------------------------------------------- + +// newChartJob builds a top-level chartJob for a bare chart name, with the +// per-job *action.ChartPathOptions allocation resolveChartJobs guarantees. +func newChartJob(name string) chartJob { + return chartJob{ + cfg: v1.Chart{Name: name}, + opts: flags.AddChartOpts{ChartOpts: &action.ChartPathOptions{}}, + } +} + +// recordingFetcher is a chartFetcher that records the name of every chart it +// is asked to fetch and returns the dependencies graph[name] declares. +type recordingFetcher struct { + mu sync.Mutex + seen []string + graph map[string][]string +} + +func (r *recordingFetcher) fetch(_ context.Context, j chartJob) ([]imageJob, []chartJob, error) { + r.mu.Lock() + r.seen = append(r.seen, j.cfg.Name) + deps := r.graph[j.cfg.Name] + r.mu.Unlock() + + out := make([]chartJob, 0, len(deps)) + for _, d := range deps { + out = append(out, newChartJob(d)) + } + return nil, out, nil +} + +func (r *recordingFetcher) count(name string) int { + r.mu.Lock() + defer r.mu.Unlock() + n := 0 + for _, s := range r.seen { + if s == name { + n++ + } + } + return n +} + +// TestTraverseChartLevels_TerminatesOnCycle pins the seen-set cycle guard, +// without which the walk recurses forever: A depends on B, B depends back on +// A, and each must be fetched exactly once. +func TestTraverseChartLevels_TerminatesOnCycle(t *testing.T) { + f := &recordingFetcher{graph: map[string][]string{ + "a": {"b"}, + "b": {"a"}, + }} + + _, charts, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("a")}, 2, f.fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + + if got := f.count("a"); got != 1 { + t.Errorf("chart a fetched %d times, want exactly 1", got) + } + if got := f.count("b"); got != 1 { + t.Errorf("chart b fetched %d times, want exactly 1", got) + } + if charts != 2 { + t.Errorf("charts fetched = %d, want 2", charts) + } +} + +// TestTraverseChartLevels_SelfCycleTerminates covers the degenerate cycle a +// chart declaring itself as a dependency would produce. +func TestTraverseChartLevels_SelfCycleTerminates(t *testing.T) { + f := &recordingFetcher{graph: map[string][]string{"a": {"a"}}} + + _, charts, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("a")}, 2, f.fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + if charts != 1 { + t.Errorf("charts fetched = %d, want 1", charts) + } +} + +// TestTraverseChartLevels_SeenKeyIncludesRepoAndVersion asserts the seen set +// is keyed on name|repoURL|version rather than name alone: the same chart +// name at two versions is two genuinely different pulls. +func TestTraverseChartLevels_SeenKeyIncludesRepoAndVersion(t *testing.T) { + var mu sync.Mutex + var fetched []string + + fetch := func(_ context.Context, j chartJob) ([]imageJob, []chartJob, error) { + mu.Lock() + fetched = append(fetched, j.cfg.Name+"@"+j.cfg.Version) + mu.Unlock() + + if j.cfg.Version != "" { + return nil, nil, nil + } + return nil, []chartJob{ + {cfg: v1.Chart{Name: "dep", RepoURL: "https://charts.example.com", Version: "1.0.0"}, opts: flags.AddChartOpts{ChartOpts: &action.ChartPathOptions{}}}, + {cfg: v1.Chart{Name: "dep", RepoURL: "https://charts.example.com", Version: "2.0.0"}, opts: flags.AddChartOpts{ChartOpts: &action.ChartPathOptions{}}}, + }, nil + } + + _, charts, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("parent")}, 2, fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + if charts != 3 { + t.Fatalf("charts fetched = %d, want 3 (parent + dep@1.0.0 + dep@2.0.0), got %v", charts, fetched) + } +} + +// TestTraverseChartLevels_DedupesRepeatedDependenciesInOneLevel covers two +// siblings declaring the identical dependency: it is fetched once, not twice. +func TestTraverseChartLevels_DedupesRepeatedDependenciesInOneLevel(t *testing.T) { + f := &recordingFetcher{graph: map[string][]string{ + "a": {"shared"}, + "b": {"shared"}, + "shared": nil, + }} + + _, charts, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("a"), newChartJob("b")}, 2, f.fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + if got := f.count("shared"); got != 1 { + t.Errorf("shared dependency fetched %d times, want exactly 1", got) + } + if charts != 3 { + t.Errorf("charts fetched = %d, want 3", charts) + } +} + +// TestTraverseChartLevels_DepthCapHolds pins maxChartDepth independently of +// the seen set: an infinitely deep chain of distinct charts (which the seen +// set can never stop) must terminate at exactly maxChartDepth levels. +func TestTraverseChartLevels_DepthCapHolds(t *testing.T) { + var mu sync.Mutex + n := 0 + + fetch := func(_ context.Context, j chartJob) ([]imageJob, []chartJob, error) { + mu.Lock() + n++ + mu.Unlock() + return nil, []chartJob{newChartJob(j.cfg.Name + "-child")}, nil + } + + _, charts, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("root")}, 2, fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if n != maxChartDepth { + t.Errorf("fetch called %d times, want exactly maxChartDepth (%d) -- one chart per level on an unbounded chain", n, maxChartDepth) + } + if charts != maxChartDepth { + t.Errorf("charts fetched = %d, want %d", charts, maxChartDepth) + } +} + +// TestTraverseChartLevels_CollectsImagesAcrossLevels asserts images +// discovered at every depth reach the caller, not just the top level's. +func TestTraverseChartLevels_CollectsImagesAcrossLevels(t *testing.T) { + fetch := func(_ context.Context, j chartJob) ([]imageJob, []chartJob, error) { + imgs := []imageJob{{img: v1.Image{Name: j.cfg.Name + "-image"}}} + if j.cfg.Name == "parent" { + return imgs, []chartJob{newChartJob("child")}, nil + } + return imgs, nil, nil + } + + images, charts, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("parent")}, 2, fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + if charts != 2 { + t.Fatalf("charts fetched = %d, want 2", charts) + } + + got := map[string]bool{} + for _, i := range images { + got[i.img.Name] = true + } + for _, want := range []string{"parent-image", "child-image"} { + if !got[want] { + t.Errorf("image %q missing from discovered set %v", want, got) + } + } +} + +// TestTraverseChartLevels_PropagatesFetchError asserts a fetch failure +// surfaces verbatim rather than as an errgroup cancellation, matching +// runImageJobs' semantics. +func TestTraverseChartLevels_PropagatesFetchError(t *testing.T) { + sentinel := errors.New("chart repo unreachable") + + fetch := func(_ context.Context, j chartJob) ([]imageJob, []chartJob, error) { + if j.cfg.Name == "bad" { + return nil, nil, sentinel + } + return nil, nil, nil + } + + _, _, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("bad")}, 1, fetch) + if !errors.Is(err, sentinel) { + t.Fatalf("traverseChartLevels error = %v, want it to wrap %v", err, sentinel) + } +} + +// TestTraverseChartLevels_ErrorInDeeperLevelStopsTraversal asserts a failure +// discovered at depth 1 aborts before any depth-2 chart is fetched. +func TestTraverseChartLevels_ErrorInDeeperLevelStopsTraversal(t *testing.T) { + sentinel := errors.New("subchart missing") + f := &recordingFetcher{graph: map[string][]string{ + "root": {"mid"}, + "mid": {"leaf"}, + }} + + fetch := func(ctx context.Context, j chartJob) ([]imageJob, []chartJob, error) { + if j.cfg.Name == "mid" { + return nil, nil, sentinel + } + return f.fetch(ctx, j) + } + + if _, _, err := traverseChartLevels(context.Background(), []chartJob{newChartJob("root")}, 1, fetch); !errors.Is(err, sentinel) { + t.Fatalf("traverseChartLevels error = %v, want it to wrap %v", err, sentinel) + } + if got := f.count("leaf"); got != 0 { + t.Errorf("leaf fetched %d times after its parent level failed, want 0", got) + } +} + +// TestTraverseChartLevels_NoJobs covers the empty-input path. +func TestTraverseChartLevels_NoJobs(t *testing.T) { + fetch := func(context.Context, chartJob) ([]imageJob, []chartJob, error) { + return nil, nil, fmt.Errorf("fetch must not be called with no jobs") + } + + images, charts, err := traverseChartLevels(context.Background(), nil, 4, fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + if len(images) != 0 || charts != 0 { + t.Errorf("images = %v, charts = %d, want none", images, charts) + } +} + +// -------------------------------------------------------------------------- +// Concurrent chart pipeline tests +// +// These cover that traverseChartLevels honors its concurrency bound, that +// dependency-derived jobs never share an *action.ChartPathOptions with their +// parent, and that a file:// dependency still resolves after its parent's +// level has completed -- the regression that runChartJobs' single shared temp +// root exists to prevent. +// -------------------------------------------------------------------------- + +// newFileDependencyChartJob builds a top-level chartJob for +// testdata/chart-with-file-dependency-chart-1.0.0.tgz, whose Chart.yaml +// declares two dependencies that resolve inside the parent's own expanded +// directory (one via file://, one via an empty repository field). +func newFileDependencyChartJob() chartJob { + return chartJob{ + cfg: v1.Chart{Name: "chart-with-file-dependency-chart-1.0.0.tgz", RepoURL: chartTestdataDir}, + opts: flags.AddChartOpts{ + ChartOpts: &action.ChartPathOptions{RepoURL: chartTestdataDir}, + AddDependencies: true, + }, + } +} + +// TestTraverseChartLevels_BoundedFanOut asserts the observed peak of +// simultaneously in-flight fetches never exceeds the requested concurrency, +// at every level of the walk. +func TestTraverseChartLevels_BoundedFanOut(t *testing.T) { + const perLevel = 12 + + for _, concurrency := range []int{1, 2, 4} { + t.Run(fmt.Sprintf("concurrency=%d", concurrency), func(t *testing.T) { + var ( + mu sync.Mutex + inFlight int + peak int + ) + + fetch := func(_ context.Context, j chartJob) ([]imageJob, []chartJob, error) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + + // Long enough that a broken bound would overlap observably; + // short enough to keep the whole table under a second. + time.Sleep(2 * time.Millisecond) + + mu.Lock() + inFlight-- + mu.Unlock() + + if strings.HasSuffix(j.cfg.Name, "-child") { + return nil, nil, nil + } + return nil, []chartJob{newChartJob(j.cfg.Name + "-child")}, nil + } + + jobs := make([]chartJob, 0, perLevel) + for i := 0; i < perLevel; i++ { + jobs = append(jobs, newChartJob(fmt.Sprintf("chart%d", i))) + } + + _, charts, err := traverseChartLevels(context.Background(), jobs, concurrency, fetch) + if err != nil { + t.Fatalf("traverseChartLevels: %v", err) + } + if charts != 2*perLevel { + t.Errorf("charts fetched = %d, want %d", charts, 2*perLevel) + } + + mu.Lock() + defer mu.Unlock() + if peak > concurrency { + t.Errorf("peak in-flight fetches = %d, want <= %d", peak, concurrency) + } + if peak < 1 { + t.Errorf("peak in-flight fetches = %d, want at least 1", peak) + } + }) + } +} + +// TestFetchChart_DependencyJobsDoNotShareChartOpts is the pointer-identity +// regression test for the derived-job path. resolveChartJobs' equivalent test +// covers only top-level jobs; a `depJob := parentJob` struct copy would pass +// that one while still sharing the *action.ChartPathOptions pointee here, +// which under concurrency is a live data race on RepoURL/Version. +func TestFetchChart_DependencyJobsDoNotShareChartOpts(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + parent := newFileDependencyChartJob() + _, deps, err := fetchChart(ctx, s, parent, t.TempDir(), rso, ro) + if err != nil { + t.Fatalf("fetchChart: %v", err) + } + if len(deps) != 2 { + t.Fatalf("dependency jobs = %d, want 2 (child + crds)", len(deps)) + } + + for i, d := range deps { + if d.opts.ChartOpts == parent.opts.ChartOpts { + t.Fatalf("deps[%d] shares its parent's *action.ChartPathOptions pointee", i) + } + } + if deps[0].opts.ChartOpts == deps[1].opts.ChartOpts { + t.Fatal("deps[0] and deps[1] share one *action.ChartPathOptions pointee") + } + + parentRepoURL := parent.opts.ChartOpts.RepoURL + deps[0].opts.ChartOpts.RepoURL = "https://mutated.example.com" + deps[0].opts.ChartOpts.Version = "0.0.0-mutated" + + if got := parent.opts.ChartOpts.RepoURL; got != parentRepoURL { + t.Errorf("parent RepoURL = %q after mutating a dependency's, want %q", got, parentRepoURL) + } + if got := deps[1].opts.ChartOpts.RepoURL; got == "https://mutated.example.com" { + t.Error("mutating deps[0] was visible through deps[1]") + } + + parent.opts.ChartOpts.Version = "9.9.9-mutated" + for i, d := range deps { + if d.opts.ChartOpts.Version == "9.9.9-mutated" { + t.Errorf("mutating the parent was visible through deps[%d]", i) + } + } +} + +// TestFetchChart_DerivedDependencyFields pins the shape of a derived job: +// dependencies are walked further, images are not rediscovered per subchart, +// no rewrite is inherited, and parent/depth are set for attribution and the +// depth cap. +func TestFetchChart_DerivedDependencyFields(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + parent := newFileDependencyChartJob() + parent.opts.AddImages = true + parent.rewrite = "myorg/parent-chart" + + _, deps, err := fetchChart(ctx, s, parent, t.TempDir(), rso, ro) + if err != nil { + t.Fatalf("fetchChart: %v", err) + } + if len(deps) != 2 { + t.Fatalf("dependency jobs = %d, want 2", len(deps)) + } + + for i, d := range deps { + if !d.opts.AddDependencies { + t.Errorf("deps[%d].opts.AddDependencies = false, want true", i) + } + if d.opts.AddImages { + t.Errorf("deps[%d].opts.AddImages = true, want false: the parent's render already covered the whole tree", i) + } + if d.rewrite != "" { + t.Errorf("deps[%d].rewrite = %q, want empty", i, d.rewrite) + } + if d.depth != parent.depth+1 { + t.Errorf("deps[%d].depth = %d, want %d", i, d.depth, parent.depth+1) + } + if !strings.Contains(d.parent, "chart-with-file-dependency-chart") { + t.Errorf("deps[%d].parent = %q, want it to name the parent chart's ref", i, d.parent) + } + } +} + +// TestRunChartJobs_FileDependencyResolvesAfterParentLevel is the +// shared-temp-root regression test. A file:// dependency's chartJob names a +// path *inside* its parent's expanded directory, and BFS fetches it only +// after the parent's whole level has returned. A per-chart temp dir removed +// when fetchChart returns would delete that path out from under the child, so +// this test fails outright under that design rather than flaking. +func TestRunChartJobs_FileDependencyResolvesAfterParentLevel(t *testing.T) { + for _, concurrency := range []int{1, 4} { + t.Run(fmt.Sprintf("concurrency=%d", concurrency), func(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := runChartJobs(ctx, s, []chartJob{newFileDependencyChartJob()}, concurrency, rso, ro, nil); err != nil { + t.Fatalf("runChartJobs: %v", err) + } + + assertArtifactInStore(t, s, "chart-with-file-dependency-chart:1.0.0") + assertArtifactInStore(t, s, "child:2.0.0") + assertArtifactInStore(t, s, "crds:0.0.1") + }) + } +} + +// TestRunChartJobs_RemovesTempRoot asserts the shared temp root does not +// outlive the call: it holds every chart's expansion, so leaking one per +// `store sync` invocation would accumulate on disk. +func TestRunChartJobs_RemovesTempRoot(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + rso.TempOverride = t.TempDir() + ro := defaultCliOpts() + + if err := runChartJobs(ctx, s, []chartJob{newFileDependencyChartJob()}, 2, rso, ro, nil); err != nil { + t.Fatalf("runChartJobs: %v", err) + } + + entries, err := os.ReadDir(rso.TempOverride) + if err != nil { + t.Fatalf("ReadDir %s: %v", rso.TempOverride, err) + } + for _, e := range entries { + t.Errorf("temp override still contains %q after runChartJobs returned", filepath.Join(rso.TempOverride, e.Name())) + } +} + +// TestFormatAddedLine_NilStats proves formatAddedLine never prints +// "0 layers" when stats is nil, and produces a sane elapsed-only line. +func TestFormatAddedLine_NilStats(t *testing.T) { + got := formatAddedLine("example.com/repo:v1", nil, 1500*time.Millisecond) + + if strings.Contains(got, "0 layer") { + t.Errorf("formatAddedLine with nil stats must never print \"0 layers\", got %q", got) + } + if !strings.Contains(got, "example.com/repo:v1") { + t.Errorf("formatAddedLine must include the ref, got %q", got) + } + if !strings.Contains(got, "1.5s") { + t.Errorf("formatAddedLine must include the elapsed time as %%.1fs, got %q", got) + } + if !strings.HasPrefix(got, "✓ added") { + t.Errorf("formatAddedLine must start with \"✓ added\", got %q", got) + } +} + +// TestFormatAddedLine_ZeroValueStats proves formatAddedLine treats a +// zero-value *store.ImageStats (Layers == 0) the same as nil: elapsed-only, +// never "0 layers". +func TestFormatAddedLine_ZeroValueStats(t *testing.T) { + stats := &store.ImageStats{} + got := formatAddedLine("example.com/repo:v1", stats, 2*time.Second) + + if strings.Contains(got, "0 layer") { + t.Errorf("formatAddedLine with zero-value stats must never print \"0 layers\", got %q", got) + } + if !strings.Contains(got, "2.0s") { + t.Errorf("formatAddedLine must include the elapsed time as %%.1fs, got %q", got) + } +} + +// TestFormatAddedLine_WithStats proves formatAddedLine includes layer +// count (correctly pluralized), human-readable byte size, and elapsed time +// when stats has at least one layer. +func TestFormatAddedLine_WithStats(t *testing.T) { + tests := []struct { + name string + layers int64 + bytes int64 + wantSub string + }{ + {name: "singular layer", layers: 1, bytes: 100, wantSub: "1 layer,"}, + {name: "plural layers", layers: 3, bytes: 1_500_000, wantSub: "3 layers,"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stats := &store.ImageStats{} + stats.Layers.Store(tt.layers) + stats.Bytes.Store(tt.bytes) + + got := formatAddedLine("example.com/repo:v1", stats, 3*time.Second) + + if !strings.Contains(got, tt.wantSub) { + t.Errorf("formatAddedLine = %q, want substring %q", got, tt.wantSub) + } + wantSize := humanize.Bytes(uint64(tt.bytes)) + if !strings.Contains(got, wantSize) { + t.Errorf("formatAddedLine = %q, want it to contain human size %q", got, wantSize) + } + if !strings.Contains(got, "3.0s") { + t.Errorf("formatAddedLine = %q, want it to contain elapsed \"3.0s\"", got) + } + }) + } } diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index 14b8f64..a40cffb 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -33,6 +33,8 @@ func CopyCmd(ctx context.Context, o *flags.CopyOpts, s *store.Layout, targetRef return fmt.Errorf("store index not found: run 'hauler store add/sync/load' first") } + ignoreErrors := flags.ShouldIgnoreErrors(ro) + components := strings.SplitN(targetRef, "://", 2) switch components[0] { case "directory", "dir": @@ -243,7 +245,7 @@ func CopyCmd(ctx context.Context, o *flags.CopyOpts, s *store.Layout, targetRef toRef, err := content.RewriteRefToRegistry(destRef, components[1]) if err != nil { - if !ro.IgnoreErrors { + if !ignoreErrors { fatalErr = fmt.Errorf("rewriting ref [%s]: %w", baseRef, err) return nil } @@ -263,7 +265,7 @@ func CopyCmd(ctx context.Context, o *flags.CopyOpts, s *store.Layout, targetRef pushed, copyErr = s.Copy(ctx, reference, target, toRef) return copyErr }); err != nil { - if !ro.IgnoreErrors { + if !ignoreErrors { fatalErr = err } return nil diff --git a/cmd/hauler/cli/store/copy_test.go b/cmd/hauler/cli/store/copy_test.go index 431fd75..e5d417e 100644 --- a/cmd/hauler/cli/store/copy_test.go +++ b/cmd/hauler/cli/store/copy_test.go @@ -139,7 +139,7 @@ func TestCopyCmd_Registry(t *testing.T) { s := newTestStore(t) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/copy:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/copy:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage: %v", err) } @@ -175,7 +175,7 @@ func TestCopyCmd_Registry_OnlyFilter(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() for _, repo := range []string{"myorg/repo1:v1", "myorg/repo2:v1"} { - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/" + repo}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/" + repo}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage %s: %v", repo, err) } } @@ -222,7 +222,7 @@ func TestCopyCmd_Registry_SigTagDerivation(t *testing.T) { // AddImage discovers and stores the .sig/.att/.sbom tags automatically. s := newTestStore(t) - if _, err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -262,7 +262,7 @@ func TestCopyCmd_Registry_IgnoreErrors(t *testing.T) { s := newTestStore(t) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/ignore:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/ignore:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage: %v", err) } @@ -278,6 +278,102 @@ func TestCopyCmd_Registry_IgnoreErrors(t *testing.T) { } } +// TestCopyCmd_Registry_IgnoreErrors_EnvVar verifies that a push failure to a +// non-listening address is swallowed when HAULER_IGNORE_ERRORS=true is set via +// the environment alone, without --ignore-errors. This is a regression test: +// retry.Operation used to mutate the shared ro.IgnoreErrors from the env var as +// a side effect, so the ro.IgnoreErrors read immediately after it always saw +// the env var. Now that retry.Operation is a pure read via +// flags.ShouldIgnoreErrors, the check after it must not silently start +// aborting on the first failure again. +func TestCopyCmd_Registry_IgnoreErrors_EnvVar(t *testing.T) { + ctx := newTestContext(t) + + srcHost, _ := newLocalhostRegistry(t) + seedImage(t, srcHost, "test/ignore-env", "v1") + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/ignore-env:v1"}, "", false, rso, ro, "", "", false); err != nil { + t.Fatalf("storeImage: %v", err) + } + + // localhost:1 is a port that is never listening. + o := &flags.CopyOpts{ + StoreRootOpts: defaultRootOpts(s.Root), + PlainHTTP: true, + } + roEnv := defaultCliOpts() + t.Setenv(consts.HaulerIgnoreErrors, "true") + if err := CopyCmd(ctx, o, s, "registry://localhost:1", roEnv); err != nil { + t.Errorf("expected no error with HAULER_IGNORE_ERRORS=true, got: %v", err) + } + if roEnv.IgnoreErrors { + t.Fatal("expected ro.IgnoreErrors to remain false: CopyCmd must not mutate ro") + } +} + +// TestCopyCmd_Registry_IgnoreErrors_EnvVar_MultipleArtifacts verifies that, with +// HAULER_IGNORE_ERRORS=true set via the environment only, a copy failure for one +// artifact does not prevent CopyCmd from continuing on to push the remaining +// artifacts to the target registry. +func TestCopyCmd_Registry_IgnoreErrors_EnvVar_MultipleArtifacts(t *testing.T) { + ctx := newTestContext(t) + + srcHost, _ := newLocalhostRegistry(t) + seedImage(t, srcHost, "test/ignore-env-a", "v1") + seedImage(t, srcHost, "test/ignore-env-b", "v1") + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + for _, repo := range []string{"test/ignore-env-a:v1", "test/ignore-env-b:v1"} { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/" + repo}, "", false, rso, ro, "", "", false); err != nil { + t.Fatalf("storeImage %s: %v", repo, err) + } + } + + // Seed an undeliverable referrer descriptor directly, per the technique used + // in TestCopy_UndeliverableArtifact_RespectsIgnoreErrors, so that one artifact + // fails ref-rewriting (copy.go's earlier IgnoreErrors check) while the two + // images above should still make it to the target registry. + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.Digest("sha256:not a valid digest"), + Size: 1, + Annotations: map[string]string{ + ocispec.AnnotationRefName: "myorg/undeliverable", + consts.ContainerdImageNameKey: "myorg/undeliverable", + consts.KindAnnotationName: consts.KindAnnotationReferrers + "/" + strings.Repeat("a", 64), + }, + } + if err := s.OCI.AddIndex(desc); err != nil { + t.Fatalf("AddIndex: %v", err) + } + + dstHost, dstOpts := newTestRegistry(t) + o := &flags.CopyOpts{ + StoreRootOpts: defaultRootOpts(s.Root), + PlainHTTP: true, + } + roEnv := defaultCliOpts() + t.Setenv(consts.HaulerIgnoreErrors, "true") + if err := CopyCmd(ctx, o, s, "registry://"+dstHost, roEnv); err != nil { + t.Fatalf("expected no error with HAULER_IGNORE_ERRORS=true, got: %v", err) + } + + for _, repo := range []string{"test/ignore-env-a", "test/ignore-env-b"} { + ref, err := name.NewTag(dstHost+"/"+repo+":v1", name.Insecure) + if err != nil { + t.Fatalf("name.NewTag %s: %v", repo, err) + } + if _, err := remote.Get(ref, dstOpts...); err != nil { + t.Errorf("%s should be in target registry despite unrelated undeliverable artifact, but was not found: %v", repo, err) + } + } +} + // TestCopy_UndeliverableArtifact_RespectsIgnoreErrors verifies that when // CopyCmd's registry branch derives an unparseable destination ref for an // artifact (RewriteRefToRegistry failure), the walk fails by default and @@ -329,6 +425,23 @@ func TestCopy_UndeliverableArtifact_RespectsIgnoreErrors(t *testing.T) { t.Errorf("expected no error with IgnoreErrors=true, got: %v", err) } }) + + // Regression test: with retry.Operation no longer mutating the shared + // ro.IgnoreErrors as a side effect, the ref-rewrite failure check earlier in + // CopyCmd's registry walk must still honor HAULER_IGNORE_ERRORS set via the + // environment alone, without --ignore-errors. + t.Run("ignore errors via env var returns nil", func(t *testing.T) { + s := buildStore(t) + o := &flags.CopyOpts{StoreRootOpts: defaultRootOpts(s.Root), PlainHTTP: true} + ro := defaultCliOpts() + t.Setenv(consts.HaulerIgnoreErrors, "true") + if err := CopyCmd(ctx, o, s, "registry://"+dstHost, ro); err != nil { + t.Errorf("expected no error with HAULER_IGNORE_ERRORS=true, got: %v", err) + } + if ro.IgnoreErrors { + t.Fatal("expected ro.IgnoreErrors to remain false: CopyCmd must not mutate ro") + } + }) } // TestCopyCmd_Registry_InvalidFilenameSkipTest verifies that CopyCmd emits a @@ -434,7 +547,7 @@ func TestCopyCmd_Dir_SkipsImages(t *testing.T) { s := newTestStore(t) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/imgskip:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/imgskip:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage: %v", err) } diff --git a/cmd/hauler/cli/store/extract_test.go b/cmd/hauler/cli/store/extract_test.go index c59e183..61a69af 100644 --- a/cmd/hauler/cli/store/extract_test.go +++ b/cmd/hauler/cli/store/extract_test.go @@ -167,7 +167,7 @@ func TestExtractCmd_OciArtifactKindImage(t *testing.T) { // Pull into a fresh store — AddImage sets kind=KindAnnotationImage on all manifests. s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -249,7 +249,7 @@ func TestExtractCmd_OciImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -359,7 +359,7 @@ func TestExtractCmd_NestedImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -427,7 +427,7 @@ func TestExtractCmd_ContainerImage_Skipped(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -497,7 +497,7 @@ func TestExtractCmd_ContainerImageIndex_Skipped(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/info_test.go b/cmd/hauler/cli/store/info_test.go index 39dc428..4050d70 100644 --- a/cmd/hauler/cli/store/info_test.go +++ b/cmd/hauler/cli/store/info_test.go @@ -352,7 +352,7 @@ func TestInfoCmd_CheckHealthyStore(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/healthy", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/healthy:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/healthy:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -403,11 +403,11 @@ func TestInfoCmd_CheckCorruptBlob_HealthySiblingOmitted(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/corrupt-a", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/corrupt-a:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/corrupt-a:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage corrupt-a: %v", err) } seedImage(t, host, "test/healthy-b", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/healthy-b:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/healthy-b:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage healthy-b: %v", err) } @@ -468,7 +468,7 @@ func TestInfoCmd_CheckCorruptManifest_RegressionGuard(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/badmanifest", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/badmanifest:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/badmanifest:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -511,7 +511,7 @@ func TestInfoCmd_NoCheck_OutputUnchanged(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/plain", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -567,7 +567,7 @@ func TestInfoCmd_CheckWithTypeFilter_SkipsFilteredCorruption(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/filterhealthy", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/filterhealthy:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/filterhealthy:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -621,7 +621,7 @@ func TestInfoCmd_CheckHealthyStore_TableFormat_NoTableRendered(t *testing.T) { var buf bytes.Buffer ctx := newCapturingContext(&buf) - if _, err := s.AddImage(ctx, host+"/test/healthytable:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/healthytable:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -668,7 +668,7 @@ func TestInfoCmd_CheckCorruptBlob_RemediationHintLogged(t *testing.T) { ctx := newCapturingContext(&buf) seedImage(t, host, "test/remediation", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/remediation:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/remediation:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -711,7 +711,7 @@ func TestInfoCmd_RemediationHint_DedupedForMultiPlatformImage(t *testing.T) { logCtx := newCapturingContext(&buf) idx := seedIndex(t, host, "test/remediation-multiarch", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/remediation-multiarch:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/remediation-multiarch:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -770,7 +770,7 @@ func TestInfoCmd_CheckMultipleBadBlobs_OneRowPerProblem(t *testing.T) { host, opts := newTestRegistry(t) img := seedImage(t, host, "test/doublebad", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/doublebad:v1", "", true, opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/doublebad:v1", "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/lifecycle_test.go b/cmd/hauler/cli/store/lifecycle_test.go index dc9d25b..a8ef96e 100644 --- a/cmd/hauler/cli/store/lifecycle_test.go +++ b/cmd/hauler/cli/store/lifecycle_test.go @@ -109,7 +109,7 @@ func TestLifecycle_Image_AddSaveLoadCopyRegistry(t *testing.T) { storeA := newTestStore(t) rso := defaultRootOpts(storeA.Root) ro := defaultCliOpts() - if err := storeImage(ctx, storeA, v1.Image{Name: srcHost + "/lifecycle/app:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, storeA, v1.Image{Name: srcHost + "/lifecycle/app:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage: %v", err) } assertArtifactInStore(t, storeA, "lifecycle/app:v1") @@ -260,7 +260,7 @@ func TestLifecycle_DigestOnlyImage_AddSaveLoad(t *testing.T) { rso := defaultRootOpts(storeA.Root) ro := defaultCliOpts() digestRef := srcHost + "/lifecycle/digestonly@" + hash.String() - if err := storeImage(ctx, storeA, v1.Image{Name: digestRef}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, storeA, v1.Image{Name: digestRef}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage by digest: %v", err) } // The image should be findable by its digest hex diff --git a/cmd/hauler/cli/store/remove_test.go b/cmd/hauler/cli/store/remove_test.go index e887ddd..e8b5b82 100644 --- a/cmd/hauler/cli/store/remove_test.go +++ b/cmd/hauler/cli/store/remove_test.go @@ -138,7 +138,7 @@ func TestRemoveCmd_ContainerdImageName(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/repo:v1"}, "", false, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/repo:v1"}, "", false, rso, ro, "", "", false); err != nil { t.Fatalf("storeImage: %v", err) } diff --git a/cmd/hauler/cli/store/save_test.go b/cmd/hauler/cli/store/save_test.go index 37988dc..b9d138c 100644 --- a/cmd/hauler/cli/store/save_test.go +++ b/cmd/hauler/cli/store/save_test.go @@ -59,7 +59,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v1", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -78,7 +78,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v2", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -126,7 +126,7 @@ func TestWriteExportsManifest_DigestOnlyImageHasRepoTag(t *testing.T) { // Add the image BY DIGEST s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false, ""); err != nil { t.Fatalf("AddImage by digest: %v", err) } @@ -174,7 +174,7 @@ func TestSaveCmd(t *testing.T) { seedImage(t, host, "test/save", "v1") s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/save:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/save:v1", "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -207,7 +207,7 @@ func TestSaveCmd_ContainerdCompatibility(t *testing.T) { seedImage(t, host, "test/containerd-compat", "v1") s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -306,7 +306,7 @@ func TestSaveCmd_ChunkSize(t *testing.T) { seedImage(t, host, "test/chunksave", "v1") s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index beefb0c..9ccdb3b 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -3,30 +3,35 @@ package store import ( "bufio" "context" + "errors" "fmt" "io" "net/url" "os" "path/filepath" "strings" + "time" + "github.com/dustin/go-humanize" "github.com/google/go-containerregistry/pkg/authn" gname "github.com/google/go-containerregistry/pkg/name" gv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/mitchellh/go-homedir" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "helm.sh/helm/v4/pkg/action" + "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/util/yaml" "hauler.dev/go/hauler/v2/internal/flags" v1 "hauler.dev/go/hauler/v2/pkg/apis/hauler.cattle.io/v1" + "hauler.dev/go/hauler/v2/pkg/artifacts/file" "hauler.dev/go/hauler/v2/pkg/consts" "hauler.dev/go/hauler/v2/pkg/content" "hauler.dev/go/hauler/v2/pkg/cosign" "hauler.dev/go/hauler/v2/pkg/getter" "hauler.dev/go/hauler/v2/pkg/log" "hauler.dev/go/hauler/v2/pkg/reference" + "hauler.dev/go/hauler/v2/pkg/retry" "hauler.dev/go/hauler/v2/pkg/store" ) @@ -99,6 +104,20 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags return nil } + // Everything below runs with a real store (s != nil; the dry-run branch + // above already returned). Force one durable index checkpoint at the end + // of the run since the per-artifact path only fsyncs on + // indexCheckpointInterval -- deferred so it still runs on error paths, + // where a partially-populated index is worth persisting. This does NOT + // run on Ctrl-C (no signal handler is installed), which is fine: process + // death doesn't lose page cache, so the index still reaches disk. + defer func() { + if err := s.OCI.SaveIndex(); err != nil { + l.Warnf("failed to save index at end of sync: %v", err) + } + l.Debugf("%s", formatIOStats(s.OCI.Stats().Snapshot(), s.OCI.BlobConcurrency())) + }() + tempOverride := rso.TempOverride if tempOverride == "" { @@ -131,7 +150,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags img := v1.Image{ Name: manifestLoc, } - err := storeImage(ctx, s, img, o.Platform, o.ExcludeExtras, rso, ro, "") + err := storeImage(ctx, s, img, o.Platform, o.ExcludeExtras, rso, ro, "", "", false) if err != nil { return fmt.Errorf("failed to fetch product manifest for [%s]: %w", productName, err) } @@ -295,10 +314,9 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor if err := yaml.Unmarshal(doc, &cfg); err != nil { return err } - for _, f := range cfg.Spec.Files { - if err := storeFile(ctx, s, f, ro, rso); err != nil { - return err - } + jobs := resolveFileJobs(cfg.Spec.Files) + if err := runFileJobs(ctx, s, jobs, o.Concurrency, rso, ro, newSyncProgress(o, ro)); err != nil { + return err } default: @@ -314,158 +332,13 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor } a := cfg.GetAnnotations() - for _, i := range cfg.Spec.Images { - - if !i.Local && (a[consts.ImageAnnotationRegistry] != "" || o.Registry != "") { - newRef, _ := reference.Parse(i.Name) - newReg := o.Registry - if o.Registry == "" && a[consts.ImageAnnotationRegistry] != "" { - newReg = a[consts.ImageAnnotationRegistry] - } - if newRef.Context().RegistryStr() == "" { - newRef, err = reference.Relocate(i.Name, newReg) - if err != nil { - return err - } - } - i.Name = newRef.Name() - } - - if i.Local { - needsPubKeyVerification := a[consts.ImageAnnotationKey] != "" || o.Key != "" || i.Key != "" - needsKeylessVerification := a[consts.ImageAnnotationCertIdentityRegexp] != "" || a[consts.ImageAnnotationCertIdentity] != "" || - o.CertIdentityRegexp != "" || o.CertIdentity != "" || - i.CertIdentityRegexp != "" || i.CertIdentity != "" - if needsPubKeyVerification || needsKeylessVerification { - return fmt.Errorf("image [%s]: --local cannot be combined with cosign verification options", i.Name) - } - - rewrite := "" - if i.Rewrite != "" { - rewrite = i.Rewrite - } - if err := storeLocalImage(ctx, s, i, rso, ro, rewrite); err != nil { - return err - } - continue - } - - hasAnnotationIdentityOptions := a[consts.ImageAnnotationCertIdentityRegexp] != "" || a[consts.ImageAnnotationCertIdentity] != "" - hasCliIdentityOptions := o.CertIdentityRegexp != "" || o.CertIdentity != "" - hasImageIdentityOptions := i.CertIdentityRegexp != "" || i.CertIdentity != "" - - needsKeylessVerificaton := hasAnnotationIdentityOptions || hasCliIdentityOptions || hasImageIdentityOptions - needsPubKeyVerification := a[consts.ImageAnnotationKey] != "" || o.Key != "" || i.Key != "" - if needsPubKeyVerification { - key := o.Key - if o.Key == "" && a[consts.ImageAnnotationKey] != "" { - key, err = homedir.Expand(a[consts.ImageAnnotationKey]) - if err != nil { - return err - } - } - if i.Key != "" { - key, err = homedir.Expand(i.Key) - if err != nil { - return err - } - } - l.Debugf("key for image [%s]", key) - - tlog := o.Tlog - if !o.Tlog && a[consts.ImageAnnotationTlog] == "true" { - tlog = true - } - if i.Tlog { - tlog = i.Tlog - } - l.Debugf("transparency log for verification [%t]", tlog) - - if err := cosign.VerifySignature(ctx, key, tlog, i.Name, rso, ro); err != nil { - l.Errorf("signature verification failed for image [%s]... skipping...\n%v", i.Name, err) - continue - } - l.Infof("signature verified for image [%s]", i.Name) - } else if needsKeylessVerificaton { //Keyless signature verification - certIdentityRegexp := o.CertIdentityRegexp - if o.CertIdentityRegexp == "" && a[consts.ImageAnnotationCertIdentityRegexp] != "" { - certIdentityRegexp = a[consts.ImageAnnotationCertIdentityRegexp] - } - if i.CertIdentityRegexp != "" { - certIdentityRegexp = i.CertIdentityRegexp - } - l.Debugf("certIdentityRegexp for image [%s]", certIdentityRegexp) - - certIdentity := o.CertIdentity - if o.CertIdentity == "" && a[consts.ImageAnnotationCertIdentity] != "" { - certIdentity = a[consts.ImageAnnotationCertIdentity] - } - if i.CertIdentity != "" { - certIdentity = i.CertIdentity - } - l.Debugf("certIdentity for image [%s]", certIdentity) - - certOidcIssuer := o.CertOidcIssuer - if o.CertOidcIssuer == "" && a[consts.ImageAnnotationCertOidcIssuer] != "" { - certOidcIssuer = a[consts.ImageAnnotationCertOidcIssuer] - } - if i.CertOidcIssuer != "" { - certOidcIssuer = i.CertOidcIssuer - } - l.Debugf("certOidcIssuer for image [%s]", certOidcIssuer) - - certOidcIssuerRegexp := o.CertOidcIssuerRegexp - if o.CertOidcIssuerRegexp == "" && a[consts.ImageAnnotationCertOidcIssuerRegexp] != "" { - certOidcIssuerRegexp = a[consts.ImageAnnotationCertOidcIssuerRegexp] - } - if i.CertOidcIssuerRegexp != "" { - certOidcIssuerRegexp = i.CertOidcIssuerRegexp - } - l.Debugf("certOidcIssuerRegexp for image [%s]", certOidcIssuerRegexp) - - certGithubWorkflowRepository := o.CertGithubWorkflowRepository - if o.CertGithubWorkflowRepository == "" && a[consts.ImageAnnotationCertGithubWorkflowRepository] != "" { - certGithubWorkflowRepository = a[consts.ImageAnnotationCertGithubWorkflowRepository] - } - if i.CertGithubWorkflowRepository != "" { - certGithubWorkflowRepository = i.CertGithubWorkflowRepository - } - l.Debugf("certGithubWorkflowRepository for image [%s]", certGithubWorkflowRepository) - - // Keyless (Fulcio) certs expire after ~10 min; tlog is always - // required to prove the cert was valid at signing time. - if err := cosign.VerifyKeylessSignature(ctx, certIdentity, certIdentityRegexp, certOidcIssuer, certOidcIssuerRegexp, certGithubWorkflowRepository, i.Name, rso, ro); err != nil { - l.Errorf("signature verification failed for image [%s]... skipping...\n%v", i.Name, err) - continue - } - l.Infof("keyless signature verified for image [%s]", i.Name) - } - platform := o.Platform - if o.Platform == "" && a[consts.ImageAnnotationPlatform] != "" { - platform = a[consts.ImageAnnotationPlatform] - } - if i.Platform != "" { - platform = i.Platform - } - - rewrite := "" - if i.Rewrite != "" { - rewrite = i.Rewrite - } - - excludeExtras := o.ExcludeExtras - if !o.ExcludeExtras && a[consts.ImageAnnotationExcludeExtras] == "true" { - excludeExtras = true - } - if i.ExcludeExtras { - excludeExtras = i.ExcludeExtras - } - - if err := storeImage(ctx, s, i, platform, excludeExtras, rso, ro, rewrite); err != nil { - return err - } + jobs, err := resolveImageJobs(o, a, cfg.Spec.Images) + if err != nil { + return err + } + if err := runImageJobs(ctx, s, jobs, o.Concurrency, rso, ro, newSyncProgress(o, ro)); err != nil { + return err } - s.CopyAll(ctx, s.OCI, nil) default: return fmt.Errorf("unsupported version [%s] for kind [%s]... valid versions are [v1]", gvk.Version, gvk.Kind) @@ -478,70 +351,12 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor if err := yaml.Unmarshal(doc, &cfg); err != nil { return err } - registry := o.Registry - annotation := cfg.GetAnnotations() - if registry == "" { - if annotation != nil { - registry = annotation[consts.ImageAnnotationRegistry] - } + jobs, err := resolveChartJobs(o, cfg.GetAnnotations(), filepath.Dir(fi.Name()), cfg.Spec.Charts) + if err != nil { + return err } - - for i, ch := range cfg.Spec.Charts { - // Resolve excludeExtras: per-chart field > chart manifest annotation > CLI flag. - excludeExtras := o.ExcludeExtras - if !o.ExcludeExtras && annotation != nil && annotation[consts.ImageAnnotationExcludeExtras] == "true" { - excludeExtras = true - } - if ch.ExcludeExtras { - excludeExtras = ch.ExcludeExtras - } - - var valuesFiles []string - for _, path := range ch.ValuesFiles { - valuesFiles = append(valuesFiles, filepath.Join(filepath.Dir(fi.Name()), path)) - } - - platform := o.Platform - if annotation != nil && annotation[consts.ImageAnnotationPlatform] != "" { - platform = annotation[consts.ImageAnnotationPlatform] - } - if ch.Platform != "" { - platform = ch.Platform - } - - chartUsername, chartPassword, err := resolveChartCreds(ch) - if err != nil { - return err - } - - if err := storeChart(ctx, s, ch, - &flags.AddChartOpts{ - ChartOpts: &action.ChartPathOptions{ - RepoURL: ch.RepoURL, - Version: ch.Version, - Verify: ch.Verify, - Keyring: ch.Keyring, - Username: chartUsername, - Password: chartPassword, - PassCredentialsAll: ch.PassCredentialsAll, - CertFile: ch.CertFile, - KeyFile: ch.KeyFile, - CaFile: ch.CaFile, - InsecureSkipTLSVerify: ch.InsecureSkipTLSVerify, - PlainHTTP: ch.PlainHTTP, - }, - AddImages: ch.AddImages, - AddDependencies: ch.AddDependencies, - ExcludeExtras: excludeExtras, - Registry: registry, - Platform: platform, - ValuesFiles: valuesFiles, - }, - rso, ro, - cfg.Spec.Charts[i].Rewrite, - ); err != nil { - return err - } + if err := runChartJobs(ctx, s, jobs, o.Concurrency, rso, ro, newSyncProgress(o, ro)); err != nil { + return err } default: @@ -577,17 +392,709 @@ func resolveChartCreds(ch v1.Chart) (username, password string, err error) { func processImageTxt(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) l.Infof("syncing images from [%s] to store", filepath.Base(fi.Name())) + var jobs []imageJob scanner := bufio.NewScanner(fi) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } - img := v1.Image{Name: line} - l.Infof("adding image [%s] to the store [%s]", line, o.StoreDir) - if err := storeImage(ctx, s, img, o.Platform, o.ExcludeExtras, rso, ro, ""); err != nil { + l.Debugf("adding image [%s] to the store [%s]", line, o.StoreDir) + jobs = append(jobs, imageJob{ + img: v1.Image{Name: line}, + platform: o.Platform, + excludeExtras: o.ExcludeExtras, + }) + } + if err := scanner.Err(); err != nil { + return err + } + return runImageJobs(ctx, s, jobs, o.Concurrency, rso, ro, newSyncProgress(o, ro)) +} + +// newSyncProgress returns a live progress Renderer over os.Stdout when +// eligible (see log.ShouldShowProgress), or nil otherwise; runImageJobs +// treats a nil progress as a no-op. +// +// The session spans verification, which runs inside the pull worker. A live +// session survives a concurrent log.CaptureOutput regardless: log.NewLogger +// binds its writer once at construction (pkg/log/log.go) and the Renderer holds +// the real *os.File, so CaptureOutput's swap of the os.Stdout/os.Stderr package +// variables reaches neither. runChartJobs depends on that, running its Helm +// capture inside a live session. +func newSyncProgress(o *flags.SyncOpts, ro *flags.CliRootOpts) *log.Renderer { + return newProgressRenderer(o.NoProgress, ro.LogLevel) +} + +// newProgressRenderer returns a live progress Renderer when the run is +// eligible (see log.ShouldShowProgress), or nil otherwise; the run* helpers +// treat nil as "no progress display". +func newProgressRenderer(noProgress bool, logLevel string) *log.Renderer { + if !log.ShouldShowProgress(noProgress, logLevel) { + return nil + } + return log.NewRenderer(os.Stdout) +} + +// formatIOStats renders one line summarizing a sync's disk contention. +// ceiling is the store's configured blob-write limit, so peak-inflight +// reads as a fraction of what was permitted rather than a bare number. +// +// blobs/written/cached/bytes cover only the WriteBlob path (store.AddImage +// and friends); a registry push shares the same blob semaphore without +// calling WriteBlob, so those counters can read zero on a store-to-store +// copy. blobsem-wait sums wait time across every goroutine that touched the +// semaphore, not wall-clock, so it can exceed the run's total duration. +func formatIOStats(st content.IOStatsSnapshot, ceiling int) string { + return fmt.Sprintf( + "io stats: blobs=%d written=%d cached=%d bytes=%s peak-inflight=%d/%d blobsem-wait=%s index-writes=%d durable=%d index-bytes=%s index-lock-wait=%s", + st.BlobsWritten+st.BlobsCached, + st.BlobsWritten, + st.BlobsCached, + humanize.Bytes(uint64(st.BlobBytesWritten)), + st.BlobPeakInFlight, + ceiling, + st.BlobSemWait.Round(time.Millisecond), + st.IndexWrites, + st.IndexDurableWrites, + humanize.Bytes(uint64(st.IndexBytesWritten)), + st.IndexLockWait.Round(time.Millisecond), + ) +} + +// imageJob is the fully-resolved set of inputs needed to verify (if +// applicable) and store a single image; see resolveImageJobs. +type imageJob struct { + img v1.Image // Name already relocated to the target registry if applicable + platform string + excludeExtras bool + rewrite string + local bool + + // resolved verification inputs, collapsed into a cosign.Config by + // verifyConfig and consumed by the pull worker + needsPubKey, needsKeyless bool + key string + tlog bool + certIdentity, certIdentityRegexp string + certOidcIssuer, certOidcIssuerRegexp string + certGithubWorkflowRepository string +} + +// resolveImageJobs applies the precedence rules (per-image > annotation > +// CLI, except registry relocation which is CLI > annotation) to every image +// in images, producing one imageJob per image. It is pure -- cosign +// verification happens later, inside the pull worker; see resolveAndVerify. +func resolveImageJobs(o *flags.SyncOpts, a map[string]string, images []v1.Image) ([]imageJob, error) { + var jobs []imageJob + + for _, i := range images { + if !i.Local && (a[consts.ImageAnnotationRegistry] != "" || o.Registry != "") { + newRef, _ := reference.Parse(i.Name) + newReg := o.Registry + if o.Registry == "" && a[consts.ImageAnnotationRegistry] != "" { + newReg = a[consts.ImageAnnotationRegistry] + } + if newRef.Context().RegistryStr() == "" { + var relErr error + newRef, relErr = reference.Relocate(i.Name, newReg) + if relErr != nil { + return nil, relErr + } + } + i.Name = newRef.Name() + } + + if i.Local { + needsPubKeyVerification := a[consts.ImageAnnotationKey] != "" || o.Key != "" || i.Key != "" + needsKeylessVerification := a[consts.ImageAnnotationCertIdentityRegexp] != "" || a[consts.ImageAnnotationCertIdentity] != "" || + o.CertIdentityRegexp != "" || o.CertIdentity != "" || + i.CertIdentityRegexp != "" || i.CertIdentity != "" + if needsPubKeyVerification || needsKeylessVerification { + return nil, fmt.Errorf("image [%s]: --local cannot be combined with cosign verification options", i.Name) + } + + rewrite := "" + if i.Rewrite != "" { + rewrite = i.Rewrite + } + jobs = append(jobs, imageJob{img: i, local: true, rewrite: rewrite}) + continue + } + + hasAnnotationIdentityOptions := a[consts.ImageAnnotationCertIdentityRegexp] != "" || a[consts.ImageAnnotationCertIdentity] != "" + hasCliIdentityOptions := o.CertIdentityRegexp != "" || o.CertIdentity != "" + hasImageIdentityOptions := i.CertIdentityRegexp != "" || i.CertIdentity != "" + + needsKeylessVerificaton := hasAnnotationIdentityOptions || hasCliIdentityOptions || hasImageIdentityOptions + needsPubKeyVerification := a[consts.ImageAnnotationKey] != "" || o.Key != "" || i.Key != "" + + job := imageJob{img: i} + + if needsPubKeyVerification { + key := o.Key + if o.Key == "" && a[consts.ImageAnnotationKey] != "" { + expanded, err := homedir.Expand(a[consts.ImageAnnotationKey]) + if err != nil { + return nil, err + } + key = expanded + } + if i.Key != "" { + expanded, err := homedir.Expand(i.Key) + if err != nil { + return nil, err + } + key = expanded + } + + tlog := o.Tlog + if !o.Tlog && a[consts.ImageAnnotationTlog] == "true" { + tlog = true + } + if i.Tlog { + tlog = i.Tlog + } + + job.needsPubKey = true + job.key = key + job.tlog = tlog + } else if needsKeylessVerificaton { //Keyless signature verification + certIdentityRegexp := o.CertIdentityRegexp + if o.CertIdentityRegexp == "" && a[consts.ImageAnnotationCertIdentityRegexp] != "" { + certIdentityRegexp = a[consts.ImageAnnotationCertIdentityRegexp] + } + if i.CertIdentityRegexp != "" { + certIdentityRegexp = i.CertIdentityRegexp + } + + certIdentity := o.CertIdentity + if o.CertIdentity == "" && a[consts.ImageAnnotationCertIdentity] != "" { + certIdentity = a[consts.ImageAnnotationCertIdentity] + } + if i.CertIdentity != "" { + certIdentity = i.CertIdentity + } + + certOidcIssuer := o.CertOidcIssuer + if o.CertOidcIssuer == "" && a[consts.ImageAnnotationCertOidcIssuer] != "" { + certOidcIssuer = a[consts.ImageAnnotationCertOidcIssuer] + } + if i.CertOidcIssuer != "" { + certOidcIssuer = i.CertOidcIssuer + } + + certOidcIssuerRegexp := o.CertOidcIssuerRegexp + if o.CertOidcIssuerRegexp == "" && a[consts.ImageAnnotationCertOidcIssuerRegexp] != "" { + certOidcIssuerRegexp = a[consts.ImageAnnotationCertOidcIssuerRegexp] + } + if i.CertOidcIssuerRegexp != "" { + certOidcIssuerRegexp = i.CertOidcIssuerRegexp + } + + certGithubWorkflowRepository := o.CertGithubWorkflowRepository + if o.CertGithubWorkflowRepository == "" && a[consts.ImageAnnotationCertGithubWorkflowRepository] != "" { + certGithubWorkflowRepository = a[consts.ImageAnnotationCertGithubWorkflowRepository] + } + if i.CertGithubWorkflowRepository != "" { + certGithubWorkflowRepository = i.CertGithubWorkflowRepository + } + + job.needsKeyless = true + job.certIdentity = certIdentity + job.certIdentityRegexp = certIdentityRegexp + job.certOidcIssuer = certOidcIssuer + job.certOidcIssuerRegexp = certOidcIssuerRegexp + job.certGithubWorkflowRepository = certGithubWorkflowRepository + } + + platform := o.Platform + if o.Platform == "" && a[consts.ImageAnnotationPlatform] != "" { + platform = a[consts.ImageAnnotationPlatform] + } + if i.Platform != "" { + platform = i.Platform + } + + rewrite := "" + if i.Rewrite != "" { + rewrite = i.Rewrite + } + + excludeExtras := o.ExcludeExtras + if !o.ExcludeExtras && a[consts.ImageAnnotationExcludeExtras] == "true" { + excludeExtras = true + } + if i.ExcludeExtras { + excludeExtras = i.ExcludeExtras + } + + job.platform = platform + job.rewrite = rewrite + job.excludeExtras = excludeExtras + + jobs = append(jobs, job) + } + + return jobs, nil +} + +// verifyConfig collapses j's resolved verification inputs into the key +// cosign.Cache uses to share one Verifier -- and therefore one trust-material +// setup -- across every image with identical settings. +// +// The branch mirrors resolveImageJobs' own exclusive key-then-keyless +// precedence rather than forwarding whatever fields happen to be set. A +// manifest naming both a key and an identity has always verified against the +// key alone, and cosign.Config.validate rejects that pairing outright, so +// building the Config from the raw inputs would turn a working manifest into a +// hard error. +// +// It returns the zero Config -- the one cosign.Config.Empty reports -- exactly +// when neither flag is set, which is what keeps the "does this image verify?" +// gate identical to the one the old batch pass used. +func (j imageJob) verifyConfig() cosign.Config { + switch { + case j.needsPubKey: + return cosign.Config{Key: j.key, Tlog: j.tlog} + case j.needsKeyless: + return cosign.Config{ + CertIdentity: j.certIdentity, + CertIdentityRegexp: j.certIdentityRegexp, + CertOidcIssuer: j.certOidcIssuer, + CertOidcIssuerRegexp: j.certOidcIssuerRegexp, + CertGithubWorkflowRepository: j.certGithubWorkflowRepository, + } + default: + return cosign.Config{} + } +} + +// resolveAndVerify pins j's tag to a digest and verifies that exact digest, +// returning the digest for storeImage to fetch. +// +// Resolving here rather than in a prior pass is the point of the change: a +// batch verify pass left the whole pass's duration between checking a tag and +// pulling it, during which the tag could move. Verifying the digest and handing +// the same digest to storeImage closes that window -- the bytes stored are the +// bytes checked. +// +// A job that requested no verification is not resolved at all. There is no +// window to close when nothing is checked, and an unconditional HEAD would add +// a registry round trip per image to the overwhelmingly common unsigned case. +// The empty digest it returns leaves storeImage resolving the tag as before. +// +// Every error it returns is a *verifyError, so the caller can say which of the +// four steps failed instead of blaming them all on the signature. The two +// post-pin failure branches (cache.Get, v.Verify) return the pinned digest +// alongside the error, not "": under --ignore-errors the caller stores the +// image anyway, and it must store the exact bytes that were checked even +// though the check failed, not let storeImage re-resolve the tag. The +// pre-pin branches (a bad reference, or the pin itself failing) have no +// digest to give back. +func resolveAndVerify(ctx context.Context, cache *cosign.Cache, j imageJob, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) (string, error) { + cfg := j.verifyConfig() + if cfg.Empty() { + return "", nil + } + logVerifyInputs(ctx, j) + + ref, err := gname.ParseReference(j.img.Name) + if err != nil { + return "", &verifyError{stage: "unable to parse image reference", err: err} + } + + pinned, err := pinDigest(ctx, ref, rso, ro) + if err != nil { + return "", &verifyError{stage: "unable to resolve image digest", err: err} + } + + // ctx is run-scoped (the errgroup's), never a per-image timeout, which + // cosign.Cache.Get requires: the ctx of whichever goroutine finds cfg cold + // ends up inside the registry options every image sharing cfg then uses. + v, err := cache.Get(ctx, cfg) + if err != nil { + return pinned, &verifyError{stage: "unable to configure signature verification", err: err} + } + // Verify applies --retries itself, discriminating the errors a retry may + // touch; see cosign.Verifier.verifyImage. + if err := v.Verify(ctx, ref.Context().Digest(pinned).Name()); err != nil { + return pinned, &verifyError{stage: "signature verification failed", err: err} + } + + if cfg.Keyless() { + log.BaseFromContext(ctx).Infof("✓ keyless signature verified for image [%s]", j.img.Name) + } else { + log.BaseFromContext(ctx).Infof("✓ signature verified for image [%s]", j.img.Name) + } + return pinned, nil +} + +// pinDigest resolves ref to the digest its tag currently names, under the +// caller's --retries budget. Every caller that verifies before storing goes +// through it, so the pin is retried on exactly one code path. +// +// The pin is the one network call on the verify path that a transient blip can +// lose a *valid, signed* image to: in a sync a bare failure here drops the image +// and the run still exits 0, which reads to the user as silent data loss. It +// gets the same --retries budget the verify and store steps have. +// retry.Operation checks ctx before every attempt and aborts its backoff on +// cancellation, so a cancelled run still fails fast rather than sleeping out +// the budget. +func pinDigest(ctx context.Context, ref gname.Reference, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) (string, error) { + var pinned string + err := retry.Operation(ctx, rso, ro, func() error { + desc, headErr := remote.Head(ref, + remote.WithAuthFromKeychain(authn.DefaultKeychain), + remote.WithContext(ctx), + ) + if headErr != nil { + return headErr + } + pinned = desc.Digest.String() + return nil + }) + if err != nil { + return "", err + } + return pinned, nil +} + +// verifyError names which step of resolveAndVerify failed. A bad reference, an +// unreachable registry, an unreadable key, and a signature that did not check +// out are four different problems, and reporting all of them as "signature +// verification failed" tells users with a network fault that they have a +// signing fault. +// +// stage reads as the head of " for image []". +type verifyError struct { + stage string + err error +} + +func (e *verifyError) Error() string { return e.stage + ": " + e.err.Error() } +func (e *verifyError) Unwrap() error { return e.err } + +// logVerifyFailure reports err against ref and reports whether the caller +// should propagate it (fail the run) rather than proceed to storeImage with +// whatever digest resolveAndVerify already pinned. +// +// context.Canceled always propagates and logs at DEBUG, regardless of +// ignoreErrors: under the errgroup's fail-fast, one real storeImage failure +// cancels gctx and every other in-flight job lands here with a +// context.Canceled that has nothing to do with its own image. Logging those +// at ERROR would bury the single real failure under N-1 lines claiming +// signature problems the user does not have, and under --ignore-errors, +// treating a cancellation as an ordinary ignorable failure would store an +// image whose bytes were never actually checked because the run was already +// being torn down. +// +// Every other failure's fate depends on ignoreErrors: without it, this fails +// the run (ERROR, propagate=true) -- a reversal of the old behavior of +// dropping just this one image, made because a dropped signature failure let +// a manifest sync report success while quietly missing a scheduled image. +// With it, this logs a WARN and does not propagate, so the caller falls +// through to storeImage with whatever digest resolveAndVerify already pinned +// (possibly none, if the failure happened before pinning). This function only +// reports the verification outcome; it makes no claim about what storeImage +// does next -- storeImage has its own ignoreErrors handling and logs its own +// success or skip line immediately after. An unverified image reaching the +// store (and potentially an airgapped environment) is what --ignore-errors +// buys once verification is involved, not a bug to guard against. +func logVerifyFailure(l log.Logger, ref string, err error, ignoreErrors bool) bool { + stage := "verification failed" + cause := err + var ve *verifyError + if errors.As(err, &ve) { + stage = ve.stage + cause = ve.err + } + if errors.Is(err, context.Canceled) { + l.Debugf("%s for image [%s]: %s", stage, ref, flattenVerifyError(cause)) + return true + } + if ignoreErrors { + l.Warnf("⚠ %s for image [%s]: %s", stage, ref, flattenVerifyError(cause)) + return false + } + l.Errorf("✗ %s for image [%s]: %s... aborting...", stage, ref, flattenVerifyError(cause)) + return true +} + +// flattenVerifyError renders err as one line. cosign's ErrNoMatchingSignatures +// joins one failure sentence per signature-verification attempt with "\n ", +// so a single failed image can carry the identical sentence repeated several +// times in a row; collapsing consecutive repeats keeps the log line from +// restating the same cause N times. +func flattenVerifyError(err error) string { + if err == nil { + return "" + } + + var fragments []string + for _, line := range strings.Split(err.Error(), "\n") { + if f := strings.TrimSpace(line); f != "" { + fragments = append(fragments, f) + } + } + + var out []string + for i := 0; i < len(fragments); { + j := i + 1 + for j < len(fragments) && fragments[j] == fragments[i] { + j++ + } + if n := j - i; n > 1 { + out = append(out, fmt.Sprintf("%s (x%d)", fragments[i], n)) + } else { + out = append(out, fragments[i]) + } + i = j + } + + return strings.Join(out, "; ") +} + +// logVerifyInputs echoes j's resolved verification settings at debug, so a run +// started against the wrong key or identity is diagnosable from --log-level +// debug alone. One line per job rather than one per field: at sync concurrency +// the per-field lines interleaved into an unreadable stream. +func logVerifyInputs(ctx context.Context, j imageJob) { + // The ref is named inline, so this takes the unadorned base logger and not + // the per-job one that would append a duplicating "image=" field -- the + // same convention storeImage's completion line follows. + l := log.BaseFromContext(ctx) + switch { + case j.needsPubKey: + l.Debugf("verifying image [%s] with key [%s] and transparency log [%t]", j.img.Name, j.key, j.tlog) + case j.needsKeyless: + l.Debugf("verifying image [%s] keylessly with certIdentity [%s] certIdentityRegexp [%s] certOidcIssuer [%s] certOidcIssuerRegexp [%s] certGithubWorkflowRepository [%s]", + j.img.Name, j.certIdentity, j.certIdentityRegexp, j.certOidcIssuer, j.certOidcIssuerRegexp, j.certGithubWorkflowRepository) + } +} + +// runImageJobs stores every job, local Docker daemon images first +// (serially), then remote images concurrently (bounded by concurrency). +// Local jobs run through storeLocalImage, whose ensureDockerHost mutates +// the process-wide environment via os.Setenv, and all contend on one +// Docker daemon anyway -- nothing to gain, and a mutation race to lose, +// from running them concurrently. +// +// This is the progress-session-owning wrapper around +// runRemoteImageJobsWith. The local pass deliberately runs before +// progress.Start(): storeLocalImage logs through the ambient logger, not +// the Renderer, so running it inside a live session would interleave its +// output with the Renderer's erase/redraw cycle and corrupt the display. +func runImageJobs(ctx context.Context, s *store.Layout, jobs []imageJob, concurrency int, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, progress *log.Renderer) error { + l := log.FromContext(ctx) + + var localJobs, remoteJobs []imageJob + for _, j := range jobs { + if j.local { + localJobs = append(localJobs, j) + } else { + remoteJobs = append(remoteJobs, j) + } + } + + for _, j := range localJobs { + if err := storeLocalImage(ctx, s, j.img, rso, ro, j.rewrite); err != nil { return err } } - return scanner.Err() + + // baseLogger is the logger every job's per-image logger is derived from. + // When progress is active, it's built over the Renderer instead, so + // every log line for a job -- including its "✓ added ..." line and any + // errors -- flows through the Renderer's erase/write/redraw path rather + // than writing straight to the ambient logger's destination. + baseLogger := l + if progress != nil && len(remoteJobs) > 0 { + baseLogger = log.NewLogger(progress) + progress.Start() + defer progress.Stop() + } + + return runRemoteImageJobsWith(ctx, s, remoteJobs, concurrency, rso, ro, progress, baseLogger) +} + +// runRemoteImageJobsWith stores remote image jobs concurrently, bounded by +// concurrency, inside a progress session the caller already started (or nil) +// and against a baseLogger the caller already derived. It never calls +// Start/Stop, so one caller can span a single session across several phases. +// jobs must be remote-only -- runImageJobs runs the local Docker pass itself, +// before the session opens. +// +// errgroup.WithContext + SetLimit(concurrency) gives both fail-fast and +// --ignore-errors semantics: with --ignore-errors storeImage warns and +// returns nil, so g.Wait() never observes an error; otherwise a failing job +// cancels the group's derived context, which every other in-flight storeImage +// call observes via content.OCI.WriteBlob's context-aware writes, and g.Wait() +// returns that one real error, not an aggregate. +// +// Each job resolves, verifies, and stores in one goroutine rather than across +// separate passes, so verification runs at the same concurrency as the pulls +// and no time passes between checking a tag and fetching it. +func runRemoteImageJobsWith(ctx context.Context, s *store.Layout, jobs []imageJob, concurrency int, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, progress *log.Renderer, baseLogger log.Logger) error { + if concurrency < 1 { + concurrency = 1 + } + + // The cache is per-call rather than per-run so its lifetime is bracketed by + // the g.Wait() below: Verifier.Close must not run while a Verify is still + // in flight. One call covers one document's images, which is where the + // sharing matters -- a Rancher manifest's 880 images build trust material + // once between them, not 880 times. A manifest file holding several Images + // documents builds it once per document. + cache := cosign.NewCache(rso, ro) + defer cache.Close() + + ignoreErrors := flags.ShouldIgnoreErrors(ro) + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + for _, j := range jobs { + g.Go(func() error { + // Began must be called from inside the goroutine, after + // g.Go's semaphore acquisition, not from the outer loop: + // g.Go blocks the outer loop until a concurrency slot is + // free, so this is the earliest point the job has actually + // started. Calling Began any earlier would mark a queued + // job "in flight" while it's still waiting for a slot. + // + // Finished is deferred so the verify-failure return below + // clears the row too; a live region that keeps drawing a + // dropped image never stops. + if progress != nil { + progress.Began(j.img.Name) + defer progress.Finished(j.img.Name) + } + jl := baseLogger.With(log.Fields{"image": j.img.Name}) + jctx := jl.WithContext(gctx) + // storeImage lines that already name their ref inline (e.g. + // its "✓ added ..." line) fetch this unadorned base + // logger via log.BaseFromContext instead of log.FromContext, + // so they don't duplicate the ref with the "image=..." field + // below. Lines that don't name the ref (retry.Operation's + // warnings, store-layer debug output) keep calling + // log.FromContext(ctx) and keep the field for attribution. + jctx = log.WithBaseLogger(jctx, baseLogger) + + pinned, err := resolveAndVerify(jctx, cache, j, rso, ro) + if err != nil { + // A verification failure either fails the run or is stored + // unverified, depending on --ignore-errors -- see + // logVerifyFailure's doc for the exact rule, including the + // context.Canceled case that overrides both. propagate=false + // falls through to the same storeImage call the success path + // uses, with whatever digest resolveAndVerify already pinned + // (or "" if the failure happened before the pin). + if propagate := logVerifyFailure(baseLogger, j.img.Name, err, ignoreErrors); propagate { + return err + } + } + // verified is true only when verification was both requested + // and succeeded: err is nil on success and stays non-nil on a + // failure that fell through to here under --ignore-errors. + verified := err == nil && !j.verifyConfig().Empty() + return storeImage(jctx, s, j.img, j.platform, j.excludeExtras, rso, ro, j.rewrite, pinned, verified) + }) + } + return g.Wait() +} + +// fileJob is the fully-resolved set of inputs needed to store a single +// file; see resolveFileJobs. Unlike imageJob, there's no verification pass. +type fileJob struct { + file v1.File +} + +// resolveFileJobs converts every v1.File in files into a fileJob. It is +// pure. +func resolveFileJobs(files []v1.File) []fileJob { + jobs := make([]fileJob, 0, len(files)) + for _, f := range files { + jobs = append(jobs, fileJob{file: f}) + } + return jobs +} + +// fileJobName returns the identifier used for a file job's progress row and +// per-job log field: the name override when set (matching the ref +// storeFile/reference.NewTagged will actually derive), otherwise the raw +// source path. +func fileJobName(f v1.File) string { + if f.Name != "" { + return f.Name + } + return f.Path +} + +// runFileJobs stores every job concurrently, bounded by concurrency, with +// no local/remote partitioning by scheme: unlike images, no file source +// contends on a shared process-wide resource the way storeLocalImage's +// Docker-daemon path does (see runImageJobs), so file://, directory://, and +// http(s):// sources all run through the same errgroup. pkg/content's OCI +// store blob semaphore already bounds total in-flight blob writes +// regardless of scheme. +// +// Every job shares one *file.LayerCache (pkg/artifacts/file/cache.go), +// attached to each job's context, so two Files entries with the identical +// source Path -- e.g. the same URL listed twice, once plain and once with a +// name override, as testdata/hauler-manifest-pipeline.yaml does -- fetch +// the content exactly once regardless of concurrency, rather than once per +// manifest entry. +// +// Fail-fast and --ignore-errors semantics mirror runImageJobs exactly. +// +// Like runImageJobs, this is only the progress-session-owning wrapper; the +// work lives in runFileJobsWith. +func runFileJobs(ctx context.Context, s *store.Layout, jobs []fileJob, concurrency int, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, progress *log.Renderer) error { + l := log.FromContext(ctx) + + // baseLogger is the logger every job's per-file logger is derived from -- + // see runImageJobs's identical baseLogger for the full rationale. + baseLogger := l + if progress != nil && len(jobs) > 0 { + baseLogger = log.NewLogger(progress) + progress.Start() + defer progress.Stop() + } + + return runFileJobsWith(ctx, s, jobs, concurrency, rso, ro, progress, baseLogger) +} + +// runFileJobsWith stores file jobs concurrently, bounded by concurrency, +// inside a progress session the caller already started (or nil) and against a +// baseLogger the caller already derived. It never calls Start/Stop -- see +// runRemoteImageJobsWith, its image-side counterpart. +func runFileJobsWith(ctx context.Context, s *store.Layout, jobs []fileJob, concurrency int, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, progress *log.Renderer, baseLogger log.Logger) error { + if concurrency < 1 { + concurrency = 1 + } + + cache := file.NewLayerCache() + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + for _, j := range jobs { + g.Go(func() error { + name := fileJobName(j.file) + // Began must be called after g.Go's semaphore acquisition; + // see runRemoteImageJobsWith. + if progress != nil { + progress.Began(name) + } + jl := baseLogger.With(log.Fields{"file": name}) + jctx := jl.WithContext(gctx) + jctx = log.WithBaseLogger(jctx, baseLogger) + jctx = file.WithLayerCacheContext(jctx, cache) + err := storeFile(jctx, s, j.file, ro, rso) + if progress != nil { + progress.Finished(name) + } + return err + }) + } + return g.Wait() } diff --git a/cmd/hauler/cli/store/sync_test.go b/cmd/hauler/cli/store/sync_test.go index 11d3507..9f2d98b 100644 --- a/cmd/hauler/cli/store/sync_test.go +++ b/cmd/hauler/cli/store/sync_test.go @@ -1,18 +1,31 @@ package store import ( + "bytes" + "context" + "errors" "fmt" "io" + "net" "net/http" "net/http/httptest" "os" + "path/filepath" + "sort" "strings" + "sync" + "sync/atomic" "testing" + "time" + + "github.com/mitchellh/go-homedir" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" gcrv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/static" gvtypes "github.com/google/go-containerregistry/pkg/v1/types" @@ -22,6 +35,10 @@ import ( "hauler.dev/go/hauler/v2/internal/flags" v1 "hauler.dev/go/hauler/v2/pkg/apis/hauler.cattle.io/v1" "hauler.dev/go/hauler/v2/pkg/consts" + "hauler.dev/go/hauler/v2/pkg/content" + "hauler.dev/go/hauler/v2/pkg/cosign" + "hauler.dev/go/hauler/v2/pkg/log" + "hauler.dev/go/hauler/v2/pkg/store" ) // writeManifestFile writes yamlContent to a temp file, seeks back to the @@ -648,3 +665,2225 @@ spec: t.Errorf("dry-run stdout missing manifest name 'testproduct-files'; got:\n%s", got) } } + +// -------------------------------------------------------------------------- +// resolveImageJobs tests +// +// resolveImageJobs is a pure function (no ctx, no I/O) so these tests exercise +// the ~150 lines of precedence-resolution logic directly, without a store or +// network access. +// -------------------------------------------------------------------------- + +func TestResolveImageJobs_RegistryRelocation(t *testing.T) { + tests := []struct { + name string + imageName string + local bool + cliRegistry string + annotation string + wantName string + }{ + { + name: "CLI flag wins over annotation", + imageName: "rancher/rancher:v2.9", + cliRegistry: "cli-registry.io", + annotation: "annotation-registry.io", + wantName: "cli-registry.io/rancher/rancher:v2.9", + }, + { + name: "annotation used when no CLI flag", + imageName: "rancher/rancher:v2.9", + annotation: "annotation-registry.io", + wantName: "annotation-registry.io/rancher/rancher:v2.9", + }, + { + name: "relocation skipped when ref already carries a registry", + imageName: "ghcr.io/rancher/rancher:v2.9", + cliRegistry: "cli-registry.io", + wantName: "ghcr.io/rancher/rancher:v2.9", + }, + { + name: "relocation skipped entirely when Local is true", + imageName: "rancher/rancher:v2.9", + local: true, + cliRegistry: "cli-registry.io", + wantName: "rancher/rancher:v2.9", + }, + { + name: "no registry flag or annotation leaves name unchanged", + imageName: "rancher/rancher:v2.9", + wantName: "rancher/rancher:v2.9", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{Registry: tc.cliRegistry} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationRegistry] = tc.annotation + } + images := []v1.Image{{Name: tc.imageName, Local: tc.local}} + + jobs, err := resolveImageJobs(o, a, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if got := jobs[0].img.Name; got != tc.wantName { + t.Errorf("got name %q, want %q", got, tc.wantName) + } + }) + } +} + +func TestResolveImageJobs_LocalWithVerificationOptions_ReturnsError(t *testing.T) { + tests := []struct { + name string + o *flags.SyncOpts + a map[string]string + image v1.Image + }{ + { + name: "key via CLI", + o: &flags.SyncOpts{Key: "/some/key.pub"}, + a: map[string]string{}, + image: v1.Image{Name: "rancher/rancher:v2.9", Local: true}, + }, + { + name: "key via annotation", + o: &flags.SyncOpts{}, + a: map[string]string{consts.ImageAnnotationKey: "/some/key.pub"}, + image: v1.Image{Name: "rancher/rancher:v2.9", Local: true}, + }, + { + name: "key via per-image", + o: &flags.SyncOpts{}, + a: map[string]string{}, + image: v1.Image{Name: "rancher/rancher:v2.9", Local: true, Key: "/some/key.pub"}, + }, + { + name: "identity via CLI", + o: &flags.SyncOpts{CertIdentity: "someone@example.com"}, + a: map[string]string{}, + image: v1.Image{Name: "rancher/rancher:v2.9", Local: true}, + }, + { + name: "identity via annotation", + o: &flags.SyncOpts{}, + a: map[string]string{consts.ImageAnnotationCertIdentity: "someone@example.com"}, + image: v1.Image{Name: "rancher/rancher:v2.9", Local: true}, + }, + { + name: "identity via per-image", + o: &flags.SyncOpts{}, + a: map[string]string{}, + image: v1.Image{Name: "rancher/rancher:v2.9", Local: true, CertIdentity: "someone@example.com"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + jobs, err := resolveImageJobs(tc.o, tc.a, []v1.Image{tc.image}) + if err == nil { + t.Fatalf("expected error, got nil (jobs=%v)", jobs) + } + if !strings.Contains(err.Error(), "--local cannot be combined with cosign verification options") { + t.Errorf("unexpected error message: %v", err) + } + if len(jobs) != 0 { + t.Errorf("expected no jobs appended, got %d", len(jobs)) + } + }) + } +} + +func TestResolveImageJobs_KeyPrecedence(t *testing.T) { + homeKey, err := homedir.Expand("~/mykey.pub") + if err != nil { + t.Fatalf("homedir.Expand: %v", err) + } + + tests := []struct { + name string + cliKey string + annotation string + imageKey string + wantKey string + }{ + { + name: "CLI only", + cliKey: "/cli/key.pub", + wantKey: "/cli/key.pub", + }, + { + // Annotation only applies when the CLI key is unset — it does not + // override an explicitly-set CLI key. + name: "annotation used when CLI key unset, expanded via homedir", + annotation: "~/mykey.pub", + wantKey: homeKey, + }, + { + name: "per-image overrides annotation and CLI, expanded via homedir", + cliKey: "/cli/key.pub", + annotation: "/annotation/key.pub", + imageKey: "~/mykey.pub", + wantKey: homeKey, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{Key: tc.cliKey} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationKey] = tc.annotation + } + images := []v1.Image{{Name: "rancher/rancher:v2.9", Key: tc.imageKey}} + + jobs, err := resolveImageJobs(o, a, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + job := jobs[0] + if !job.needsPubKey { + t.Fatalf("expected needsPubKey=true") + } + if job.key != tc.wantKey { + t.Errorf("got key %q, want %q", job.key, tc.wantKey) + } + }) + } +} + +func TestResolveImageJobs_TlogPrecedence(t *testing.T) { + tests := []struct { + name string + cliTlog bool + annotation string + imageTlog bool + wantTlog bool + }{ + { + name: "CLI true", + cliTlog: true, + wantTlog: true, + }, + { + name: "annotation true overrides CLI false", + annotation: "true", + wantTlog: true, + }, + { + name: "per-image true overrides annotation/CLI false", + imageTlog: true, + wantTlog: true, + }, + { + name: "all false stays false", + wantTlog: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{Key: "/cli/key.pub", Tlog: tc.cliTlog} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationTlog] = tc.annotation + } + images := []v1.Image{{Name: "rancher/rancher:v2.9", Tlog: tc.imageTlog}} + + jobs, err := resolveImageJobs(o, a, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if jobs[0].tlog != tc.wantTlog { + t.Errorf("got tlog %v, want %v", jobs[0].tlog, tc.wantTlog) + } + }) + } +} + +func TestResolveImageJobs_PlatformPrecedence(t *testing.T) { + tests := []struct { + name string + cliPlatform string + annotation string + imagePlatform string + want string + }{ + {name: "CLI only", cliPlatform: "linux/amd64", want: "linux/amd64"}, + // Annotation only applies when the CLI platform is unset — it does not + // override an explicitly-set CLI platform. + {name: "annotation used when CLI platform unset", annotation: "linux/arm64", want: "linux/arm64"}, + {name: "per-image overrides annotation and CLI", cliPlatform: "linux/amd64", annotation: "linux/arm64", imagePlatform: "linux/386", want: "linux/386"}, + {name: "none set stays empty", want: ""}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{Platform: tc.cliPlatform} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationPlatform] = tc.annotation + } + images := []v1.Image{{Name: "rancher/rancher:v2.9", Platform: tc.imagePlatform}} + + jobs, err := resolveImageJobs(o, a, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if jobs[0].platform != tc.want { + t.Errorf("got platform %q, want %q", jobs[0].platform, tc.want) + } + }) + } +} + +func TestResolveImageJobs_RewritePrecedence(t *testing.T) { + tests := []struct { + name string + imageRewrite string + want string + }{ + {name: "no per-image rewrite stays empty", want: ""}, + {name: "per-image rewrite propagates", imageRewrite: "myregistry.io/rancher", want: "myregistry.io/rancher"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{} + a := map[string]string{} + images := []v1.Image{{Name: "rancher/rancher:v2.9", Rewrite: tc.imageRewrite}} + + jobs, err := resolveImageJobs(o, a, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if jobs[0].rewrite != tc.want { + t.Errorf("got rewrite %q, want %q", jobs[0].rewrite, tc.want) + } + }) + } +} + +func TestResolveImageJobs_ExcludeExtrasPrecedence(t *testing.T) { + tests := []struct { + name string + cliExcludeExtras bool + annotation string + imageExclude bool + want bool + }{ + {name: "CLI true", cliExcludeExtras: true, want: true}, + {name: "annotation true overrides CLI false", annotation: "true", want: true}, + {name: "per-image true overrides annotation/CLI false", imageExclude: true, want: true}, + {name: "all false stays false", want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + o := &flags.SyncOpts{ExcludeExtras: tc.cliExcludeExtras} + a := map[string]string{} + if tc.annotation != "" { + a[consts.ImageAnnotationExcludeExtras] = tc.annotation + } + images := []v1.Image{{Name: "rancher/rancher:v2.9", ExcludeExtras: tc.imageExclude}} + + jobs, err := resolveImageJobs(o, a, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + if jobs[0].excludeExtras != tc.want { + t.Errorf("got excludeExtras %v, want %v", jobs[0].excludeExtras, tc.want) + } + }) + } +} + +func TestResolveImageJobs_NoOptions_MinimalJob(t *testing.T) { + o := &flags.SyncOpts{} + images := []v1.Image{{Name: "rancher/rancher:v2.9"}} + + jobs, err := resolveImageJobs(o, nil, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + job := jobs[0] + if job.img.Name != "rancher/rancher:v2.9" { + t.Errorf("got name %q", job.img.Name) + } + if job.local { + t.Errorf("expected local=false") + } + if job.needsPubKey || job.needsKeyless { + t.Errorf("expected no verification needed, got needsPubKey=%v needsKeyless=%v", job.needsPubKey, job.needsKeyless) + } + if job.platform != "" || job.rewrite != "" || job.excludeExtras { + t.Errorf("expected zero-value platform/rewrite/excludeExtras, got platform=%q rewrite=%q excludeExtras=%v", job.platform, job.rewrite, job.excludeExtras) + } +} + +func TestResolveImageJobs_Local_NoOptions_AppendsLocalJob(t *testing.T) { + o := &flags.SyncOpts{} + images := []v1.Image{{Name: "rancher/rancher:v2.9", Local: true, Rewrite: "myregistry.io/rancher"}} + + jobs, err := resolveImageJobs(o, nil, images) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + if len(jobs) != 1 { + t.Fatalf("expected 1 job, got %d", len(jobs)) + } + job := jobs[0] + if !job.local { + t.Errorf("expected local=true") + } + if job.rewrite != "myregistry.io/rancher" { + t.Errorf("got rewrite %q", job.rewrite) + } +} + +// TestRunImageJobs_WithProgress_RendersEscapeCodesAndCompletionLines proves +// that when runImageJobs is given an explicit non-nil progress renderer +// (constructed over a *bytes.Buffer, not real stdout), the resulting output +// contains both the escape-coded erase/redraw sequences and the usual +// "✓ added" completion line for every successful image. +func TestRunImageJobs_WithProgress_RendersEscapeCodesAndCompletionLines(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + const n = 3 + var jobs []imageJob + for i := 0; i < n; i++ { + repo := fmt.Sprintf("progress%d", i) + seedImage(t, host, repo, "latest", remoteOpts...) + jobs = append(jobs, imageJob{img: v1.Image{Name: host + "/" + repo + ":latest"}}) + } + + s := newTestStore(t) + var buf bytes.Buffer + zl := zerolog.New(&buf) + ctx := zl.WithContext(t.Context()) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + progress := log.NewRenderer(&buf) + + if err := runImageJobs(ctx, s, jobs, 2, rso, ro, progress); err != nil { + t.Fatalf("runImageJobs: %v", err) + } + + out := buf.String() + + if !strings.Contains(out, "\x1b[") { + t.Errorf("expected escape-coded progress output somewhere in the buffer, got %q", out) + } + if got := strings.Count(out, "✓ added"); got != n { + t.Errorf("\"✓ added\" appeared %d times, want %d (one per successful image); full output:\n%s", got, n, out) + } +} + +// TestProcessContent_RealGatingDisablesProgress is the regression test that +// matters most: it runs the real processContent path (not runImageJobs +// directly, and not manually passing progress=nil) against a local test +// registry, with the ambient logger bound to a *bytes.Buffer. Because go +// test's real os.Stdout isn't a TTY, log.ShouldShowProgress naturally +// evaluates false inside processContent's call to newSyncProgress, so +// runImageJobs receives progress == nil through the real gating path. +// Asserts the resulting buffer contains zero escape bytes and still +// contains plain "✓ added" completion lines. +// +// The context is built via log.NewLogger(&buf).WithContext(...) rather than +// a raw zerolog.New(&buf) -- log.NewLogger routes through +// zerolog.ConsoleWriter, which is what actually emitted ANSI color codes +// unconditionally in the bug this test guards against. A raw zerolog logger +// writes plain JSON and never exercises ConsoleWriter at all, so it could +// never have caught this regression. +func TestProcessContent_RealGatingDisablesProgress(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + seedImage(t, host, "gated/image", "v1", remoteOpts...) + + manifest := fmt.Sprintf(`apiVersion: content.hauler.cattle.io/v1 +kind: Images +metadata: + name: test-images +spec: + images: + - name: %s/gated/image:v1 +`, host) + + fi := writeManifestFile(t, manifest) + + s := newTestStore(t) + var buf bytes.Buffer + l := log.NewLogger(&buf) + ctx := l.WithContext(t.Context()) + o := newSyncOpts(s.Root) + ro := defaultCliOpts() + + if err := processContent(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil { + t.Fatalf("processContent: %v", err) + } + + out := buf.Bytes() + + if bytes.Contains(out, []byte("\x1b[")) { + t.Errorf("expected zero ANSI escape bytes under go test's non-terminal stdout, got %q", out) + } + if !strings.Contains(buf.String(), "✓ added") { + t.Errorf("expected a plain \"✓ added\" completion line, got %q", out) + } +} + +// refCountInLine finds the first line in out containing marker and returns +// how many times ref appears in that line (from marker's start to the line's +// end). Used to prove a completion line names its image ref exactly once -- +// not once inline in the message and a second time via a structured +// "image=" field zerolog would otherwise append from a per-job logger. +func refCountInLine(t *testing.T, out, marker, ref string) int { + t.Helper() + idx := strings.Index(out, marker) + if idx == -1 { + t.Fatalf("expected output to contain %q, got %q", marker, out) + } + line := out[idx:] + if end := strings.Index(line, "\n"); end != -1 { + line = line[:end] + } + return strings.Count(line, ref) +} + +// TestRunImageJobs_NoProgress_CompletionLineRefAppearsOnce is the regression +// test for the reported bug: a sync job's "✓ added ..." completion +// line named its ref twice -- once inline in the message, once via the +// structured "image=" field runImageJobs attaches to every job's +// logger for attribution. progress is nil here (the non-TTY case): the bug +// was present in this path too, since the field is attached unconditionally +// regardless of whether progress is active. +func TestRunImageJobs_NoProgress_CompletionLineRefAppearsOnce(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + seedImage(t, host, "dup/image", "v1", remoteOpts...) + ref := host + "/dup/image:v1" + + jobs := []imageJob{{img: v1.Image{Name: ref}}} + + s := newTestStore(t) + var buf bytes.Buffer + l := log.NewLogger(&buf) + ctx := l.WithContext(t.Context()) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := runImageJobs(ctx, s, jobs, 1, rso, ro, nil); err != nil { + t.Fatalf("runImageJobs: %v", err) + } + + out := buf.String() + if got := refCountInLine(t, out, "✓ added", ref); got != 1 { + t.Errorf("ref %q appeared %d times in the completion line, want 1; full output:\n%s", ref, got, out) + } +} + +// TestRunImageJobs_WithProgress_CompletionLineRefAppearsOnce mirrors +// TestRunImageJobs_NoProgress_CompletionLineRefAppearsOnce but with a +// non-nil progress Renderer, proving the fix applies equally to the +// TTY/progress-enabled path. +func TestRunImageJobs_WithProgress_CompletionLineRefAppearsOnce(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + seedImage(t, host, "dup/progress", "v1", remoteOpts...) + ref := host + "/dup/progress:v1" + + jobs := []imageJob{{img: v1.Image{Name: ref}}} + + s := newTestStore(t) + var buf bytes.Buffer + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + progress := log.NewRenderer(&buf) + + if err := runImageJobs(t.Context(), s, jobs, 1, rso, ro, progress); err != nil { + t.Fatalf("runImageJobs: %v", err) + } + + out := buf.String() + if got := refCountInLine(t, out, "✓ added", ref); got != 1 { + t.Errorf("ref %q appeared %d times in the completion line, want 1; full output:\n%s", ref, got, out) + } +} + +// TestProcessImageTxt_RealGatingDisablesProgress mirrors +// TestProcessContent_RealGatingDisablesProgress exactly, but drives +// processImageTxt (the image.txt / -i path) instead of processContent (the +// hauler-manifest / -f path). Before this test existed, the -i path's +// gating behavior was only ever confirmed by hand in manual verification +// rounds -- this closes that coverage gap. +func TestProcessImageTxt_RealGatingDisablesProgress(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + seedImage(t, host, "gated/txtimage", "v1", remoteOpts...) + + fi := writeImageTxtFile(t, fmt.Sprintf("%s/gated/txtimage:v1\n", host)) + + s := newTestStore(t) + var buf bytes.Buffer + l := log.NewLogger(&buf) + ctx := l.WithContext(t.Context()) + o := newSyncOpts(s.Root) + ro := defaultCliOpts() + + if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil { + t.Fatalf("processImageTxt: %v", err) + } + + out := buf.Bytes() + + if bytes.Contains(out, []byte("\x1b[")) { + t.Errorf("expected zero ANSI escape bytes under go test's non-terminal stdout, got %q", out) + } + if !strings.Contains(buf.String(), "✓ added") { + t.Errorf("expected a plain \"✓ added\" completion line, got %q", out) + } +} + +func TestFormatIOStats(t *testing.T) { + st := content.IOStatsSnapshot{ + BlobsWritten: 387, + BlobsCached: 25, + BlobBytesWritten: 8_100_000_000, + BlobSemWait: 41200 * time.Millisecond, + BlobPeakInFlight: 18, + IndexWrites: 226, + IndexDurableWrites: 7, + IndexBytesWritten: 10_300_000, + IndexLockWait: 3100 * time.Millisecond, + } + + // BlobPeakInFlight (18) and the ceiling argument (20) are deliberately + // distinct: if formatIOStats's Sprintf ever transposed the two %d + // operands, "peak-inflight=18/20" would fail even though both values + // individually appear elsewhere in the format string. A fixture where + // both were 20 could not catch that swap. + got := formatIOStats(st, 20) + + for _, want := range []string{ + "blobs=412", + "written=387", + "cached=25", + "peak-inflight=18/20", + "blobsem-wait=41.2s", + "index-writes=226", + "durable=7", + "index-lock-wait=3.1s", + } { + if !strings.Contains(got, want) { + t.Fatalf("formatIOStats output missing %q:\n%s", want, got) + } + } + if strings.Contains(got, "\n") { + t.Fatalf("formatIOStats must produce a single line, got:\n%s", got) + } +} + +// -------------------------------------------------------------------------- +// resolveFileJobs +// -------------------------------------------------------------------------- + +func TestResolveFileJobs_OneJobPerFile(t *testing.T) { + files := []v1.File{ + {Path: "https://example.com/a.sh"}, + {Path: "https://example.com/b.sh", Name: "renamed-b.sh"}, + } + + jobs := resolveFileJobs(files) + if len(jobs) != 2 { + t.Fatalf("resolveFileJobs: got %d jobs, want 2", len(jobs)) + } + if jobs[0].file.Path != files[0].Path { + t.Errorf("jobs[0].file.Path = %q, want %q", jobs[0].file.Path, files[0].Path) + } + if jobs[1].file.Name != "renamed-b.sh" { + t.Errorf("jobs[1].file.Name = %q, want %q", jobs[1].file.Name, "renamed-b.sh") + } +} + +func TestResolveFileJobs_EmptyInput(t *testing.T) { + jobs := resolveFileJobs(nil) + if len(jobs) != 0 { + t.Errorf("resolveFileJobs(nil): got %d jobs, want 0", len(jobs)) + } +} + +// -------------------------------------------------------------------------- +// runFileJobs +// -------------------------------------------------------------------------- + +func TestRunFileJobs_AllSucceed(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + + url1 := seedFileInHTTPServer(t, "one.sh", "#!/bin/sh\necho one") + url2 := seedFileInHTTPServer(t, "two.sh", "#!/bin/sh\necho two") + + jobs := resolveFileJobs([]v1.File{{Path: url1}, {Path: url2}}) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := runFileJobs(ctx, s, jobs, 2, rso, ro, nil); err != nil { + t.Fatalf("runFileJobs: %v", err) + } + assertArtifactInStore(t, s, "one.sh") + assertArtifactInStore(t, s, "two.sh") +} + +// TestRunFileJobs_ConcurrencyOneVsFour_ProduceEquivalentStores proves that +// the resulting store content (blob digests + index entry refs) is +// identical regardless of --concurrency, comparing sorted sets rather than +// raw index.json bytes since ordering is nondeterministic under +// concurrency. +func TestRunFileJobs_ConcurrencyOneVsFour_ProduceEquivalentStores(t *testing.T) { + ctx := newTestContext(t) + + var urls []string + for i := 0; i < 6; i++ { + urls = append(urls, seedFileInHTTPServer(t, fmt.Sprintf("multi-%d.sh", i), fmt.Sprintf("#!/bin/sh\necho %d", i))) + } + var files []v1.File + for _, u := range urls { + files = append(files, v1.File{Path: u}) + } + + run := func(concurrency int) *storeSnapshot { + s := newTestStore(t) + jobs := resolveFileJobs(files) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + if err := runFileJobs(ctx, s, jobs, concurrency, rso, ro, nil); err != nil { + t.Fatalf("runFileJobs concurrency=%d: %v", concurrency, err) + } + return snapshotStore(t, s) + } + + snap1 := run(1) + snap4 := run(4) + + if !equalSnapshots(snap1, snap4) { + t.Errorf("store snapshots differ between concurrency=1 and concurrency=4:\nconcurrency=1: %+v\nconcurrency=4: %+v", snap1, snap4) + } +} + +// storeSnapshot captures the sorted sets that identify a store's content +// independent of index.json's on-disk ordering. +type storeSnapshot struct { + digests []string + refs []string +} + +func snapshotStore(t *testing.T, s *store.Layout) *storeSnapshot { + t.Helper() + snap := &storeSnapshot{} + if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error { + snap.digests = append(snap.digests, desc.Digest.String()) + snap.refs = append(snap.refs, desc.Annotations[ocispec.AnnotationRefName]) + return nil + }); err != nil { + t.Fatalf("snapshotStore walk: %v", err) + } + sort.Strings(snap.digests) + sort.Strings(snap.refs) + return snap +} + +func equalSnapshots(a, b *storeSnapshot) bool { + if len(a.digests) != len(b.digests) || len(a.refs) != len(b.refs) { + return false + } + for i := range a.digests { + if a.digests[i] != b.digests[i] { + return false + } + } + for i := range a.refs { + if a.refs[i] != b.refs[i] { + return false + } + } + return true +} + +// -------------------------------------------------------------------------- +// Dedup acceptance test +// -------------------------------------------------------------------------- + +func TestRunFileJobs_DedupesDuplicateSourceAcrossEntries(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + + var requests int32 + mux := http.NewServeMux() + mux.HandleFunc("/install.sh", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusOK) + return + } + atomic.AddInt32(&requests, 1) + io.WriteString(w, "#!/bin/sh\necho install") //nolint:errcheck + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + url := srv.URL + "/install.sh" + files := []v1.File{ + {Path: url}, + {Path: url, Name: "rke2-install.sh"}, + } + + jobs := resolveFileJobs(files) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := runFileJobs(ctx, s, jobs, 4, rso, ro, nil); err != nil { + t.Fatalf("runFileJobs: %v", err) + } + + assertArtifactInStore(t, s, "install.sh") + assertArtifactInStore(t, s, "rke2-install.sh") + if n := countArtifactsInStore(t, s); n != 2 { + t.Errorf("expected 2 index entries (one per Files entry), got %d", n) + } + // Without dedup, 2 manifest entries pointing at the identical source + // would cost 2x a single file's request count (4, see below) -- one full + // fetch cycle per entry. The dedup cache (pkg/artifacts/file/cache.go) + // collapses that to exactly one shared fetch cycle, holding the total to + // what a single, non-duplicated file already costs: pkg/layer's + // FromOpener reads the source once up front to compute digest (and + // reuses it as diffID, see pkg/layer/layer.go), then content.OCI.WriteBlob's + // own digest-keyed dedup (writeBlobShared) lets exactly one of the two + // jobs' writeLayer calls actually stream the blob to disk -- 1 + 1 = 2, + // not 1 and not 4. + if got := atomic.LoadInt32(&requests); got != 2 { + t.Errorf("expected exactly 2 GET requests (the cost of one file, not two) despite 2 manifest entries sharing the same source, got %d", got) + } +} + +// -------------------------------------------------------------------------- +// Error propagation table +// -------------------------------------------------------------------------- + +func TestRunFileJobs_ErrorPropagation(t *testing.T) { + tests := []struct { + name string + concurrency int + ignoreErrors bool + wantErr bool + wantGoodOK bool + }{ + {name: "concurrency=1 ignoreErrors=false", concurrency: 1, ignoreErrors: false, wantErr: true, wantGoodOK: false}, + {name: "concurrency=1 ignoreErrors=true", concurrency: 1, ignoreErrors: true, wantErr: false, wantGoodOK: true}, + {name: "concurrency=4 ignoreErrors=false", concurrency: 4, ignoreErrors: false, wantErr: true, wantGoodOK: false}, + {name: "concurrency=4 ignoreErrors=true", concurrency: 4, ignoreErrors: true, wantErr: false, wantGoodOK: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := newTestContext(t) + s := newTestStore(t) + + goodURL := seedFileInHTTPServer(t, fmt.Sprintf("good-%s.sh", sanitizeName(tt.name)), "#!/bin/sh\necho good") + + files := []v1.File{ + {Path: "http://127.0.0.1:1/unreachable.sh"}, + {Path: goodURL}, + } + + jobs := resolveFileJobs(files) + rso := defaultRootOpts(s.Root) + rso.Retries = 1 // avoid RetriesInterval sleeps in this table + ro := defaultCliOpts() + ro.IgnoreErrors = tt.ignoreErrors + + err := runFileJobs(ctx, s, jobs, tt.concurrency, rso, ro, nil) + if (err != nil) != tt.wantErr { + t.Errorf("runFileJobs error = %v, wantErr %v", err, tt.wantErr) + } + + if tt.wantGoodOK { + assertArtifactInStore(t, s, fmt.Sprintf("good-%s.sh", sanitizeName(tt.name))) + } + }) + } +} + +// sanitizeName turns a subtest name into a usable filename fragment. Lower- +// cased because storeFile's stored ref is normalized to lowercase (OCI/Docker +// reference names must be lowercase) -- an unlowercased fragment here would +// never match a substring check against the actual stored ref. +func sanitizeName(s string) string { + return strings.ToLower(strings.NewReplacer(" ", "-", "=", "-").Replace(s)) +} + +// -------------------------------------------------------------------------- +// Retry +// -------------------------------------------------------------------------- + +func TestRunFileJobs_RetryEventuallySucceeds(t *testing.T) { + if testing.Short() { + t.Skip("skipping: requires one RetriesInterval sleep (5s)") + } + + ctx := newTestContext(t) + s := newTestStore(t) + + var gets int32 + mux := http.NewServeMux() + mux.HandleFunc("/eventual.sh", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusOK) + return + } + if atomic.AddInt32(&gets, 1) == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + io.WriteString(w, "#!/bin/sh\necho eventual") //nolint:errcheck + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + jobs := resolveFileJobs([]v1.File{{Path: srv.URL + "/eventual.sh"}}) + rso := defaultRootOpts(s.Root) + rso.Retries = 2 + ro := defaultCliOpts() + + if err := runFileJobs(ctx, s, jobs, 1, rso, ro, nil); err != nil { + t.Fatalf("runFileJobs: %v", err) + } + assertArtifactInStore(t, s, "eventual.sh") +} + +// -------------------------------------------------------------------------- +// Cancellation +// -------------------------------------------------------------------------- + +// TestRunFileJobs_CancellationAbortsPromptly is the regression test for +// File.compute()'s context.TODO() fix (pkg/artifacts/file/file.go): a slow +// handler that never responds must not block runFileJobs past ctx's +// cancellation. +func TestRunFileJobs_CancellationAbortsPromptly(t *testing.T) { + block := make(chan struct{}) + mux := http.NewServeMux() + mux.HandleFunc("/slow.sh", func(w http.ResponseWriter, r *http.Request) { + // storeFile's Client.Name(fi.Path) call (deriving the stored ref, + // before compute()/fetch even starts) issues an HTTP HEAD via + // getter.Http.Name, which -- unlike Open -- takes no context + // parameter at all (net/http.Head has no context-aware variant) and + // so cannot be cancelled. That's a separate, narrower gap than the + // one this test targets (compute()'s context.TODO() bug and Open's + // context wiring); block only the GET that actually fetches content, + // so this test isolates the fetch-cancellation path under test + // rather than hanging on the unrelated HEAD gap. + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusOK) + return + } + <-block + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + t.Cleanup(func() { close(block) }) + + zl := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(zl.WithContext(context.Background())) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() + + s := newTestStore(t) + jobs := resolveFileJobs([]v1.File{{Path: srv.URL + "/slow.sh"}}) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + done := make(chan error, 1) + go func() { + done <- runFileJobs(ctx, s, jobs, 1, rso, ro, nil) + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("expected an error after ctx cancellation, got nil") + } + case <-time.After(5 * time.Second): + t.Fatal("runFileJobs did not return within 5s of ctx cancellation") + } +} + +// -------------------------------------------------------------------------- +// Renderer rows +// -------------------------------------------------------------------------- + +func TestRunFileJobs_WithProgress_RendersEscapeCodesAndCompletionLines(t *testing.T) { + const n = 3 + var files []v1.File + for i := 0; i < n; i++ { + files = append(files, v1.File{Path: seedFileInHTTPServer(t, fmt.Sprintf("progress-%d.sh", i), "#!/bin/sh\necho hi")}) + } + + s := newTestStore(t) + var buf bytes.Buffer + zl := zerolog.New(&buf) + ctx := zl.WithContext(t.Context()) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + progress := log.NewRenderer(&buf) + jobs := resolveFileJobs(files) + + if err := runFileJobs(ctx, s, jobs, 2, rso, ro, progress); err != nil { + t.Fatalf("runFileJobs: %v", err) + } + + out := buf.String() + if !strings.Contains(out, "\x1b[") { + t.Errorf("expected escape-coded progress output somewhere in the buffer, got %q", out) + } + if got := strings.Count(out, "✓ added"); got != n { + t.Errorf("\"✓ added\" appeared %d times, want %d; full output:\n%s", got, n, out) + } +} + +func TestRunFileJobs_NoProgress_CompletionLineRefAppearsOnce(t *testing.T) { + url := seedFileInHTTPServer(t, "dup-ref.sh", "#!/bin/sh\necho dup") + + s := newTestStore(t) + var buf bytes.Buffer + l := log.NewLogger(&buf) + ctx := l.WithContext(t.Context()) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + jobs := resolveFileJobs([]v1.File{{Path: url}}) + if err := runFileJobs(ctx, s, jobs, 1, rso, ro, nil); err != nil { + t.Fatalf("runFileJobs: %v", err) + } + + out := buf.String() + if got := refCountInLine(t, out, "✓ added", "dup-ref.sh"); got != 1 { + t.Errorf("ref appeared %d times in the completion line, want 1; full output:\n%s", got, out) + } +} + +// countingBlobHandler wraps an http.Handler and increments hits[digest] for +// every GET request to /v2//blobs/, then delegates to the +// wrapped handler. +type countingBlobHandler struct { + next http.Handler + mu sync.Mutex + hits map[string]int +} + +func newCountingBlobHandler(next http.Handler) *countingBlobHandler { + return &countingBlobHandler{next: next, hits: make(map[string]int)} +} + +func (c *countingBlobHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + if idx := strings.Index(r.URL.Path, "/blobs/"); idx != -1 { + digest := r.URL.Path[idx+len("/blobs/"):] + c.mu.Lock() + c.hits[digest]++ + c.mu.Unlock() + } + } + c.next.ServeHTTP(w, r) +} + +func (c *countingBlobHandler) count(digest string) int { + c.mu.Lock() + defer c.mu.Unlock() + return c.hits[digest] +} + +// newCountingLocalhostRegistry mirrors newLocalhostRegistry (add_test.go) +// -- listening on "localhost:0" so go-containerregistry auto-selects plain +// HTTP -- but wraps registry.New() in a countingBlobHandler so tests can +// assert on per-digest blob GET counts. +func newCountingLocalhostRegistry(t *testing.T) (host string, remoteOpts []remote.Option, counter *countingBlobHandler) { + t.Helper() + counter = newCountingBlobHandler(registry.New()) + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("newCountingLocalhostRegistry listen: %v", err) + } + srv := httptest.NewUnstartedServer(counter) + srv.Listener = l + srv.Start() + t.Cleanup(srv.Close) + host = strings.TrimPrefix(srv.URL, "http://") + remoteOpts = []remote.Option{remote.WithTransport(srv.Client().Transport)} + return host, remoteOpts, counter +} + +// TestSyncImages_SharedLayerDownloadedOnce is the acceptance test for the +// whole parallelization effort: two images that share a common layer are +// synced concurrently, and the shared layer's blob must be fetched from the +// registry exactly once regardless of --concurrency. +func TestSyncImages_SharedLayerDownloadedOnce(t *testing.T) { + for _, concurrency := range []int{1, 4} { + t.Run(fmt.Sprintf("concurrency=%d", concurrency), func(t *testing.T) { + host, remoteOpts, counter := newCountingLocalhostRegistry(t) + + sharedLayer, err := random.Layer(2048, gvtypes.OCILayer) + if err != nil { + t.Fatalf("random.Layer: %v", err) + } + sharedDigest, err := sharedLayer.Digest() + if err != nil { + t.Fatalf("sharedLayer.Digest: %v", err) + } + + img1, err := mutate.AppendLayers(empty.Image, sharedLayer) + if err != nil { + t.Fatalf("mutate.AppendLayers img1: %v", err) + } + img2, err := mutate.AppendLayers(empty.Image, sharedLayer) + if err != nil { + t.Fatalf("mutate.AppendLayers img2: %v", err) + } + + ref1, err := name.NewTag(host+"/repo1:latest", name.Insecure) + if err != nil { + t.Fatalf("NewTag ref1: %v", err) + } + ref2, err := name.NewTag(host+"/repo2:latest", name.Insecure) + if err != nil { + t.Fatalf("NewTag ref2: %v", err) + } + if err := remote.Write(ref1, img1, remoteOpts...); err != nil { + t.Fatalf("remote.Write img1: %v", err) + } + if err := remote.Write(ref2, img2, remoteOpts...); err != nil { + t.Fatalf("remote.Write img2: %v", err) + } + + s := newTestStore(t) + ctx := newTestContext(t) + jobs := []imageJob{ + {img: v1.Image{Name: host + "/repo1:latest"}}, + {img: v1.Image{Name: host + "/repo2:latest"}}, + } + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := runImageJobs(ctx, s, jobs, concurrency, rso, ro, nil); err != nil { + t.Fatalf("runImageJobs: %v", err) + } + + assertArtifactInStore(t, s, "repo1") + assertArtifactInStore(t, s, "repo2") + + hits := counter.count(sharedDigest.String()) + if hits != 1 { + t.Errorf("shared layer blob GET count = %d, want exactly 1 (concurrency=%d)", hits, concurrency) + } + }) + } +} + +// TestSyncImages_ConcurrencyOneMatchesSerial syncs the same set of images +// into two independent stores, once at concurrency=1 and once at +// concurrency=4, and asserts the resulting stores contain the same set of +// blob digests and the same set of index entries (ref+kind+digest tuples). +// +// Deliberately NOT a byte-for-byte index.json comparison: sync.Map range +// order plus the resolver's SliceStable 2-way merge make exact index.json +// byte order nondeterministic across runs. Comparing sorted logical entry +// sets instead avoids turning this into a flaky golden-file test. +func TestSyncImages_ConcurrencyOneMatchesSerial(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + const nImages = 5 + var jobs []imageJob + for i := 0; i < nImages; i++ { + repo := fmt.Sprintf("repo%d", i) + seedImage(t, host, repo, "latest", remoteOpts...) + jobs = append(jobs, imageJob{img: v1.Image{Name: host + "/" + repo + ":latest"}}) + } + + sSerial := newTestStore(t) + sParallel := newTestStore(t) + ctx := newTestContext(t) + ro := defaultCliOpts() + + if err := runImageJobs(ctx, sSerial, jobs, 1, defaultRootOpts(sSerial.Root), ro, nil); err != nil { + t.Fatalf("runImageJobs (concurrency=1): %v", err) + } + if err := runImageJobs(ctx, sParallel, jobs, 4, defaultRootOpts(sParallel.Root), ro, nil); err != nil { + t.Fatalf("runImageJobs (concurrency=4): %v", err) + } + + serialBlobs := sortedBlobDigests(t, sSerial.Root) + parallelBlobs := sortedBlobDigests(t, sParallel.Root) + if !equalStringSlices(serialBlobs, parallelBlobs) { + t.Errorf("blob digest sets differ:\nserial: %v\nparallel: %v", serialBlobs, parallelBlobs) + } + + serialEntries := sortedIndexEntries(t, sSerial) + parallelEntries := sortedIndexEntries(t, sParallel) + if !equalStringSlices(serialEntries, parallelEntries) { + t.Errorf("index entry sets differ:\nserial: %v\nparallel: %v", serialEntries, parallelEntries) + } +} + +// TestSyncImages_ErrorPropagation covers fail-fast/ignore-errors semantics +// across concurrency levels: 4 good images plus 1 that 404s. +func TestSyncImages_ErrorPropagation(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + const nGood = 4 + var goodJobs []imageJob + for i := 0; i < nGood; i++ { + repo := fmt.Sprintf("good%d", i) + seedImage(t, host, repo, "latest", remoteOpts...) + goodJobs = append(goodJobs, imageJob{img: v1.Image{Name: host + "/" + repo + ":latest"}}) + } + badRef := host + "/does-not-exist:latest" + + for _, concurrency := range []int{1, 4} { + for _, ignoreErrors := range []bool{false, true} { + t.Run(fmt.Sprintf("concurrency=%d/ignoreErrors=%v", concurrency, ignoreErrors), func(t *testing.T) { + jobs := append([]imageJob{}, goodJobs...) + jobs = append(jobs, imageJob{img: v1.Image{Name: badRef}}) + + s := newTestStore(t) + ctx := newTestContext(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + ro.IgnoreErrors = ignoreErrors + + err := runImageJobs(ctx, s, jobs, concurrency, rso, ro, nil) + + if !ignoreErrors { + if err == nil { + t.Fatal("runImageJobs: expected error, got nil") + } + if !strings.Contains(err.Error(), "does-not-exist") { + t.Errorf("runImageJobs error = %q, want it to identify the bad image (does-not-exist), not a bare cancellation", err.Error()) + } + return + } + + if err != nil { + t.Fatalf("runImageJobs (ignoreErrors=true): unexpected error: %v", err) + } + for i := 0; i < nGood; i++ { + assertArtifactInStore(t, s, fmt.Sprintf("good%d", i)) + } + }) + } + } +} + +// TestRunImageJobs_CancelledJobsDoNotLogAddingImage reproduces the log-spam +// bug: with concurrency=1 and a bad ref placed before several good refs, the +// bad job's failure cancels the errgroup's derived context. Jobs queued after +// it never get a chance to call s.AddImage, so they must never log the +// "adding image [...] to the store" INFO line either -- otherwise a failed +// sync of 1 image looks like it attempted all of them. +// +// concurrency=1 makes cancellation deterministic: errgroup.SetLimit(1) means +// each g.Go call blocks acquiring its semaphore slot until the previous job's +// goroutine has fully returned (including cancelling gctx on failure), so +// jobs run strictly in slice order and the good jobs are guaranteed to +// observe the already-cancelled context before doing anything. +func TestRunImageJobs_CancelledJobsDoNotLogAddingImage(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + const nGood = 3 + badRef := host + "/does-not-exist:latest" + jobs := []imageJob{{img: v1.Image{Name: badRef}}} + for i := 0; i < nGood; i++ { + repo := fmt.Sprintf("good%d", i) + seedImage(t, host, repo, "latest", remoteOpts...) + jobs = append(jobs, imageJob{img: v1.Image{Name: host + "/" + repo + ":latest"}}) + } + + s := newTestStore(t) + var buf bytes.Buffer + // "adding image [...]" now logs at Debug (cmd/hauler/cli/store/add.go), + // so this test needs Debug-level output visible. Per-logger .Level() is + // not sufficient on its own: zerolog's Logger.should() gates on + // max(logger.level, zerolog.GlobalLevel()) -- and GlobalLevel is + // process-global state that other tests in this package mutate (e.g. + // sync_test.go's TestSyncCmd_DryRun_Products_PrintsManifestToStdout + // resets it to InfoLevel in a t.Cleanup). Under -shuffle/-count>1 that + // can leave the global level at Info before this test runs, silently + // suppressing Debug output regardless of the per-logger Level call. Save + // and restore the global level around this test so it's deterministic + // regardless of what ran before it, and doesn't leave Debug-level + // pollution behind for tests that run after it. + prevGlobalLevel := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(prevGlobalLevel) }) + + zl := zerolog.New(&buf).Level(zerolog.DebugLevel) + ctx := zl.WithContext(context.Background()) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + err := runImageJobs(ctx, s, jobs, 1, rso, ro, nil) + if err == nil { + t.Fatal("runImageJobs: expected error, got nil") + } + + got := strings.Count(buf.String(), "adding image [") + if got != 1 { + t.Errorf("\"adding image [\" logged %d times, want exactly 1 (only the failed job should have attempted logging; the %d good jobs queued after it must never start)\nfull log:\n%s", got, nGood, buf.String()) + } +} + +// TestSyncImages_IndexCompletenessUnderParallelAdds builds 20 images, syncs +// them at concurrency=8, and asserts the store's on-disk index.json (not +// just the in-memory nameMap) contains all 20 entries after re-opening the +// store fresh. +func TestSyncImages_IndexCompletenessUnderParallelAdds(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + const n = 20 + var jobs []imageJob + for i := 0; i < n; i++ { + repo := fmt.Sprintf("img%d", i) + seedImage(t, host, repo, "latest", remoteOpts...) + jobs = append(jobs, imageJob{img: v1.Image{Name: host + "/" + repo + ":latest"}}) + } + + s := newTestStore(t) + ctx := newTestContext(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := runImageJobs(ctx, s, jobs, 8, rso, ro, nil); err != nil { + t.Fatalf("runImageJobs: %v", err) + } + + if got := countArtifactsInStore(t, s); got != n { + t.Errorf("countArtifactsInStore (same instance) = %d, want %d", got, n) + } + + reopened, err := store.NewLayout(s.Root) + if err != nil { + t.Fatalf("re-opening store: %v", err) + } + if got := countArtifactsInStore(t, reopened); got != n { + t.Errorf("countArtifactsInStore (freshly re-opened store) = %d, want %d -- entries lost from index.json on disk", got, n) + } +} + +// -------------------------------------------------------------------------- +// helpers +// -------------------------------------------------------------------------- + +func sortedBlobDigests(t *testing.T, root string) []string { + t.Helper() + dir := filepath.Join(root, ocispec.ImageBlobsDir, "sha256") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir %s: %v", dir, err) + } + var out []string + for _, e := range entries { + if e.IsDir() { + continue + } + out = append(out, e.Name()) + } + sort.Strings(out) + return out +} + +func sortedIndexEntries(t *testing.T, s *store.Layout) []string { + t.Helper() + var out []string + if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error { + out = append(out, fmt.Sprintf("%s|%s|%s", + desc.Annotations[ocispec.AnnotationRefName], + desc.Annotations["kind"], + desc.Digest.String(), + )) + return nil + }); err != nil { + t.Fatalf("Walk: %v", err) + } + sort.Strings(out) + return out +} + +func equalStringSlices(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// syncBuffer is a mutex-guarded io.Writer + String() buffer. It is +// deliberately a separate lock from the Renderer's own internal mutex, so a +// test goroutine can read the buffer's contents while the Renderer +// concurrently writes to it (from runImageJobs' background goroutine) +// without racing on the underlying storage under `go test -race`. +type syncBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.String() +} + +// blockingManifestHandler delegates every request to next, except that the +// first GET request matching path is held open: it closes entered (so a +// test can deterministically observe that the request has reached the +// server and is now stuck) and then blocks until release is closed. No +// sleeps are involved anywhere in this synchronization. +type blockingManifestHandler struct { + next http.Handler + path string + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (h *blockingManifestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == h.path { + h.once.Do(func() { close(h.entered) }) + <-h.release + } + h.next.ServeHTTP(w, r) +} + +// TestRunImageJobs_BeganFiresOnlyAfterSemaphoreAcquired reproduces the bug +// where progress.Began(name) was invoked from the single-threaded outer +// remote-jobs loop, before g.Go(...) had any chance to block acquiring its +// errgroup.SetLimit concurrency slot. +// +// At concurrency=1 with two jobs, job1's manifest fetch (a real HTTP request +// against a local test registry) is held open on a channel -- indefinitely, +// until the test releases it. Because job2's g.Go call cannot acquire the +// single available slot until job1's goroutine finishes and releases it, +// job2's own goroutine cannot have started yet. So if progress already shows +// job2 as in-flight by the time job1's request is observed to have reached +// the server (which requires job1's goroutine to have done real, comparatively +// expensive network I/O), Began(job2) can only have been called from the +// outer loop rather than from inside job2's own goroutine -- proving the bug +// is present. +func TestRunImageJobs_BeganFiresOnlyAfterSemaphoreAcquired(t *testing.T) { + // Repo/tag names are kept intentionally short (unlike other tests in this + // package) so that both refs comfortably fit within the Renderer's status + // line width budget without being truncated away by truncateNameList -- + // this test asserts on the literal presence of job2's ref substring, so + // truncation would make that assertion meaningless regardless of whether + // the underlying bug is present. + const job1Repo = "a1" + const job1Tag = "t" + const job2Repo = "b2" + const job2Tag = "t" + + handler := &blockingManifestHandler{ + path: "/v2/" + job1Repo + "/manifests/" + job1Tag, + entered: make(chan struct{}), + release: make(chan struct{}), + } + handler.next = registry.New() + + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + host := strings.TrimPrefix(srv.URL, "http://") + remoteOpts := []remote.Option{remote.WithTransport(srv.Client().Transport)} + + seedImage(t, host, job1Repo, job1Tag, remoteOpts...) + seedImage(t, host, job2Repo, job2Tag, remoteOpts...) + + s := newTestStore(t) + buf := &syncBuffer{} + progress := log.NewRenderer(buf) + + ctx := newTestContext(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + jobs := []imageJob{ + {img: v1.Image{Name: host + "/" + job1Repo + ":" + job1Tag}}, + {img: v1.Image{Name: host + "/" + job2Repo + ":" + job2Tag}}, + } + + errCh := make(chan error, 1) + go func() { + errCh <- runImageJobs(ctx, s, jobs, 1, rso, ro, progress) + }() + + select { + case <-handler.entered: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for job1's manifest request to reach the test registry") + } + + if strings.Contains(buf.String(), job2Repo) { + t.Errorf("progress already shows job2 (%q) as in-flight while job1 is still blocked and concurrency=1 -- "+ + "Began(job2) must have fired from the outer loop before job2's goroutine could possibly have acquired "+ + "a semaphore slot\nbuffer contents:\n%s", job2Repo, buf.String()) + } + + close(handler.release) + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("runImageJobs: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for runImageJobs to finish after releasing job1") + } + + out := buf.String() + if got := strings.Count(out, "✓ added"); got != 2 { + t.Errorf("\"✓ added\" appeared %d times, want 2; full output:\n%s", got, out) + } +} + +// -------------------------------------------------------------------------- +// verification fused into the pull worker +// -------------------------------------------------------------------------- + +func TestImageJobVerifyConfig(t *testing.T) { + tests := []struct { + name string + job imageJob + want cosign.Config + }{ + { + name: "no verification requested", + job: imageJob{img: v1.Image{Name: "example.com/nginx:v1"}}, + want: cosign.Config{}, + }, + { + name: "keyed", + job: imageJob{needsPubKey: true, key: "/keys/cosign.pub", tlog: true}, + want: cosign.Config{Key: "/keys/cosign.pub", Tlog: true}, + }, + { + name: "keyless", + job: imageJob{ + needsKeyless: true, + certIdentity: "me@example.com", + certIdentityRegexp: ".*@example.com", + certOidcIssuer: "https://accounts.example.com", + certOidcIssuerRegexp: "https://.*", + certGithubWorkflowRepository: "example/repo", + }, + want: cosign.Config{ + CertIdentity: "me@example.com", + CertIdentityRegexp: ".*@example.com", + CertOidcIssuer: "https://accounts.example.com", + CertOidcIssuerRegexp: "https://.*", + CertGithubWorkflowRepository: "example/repo", + }, + }, + { + // cosign.Config.validate rejects a key alongside any Cert* field, so + // a Config built from the raw resolved inputs instead of the + // branch-selected ones would turn this into a hard error. + name: "keyed job drops identity fields rather than combining them", + job: imageJob{ + needsPubKey: true, + key: "/keys/cosign.pub", + certIdentity: "me@example.com", + certOidcIssuerRegexp: "https://.*", + }, + want: cosign.Config{Key: "/keys/cosign.pub"}, + }, + { + // tlog belongs to the keyed branch; NewVerifier forces it on for + // keyless anyway. Leaking it here would also make the Config + // non-Empty and drag an unverified job into the verify path. + name: "keyless job drops the key branch's tlog flag", + job: imageJob{ + needsKeyless: true, + tlog: true, + certIdentity: "me@example.com", + certOidcIssuer: "https://accounts.example.com", + }, + want: cosign.Config{CertIdentity: "me@example.com", CertOidcIssuer: "https://accounts.example.com"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.job.verifyConfig(); got != tt.want { + t.Fatalf("verifyConfig() = %+v, want %+v", got, tt.want) + } + }) + } +} + +// The gate that decides which jobs verify must not widen: before the verify +// pass was fused into the worker, a job verified iff resolveImageJobs set +// needsPubKey or needsKeyless. cosign.Config.Empty is now that gate, so every +// job resolveImageJobs leaves unverified must map to the zero Config. +func TestVerifyConfigEmptyMatchesResolvedGate(t *testing.T) { + tests := []struct { + name string + o *flags.SyncOpts + a map[string]string + image v1.Image + wantVerify bool + }{ + {name: "bare image", o: &flags.SyncOpts{}, image: v1.Image{Name: "example.com/nginx:v1"}}, + { + // The one input that is verification-adjacent but never sufficient + // on its own: --tlog alone must still skip verification. + name: "tlog without a key", + o: &flags.SyncOpts{Tlog: true}, + image: v1.Image{Name: "example.com/nginx:v1"}, + }, + {name: "platform annotation only", o: &flags.SyncOpts{}, a: map[string]string{consts.ImageAnnotationPlatform: "linux/amd64"}, image: v1.Image{Name: "example.com/nginx:v1"}}, + {name: "cli key", o: &flags.SyncOpts{Key: "/keys/cosign.pub"}, image: v1.Image{Name: "example.com/nginx:v1"}, wantVerify: true}, + {name: "per-image key", o: &flags.SyncOpts{}, image: v1.Image{Name: "example.com/nginx:v1", Key: "/keys/cosign.pub"}, wantVerify: true}, + {name: "annotation key", o: &flags.SyncOpts{}, a: map[string]string{consts.ImageAnnotationKey: "/keys/cosign.pub"}, image: v1.Image{Name: "example.com/nginx:v1"}, wantVerify: true}, + {name: "cli identity", o: &flags.SyncOpts{CertIdentity: "me@example.com"}, image: v1.Image{Name: "example.com/nginx:v1"}, wantVerify: true}, + {name: "cli identity regexp", o: &flags.SyncOpts{CertIdentityRegexp: ".*"}, image: v1.Image{Name: "example.com/nginx:v1"}, wantVerify: true}, + { + // An issuer without a subject never triggered verification before + // and must not now: it is not a complete keyless identity. + name: "cli issuer without an identity", + o: &flags.SyncOpts{CertOidcIssuer: "https://accounts.example.com"}, + image: v1.Image{Name: "example.com/nginx:v1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jobs, err := resolveImageJobs(tt.o, tt.a, []v1.Image{tt.image}) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + job := jobs[0] + + wantGate := job.needsPubKey || job.needsKeyless + if wantGate != tt.wantVerify { + t.Fatalf("resolveImageJobs set needsPubKey=%v needsKeyless=%v, want verification=%v", job.needsPubKey, job.needsKeyless, tt.wantVerify) + } + if got := job.verifyConfig().Empty(); got == tt.wantVerify { + t.Fatalf("verifyConfig().Empty() = %v for a job whose resolved gate says verify=%v", got, tt.wantVerify) + } + }) + } +} + +// resolveImageJobs' branches are exclusive, so a manifest naming both a key and +// an identity has always verified against the key alone. cosign.Config rejects +// that pairing outright, so verifyConfig has to reproduce the precedence rather +// than forward both -- otherwise a manifest that worked yesterday hard-errors. +func TestVerifyConfigKeyWinsOverIdentityAndStaysBuildable(t *testing.T) { + keyPath := writeTestPubKey(t) + o := &flags.SyncOpts{ + Key: keyPath, + CertIdentity: "me@example.com", + CertOidcIssuer: "https://accounts.example.com", + } + + jobs, err := resolveImageJobs(o, nil, []v1.Image{{Name: "example.com/nginx:v1"}}) + if err != nil { + t.Fatalf("resolveImageJobs: %v", err) + } + + cfg := jobs[0].verifyConfig() + if cfg.Key != keyPath { + t.Fatalf("verifyConfig().Key = %q, want the resolved key %q", cfg.Key, keyPath) + } + if cfg.Keyless() { + t.Fatal("verifyConfig produced a keyless Config for a job the key branch claimed") + } + + // A keyed Config with no Cert* fields needs no trust root, so this builds + // offline (see cosign.NewVerifier's offlineWithKey). + rso, ro := defaultRootOpts(t.TempDir()), defaultCliOpts() + v, err := cosign.NewVerifier(newTestContext(t), cfg, rso, ro) + if err != nil { + t.Fatalf("cosign.NewVerifier rejected the Config sync builds for a key+identity manifest: %v", err) + } + v.Close() +} + +// A job with no verification inputs must not resolve either: the digest pin +// exists to close the gap between checking a tag and pulling it, and there is +// no gap when nothing is checked. Pointing at a registry that cannot answer is +// what proves no request was made -- a resolve attempt here would error. +func TestResolveAndVerifySkipsUnverifiedJobs(t *testing.T) { + j := imageJob{img: v1.Image{Name: "127.0.0.1:1/absent/image:v1"}} + + // A nil Cache would panic if the verify path were entered. + pinned, err := resolveAndVerify(newTestContext(t), nil, j, defaultRootOpts(t.TempDir()), defaultCliOpts()) + if err != nil { + t.Fatalf("resolveAndVerify: %v", err) + } + if pinned != "" { + t.Fatalf("resolveAndVerify pinned %q for a job that requested no verification", pinned) + } +} + +// Verification must check the digest resolveAndVerify pinned, not the tag it +// started from -- closing that window is the whole point of fusing the passes +// into one worker. The tag is therefore resolved exactly once, by +// resolveAndVerify itself; handing cosign the tag instead would make it resolve +// a second time, and whatever the tag pointed at by then is what would be +// checked. +func TestResolveAndVerifyChecksTheDigestNotTheTag(t *testing.T) { + host, remoteOpts, rec := newRecordingRegistry(t) + img := seedImage(t, host, "badsig", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "badsig", img, remoteOpts...) + digest, err := img.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + rec.reset() // seeding itself HEADs the tag + + rso, ro := defaultRootOpts(t.TempDir()), defaultCliOpts() + cache := cosign.NewCache(rso, ro) + defer cache.Close() + + // The signature manifest exists but carries no usable signature, so + // verification fails closed after doing all its registry work. + j := imageJob{img: v1.Image{Name: host + "/badsig:v1"}, needsPubKey: true, key: writeTestPubKey(t)} + if _, err := resolveAndVerify(newTestContext(t), cache, j, rso, ro); err == nil { + t.Fatal("resolveAndVerify accepted an image whose only signature is unusable") + } + + sigTag := "manifests/" + strings.ReplaceAll(digest.String(), ":", "-") + ".sig" + if rec.countContaining(sigTag) == 0 { + t.Fatalf("verification never fetched %s, so it did not run against the pinned digest; requests:\n%v", sigTag, rec.snapshot()) + } + if got := rec.countContaining("manifests/v1"); got != 1 { + t.Fatalf("the tag was resolved %d times, want exactly 1 (resolveAndVerify's own pin); verification is re-resolving the tag\nrequests:\n%v", got, rec.snapshot()) + } +} + +// The feature's core guarantee, end to end: the digest that passed verification +// is the digest that lands in the store. +// +// Both halves are asserted, because either alone is weak. The stored descriptor +// proves what was written; the request log proves how -- the tag is resolved +// exactly once, by resolveAndVerify's pin, so storeImage fetched by digest and +// never re-read the tag it could have found moved. +func TestRunImageJobs_StoresTheDigestItVerified(t *testing.T) { + host, remoteOpts, rec := newRecordingRegistry(t) + img, keyPath := seedSignedImage(t, host, "signed", "v1", remoteOpts...) + want, err := img.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + rec.reset() // seeding itself HEADs the tag + + s := newTestStore(t) + ref := host + "/signed:v1" + jobs := []imageJob{{img: v1.Image{Name: ref}, needsPubKey: true, key: keyPath, excludeExtras: true}} + + if err := runImageJobs(newTestContext(t), s, jobs, 1, defaultRootOpts(s.Root), defaultCliOpts(), nil); err != nil { + t.Fatalf("runImageJobs: %v", err) + } + + got := storedDigest(t, s, "signed:v1") + if got == "" { + t.Fatalf("a validly signed image was not stored; requests:\n%v", rec.snapshot()) + } + if got != want.String() { + t.Fatalf("stored digest %s, want the verified digest %s", got, want) + } + if n := rec.countContaining("manifests/v1"); n != 1 { + t.Fatalf("the tag was resolved %d times, want exactly 1 (resolveAndVerify's pin); storeImage re-resolved the tag instead of using the pinned digest\nrequests:\n%v", n, rec.snapshot()) + } +} + +// Without --ignore-errors, a verify failure now fails the whole run instead of +// being dropped as it used to be: dropping let an unverified/unsigned image go +// silently missing from an otherwise-exit-0 run, which is exactly the failure +// mode --ignore-errors exists to opt into deliberately. Nothing is stored -- +// the good job's context is cancelled by the errgroup's fail-fast before it can +// complete. +func TestRunImageJobs_VerifyFailureFailsTheRunByDefault(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + bad := seedImage(t, host, "test/badsig", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "test/badsig", bad, remoteOpts...) + seedImage(t, host, "test/good", "v1", remoteOpts...) + + s := newTestStore(t) + jobs := []imageJob{ + {img: v1.Image{Name: host + "/test/badsig:v1"}, needsPubKey: true, key: writeTestPubKey(t), excludeExtras: true}, + {img: v1.Image{Name: host + "/test/good:v1"}, excludeExtras: true}, + } + + // concurrency=1 runs the jobs in slice order (errgroup.SetLimit(1)), so the + // bad job has already failed -- and cancelled gctx -- before the good job's + // goroutine can acquire the semaphore. That makes the good job's + // cancellation deterministic instead of a race against how far its own + // pull got before the group's context died. + if err := runImageJobs(newTestContext(t), s, jobs, 1, defaultRootOpts(s.Root), defaultCliOpts(), nil); err == nil { + t.Fatal("runImageJobs succeeded despite a verification failure; the default is now to fail the run, not drop the one image") + } + if got := countArtifactsInStore(t, s); got != 0 { + t.Fatalf("store holds %d artifacts, want 0; a failed run must not have stored anything", got) + } +} + +// With --ignore-errors, a verify failure is now a WARN, not a dropped image: +// the image is stored unverified rather than being left out of the run. This +// is the deliberate tradeoff --ignore-errors buys -- an unverified image +// reaching the store (and potentially an airgapped environment) rather than +// the run failing outright. +func TestRunImageJobs_VerifyFailureIgnoreErrors_StoresUnverified(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + bad := seedImage(t, host, "test/badsig", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "test/badsig", bad, remoteOpts...) + seedImage(t, host, "test/good", "v1", remoteOpts...) + + s := newTestStore(t) + ro := defaultCliOpts() + ro.IgnoreErrors = true + badRef := host + "/test/badsig:v1" + goodRef := host + "/test/good:v1" + jobs := []imageJob{ + {img: v1.Image{Name: badRef}, needsPubKey: true, key: writeTestPubKey(t), excludeExtras: true}, + {img: v1.Image{Name: goodRef}, excludeExtras: true}, + } + + var buf bytes.Buffer + l := log.NewLogger(&buf) + ctx := l.WithContext(t.Context()) + + if err := runImageJobs(ctx, s, jobs, 2, defaultRootOpts(s.Root), ro, nil); err != nil { + t.Fatalf("run returned an error under --ignore-errors: %v", err) + } + assertArtifactInStore(t, s, "test/good:v1") + // The assertion that matters most: the image that failed verification is + // in the store anyway, unverified. + assertArtifactInStore(t, s, "test/badsig:v1") + if got := countArtifactsInStore(t, s); got != 2 { + t.Fatalf("store holds %d artifacts, want 2 (both images, the bad one unverified)", got) + } + + out := buf.String() + if !strings.Contains(out, "WRN") || !strings.Contains(out, badRef) { + t.Fatalf("expected a WARN line naming %q, got:\n%s", badRef, out) + } +} + +// storeImage's audit entry must report whether verification actually +// succeeded, not merely whether it was requested -- see the "verified" field +// bug this guards: before this fix storeImage derived "verified" from +// whether i.Key/i.CertIdentity/i.CertIdentityRegexp were set, which stayed +// true even when --ignore-errors stored an image that failed its check. +// +// Table-tested against runImageJobs, the general path both `store sync` and +// `store add chart`'s discovered-image pass funnel through. +func TestRunImageJobs_AuditVerifiedFlag(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + tests := []struct { + name string + buildJob func(t *testing.T) imageJob + ignoreErrors bool + want bool + }{ + { + name: "not requested", + buildJob: func(t *testing.T) imageJob { + seedImage(t, host, "test/audit-unrequested", "v1", remoteOpts...) + return imageJob{img: v1.Image{Name: host + "/test/audit-unrequested:v1"}, excludeExtras: true} + }, + want: false, + }, + { + name: "requested and passed", + buildJob: func(t *testing.T) imageJob { + _, keyPath := seedSignedImage(t, host, "test/audit-passed", "v1", remoteOpts...) + return imageJob{img: v1.Image{Name: host + "/test/audit-passed:v1"}, needsPubKey: true, key: keyPath, excludeExtras: true} + }, + want: true, + }, + { + name: "requested and failed, stored anyway under --ignore-errors", + buildJob: func(t *testing.T) imageJob { + bad := seedImage(t, host, "test/audit-failed", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "test/audit-failed", bad, remoteOpts...) + keyPath := writeTestPubKey(t) + // img.Key is set alongside imageJob.key so this subtest also + // catches a regression to the old "verified" proxy + // (i.Key != "" || ...), which reads img.Key, not job.key. + return imageJob{img: v1.Image{Name: host + "/test/audit-failed:v1", Key: keyPath}, needsPubKey: true, key: keyPath, excludeExtras: true} + }, + ignoreErrors: true, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + job := tc.buildJob(t) + + s := newTestStore(t) + ro := defaultCliOpts() + ro.AuditLevel = "verbose" + ro.IgnoreErrors = tc.ignoreErrors + ro.HaulerDir = t.TempDir() + + if err := runImageJobs(newTestContext(t), s, []imageJob{job}, 1, defaultRootOpts(s.Root), ro, nil); err != nil { + t.Fatalf("runImageJobs: %v", err) + } + + flags := lastAuditEntryFlags(t, ro.HaulerDir) + got, ok := flags["verified"].(bool) + if !ok { + t.Fatalf("audit entry's flags[\"verified\"] is %v (%T), want a bool", flags["verified"], flags["verified"]) + } + if got != tc.want { + t.Errorf("flags[\"verified\"] = %v, want %v", got, tc.want) + } + }) + } +} + +// The progress row for a job that hit a verify failure has to be cleared like +// any other, or the live region keeps showing an image that is no longer being +// worked on for the rest of the run. The default (fail-the-run) case is used +// here rather than --ignore-errors, since that is the one where the job's +// goroutine actually returns an error -- matching the "even when its job +// errors out" case the deferred progress.Finished exists to cover. +// +// concurrency=1 makes the ordering deterministic: errgroup.SetLimit(1) runs the +// jobs in slice order, so every frame naming the good image is drawn after the +// bad one finished. If Finished were skipped, the bad ref would reappear in +// each of those frames. +func TestRunImageJobs_VerifyFailureClearsProgressRow(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + bad := seedImage(t, host, "badsig", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "badsig", bad, remoteOpts...) + seedImage(t, host, "goodimage", "v1", remoteOpts...) + + badRef := host + "/badsig:v1" + goodRef := host + "/goodimage:v1" + jobs := []imageJob{ + {img: v1.Image{Name: badRef}, needsPubKey: true, key: writeTestPubKey(t), excludeExtras: true}, + {img: v1.Image{Name: goodRef}, excludeExtras: true}, + } + + s := newTestStore(t) + var buf bytes.Buffer + if err := runImageJobs(newTestContext(t), s, jobs, 1, defaultRootOpts(s.Root), defaultCliOpts(), log.NewRenderer(&buf)); err == nil { + t.Fatal("runImageJobs succeeded despite a verification failure; expected the default fail-the-run behavior") + } + + out := buf.String() + firstGood := strings.Index(out, goodRef) + if firstGood == -1 { + t.Fatalf("the good image never reached the progress display; full output:\n%s", out) + } + if last := strings.LastIndex(out, badRef); last > firstGood { + t.Errorf("%q is still drawn after the good image started; its progress row was never cleared\nfull output:\n%s", badRef, out) + } +} + +// Four different failures must not all read as "your signature is bad". A user +// whose registry is unreachable, whose reference is malformed, or whose key +// file is missing has a different problem to fix in each case. +// +// It also asserts resolveAndVerify's pinned-digest-on-failure contract: a +// failure at or before the pin (malformed reference, unresolvable digest) has +// no digest to hand back, but a failure after a successful pin (verifier +// setup, signature check) must still return it, so the caller can store +// exactly the bytes that were checked even when the check failed -- +// wantPinned is "" for the former and the seeded image's digest for the +// latter. +func TestResolveAndVerifyNamesTheStageThatFailed(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + badsig := seedImage(t, host, "badsig", "v1", remoteOpts...) + seedCosignV2Artifacts(t, host, "badsig", badsig, remoteOpts...) + badsigDigest, err := badsig.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + unreadablekey := seedImage(t, host, "unreadablekey", "v1", remoteOpts...) + unreadablekeyDigest, err := unreadablekey.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + + tests := []struct { + name string + job imageJob + want string + wantPinned string // "" means the failure happened at or before the pin + }{ + { + name: "malformed reference", + job: imageJob{img: v1.Image{Name: "NOT A REF"}, needsPubKey: true, key: writeTestPubKey(t)}, + want: "unable to parse image reference", + wantPinned: "", + }, + { + // 127.0.0.1:1 refuses connections, so this never reaches cosign. + name: "unreachable registry", + job: imageJob{img: v1.Image{Name: "127.0.0.1:1/absent/image:v1"}, needsPubKey: true, key: writeTestPubKey(t)}, + want: "unable to resolve image digest", + wantPinned: "", + }, + { + name: "unreadable key", + job: imageJob{img: v1.Image{Name: host + "/unreadablekey:v1"}, needsPubKey: true, key: filepath.Join(t.TempDir(), "missing.pub")}, + want: "unable to configure signature verification", + wantPinned: unreadablekeyDigest.String(), + }, + { + name: "signature that does not check out", + job: imageJob{img: v1.Image{Name: host + "/badsig:v1"}, needsPubKey: true, key: writeTestPubKey(t)}, + want: "signature verification failed", + wantPinned: badsigDigest.String(), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rso, ro := defaultRootOpts(t.TempDir()), defaultCliOpts() + cache := cosign.NewCache(rso, ro) + defer cache.Close() + + pinned, err := resolveAndVerify(newTestContext(t), cache, tt.job, rso, ro) + if err == nil { + t.Fatal("resolveAndVerify succeeded") + } + var ve *verifyError + if !errors.As(err, &ve) { + t.Fatalf("got %T (%v), want a *verifyError naming the failed stage", err, err) + } + if ve.stage != tt.want { + t.Fatalf("stage = %q, want %q (underlying: %v)", ve.stage, tt.want, ve.err) + } + if pinned != tt.wantPinned { + t.Fatalf("pinned digest = %q, want %q", pinned, tt.wantPinned) + } + }) + } +} + +// cosign's ErrNoMatchingSignatures joins one failure sentence per +// signature-verification attempt with "\n ", so the same sentence can appear +// several times back to back for a single failed image. flattenVerifyError +// collapses those consecutive repeats so logVerifyFailure prints one line. +func TestFlattenVerifyError(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + { + name: "nil error", + err: nil, + want: "", + }, + { + name: "empty message", + err: errors.New(""), + want: "", + }, + { + name: "single-line error unchanged", + err: errors.New("invalid signature when validating ASN.1 encoded signature"), + want: "invalid signature when validating ASN.1 encoded signature", + }, + { + name: "three identical consecutive fragments collapse with count", + err: errors.New("invalid signature\n invalid signature\n invalid signature"), + want: "invalid signature (x3)", + }, + { + name: "distinct fragments joined with semicolons", + err: errors.New("fragment one\nfragment two\nfragment three"), + want: "fragment one; fragment two; fragment three", + }, + { + name: "leading and trailing whitespace is trimmed before comparing", + err: errors.New(" invalid signature \n\tinvalid signature\t\n invalid signature "), + want: "invalid signature (x3)", + }, + { + name: "empty fragments between real ones are dropped", + err: errors.New("fragment one\n\nfragment two"), + want: "fragment one; fragment two", + }, + { + name: "non-consecutive repeats are not merged", + err: errors.New("fragment one\nfragment two\nfragment one"), + want: "fragment one; fragment two; fragment one", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := flattenVerifyError(tt.err); got != tt.want { + t.Fatalf("flattenVerifyError(%v) = %q, want %q", tt.err, got, tt.want) + } + }) + } +} + +// The digest pin is the one network call on the verified path that can lose a +// valid, signed image to a transient blip, so it carries the same --retries +// budget as the verify and store steps. retry.Operation's exhaustion wrapper is +// the evidence: a bare remote.Head error would not have one. +func TestResolveAndVerifyRetriesTheDigestPin(t *testing.T) { + rso, ro := defaultRootOpts(t.TempDir()), defaultCliOpts() // Retries: 1 + cache := cosign.NewCache(rso, ro) + defer cache.Close() + + j := imageJob{img: v1.Image{Name: "127.0.0.1:1/absent/image:v1"}, needsPubKey: true, key: writeTestPubKey(t)} + _, err := resolveAndVerify(newTestContext(t), cache, j, rso, ro) + if err == nil { + t.Fatal("resolveAndVerify succeeded against a refused connection") + } + if want := fmt.Sprintf("operation unsuccessful after %d attempts", rso.Retries); !strings.Contains(err.Error(), want) { + t.Fatalf("resolve error %q does not carry %q; the digest pin is running outside retry.Operation", err, want) + } +} + +// Under fail-fast, one real storeImage failure cancels the group and every +// other in-flight job's verification collapses with it. Those cancellations +// must not be reported as signature problems, or a single bad image produces +// N-1 lines telling the user their signing is broken. storeImage guards the +// identical case at add.go's context.Canceled branch. +// +// This is a default-only (ignoreErrors=false) test: storeImage's ignoreErrors +// branch swallows every error it sees, including a mid-flight +// context.Canceled, before ever asking what the error was -- so under +// --ignore-errors a plain storeImage failure like this one never reaches the +// point of cancelling gctx at all, and there is nothing left to cascade to the +// verifying jobs. See TestRunImageJobs_AlreadyCancelledContext for the +// --ignore-errors-covering case, which cancels the context from outside the +// run instead of relying on one job's failure to do it. +// +// concurrency=1 makes this deterministic: errgroup.SetLimit(1) runs the jobs in +// slice order, so the verifying jobs are guaranteed to start after the bad +// image has already failed and cancelled gctx. +func TestRunImageJobs_CancelledVerifyIsNotReportedAsSignatureFailure(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + + jobs := []imageJob{{img: v1.Image{Name: host + "/does-not-exist:latest"}}} + var verifyRefs []string + for i := 0; i < 3; i++ { + repo := fmt.Sprintf("signed%d", i) + _, keyPath := seedSignedImage(t, host, repo, "v1", remoteOpts...) + ref := host + "/" + repo + ":v1" + verifyRefs = append(verifyRefs, ref) + jobs = append(jobs, imageJob{img: v1.Image{Name: ref}, needsPubKey: true, key: keyPath, excludeExtras: true}) + } + + s := newTestStore(t) + var buf bytes.Buffer + prevGlobalLevel := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(prevGlobalLevel) }) + ctx := zerolog.New(&buf).Level(zerolog.DebugLevel).WithContext(context.Background()) + + err := runImageJobs(ctx, s, jobs, 1, defaultRootOpts(s.Root), defaultCliOpts(), nil) + if err == nil { + t.Fatal("runImageJobs: expected the real failure to propagate, got nil") + } + if got := countArtifactsInStore(t, s); got != 0 { + t.Fatalf("store holds %d artifacts, want 0; a cancelled run must not store any of the still-verifying images", got) + } + + for _, line := range strings.Split(buf.String(), "\n") { + if !strings.Contains(line, `"level":"error"`) { + continue + } + for _, ref := range verifyRefs { + if strings.Contains(line, ref) { + t.Errorf("a cancelled job logged at ERROR, which buries the one real failure:\n%s", line) + } + } + } +} + +// TestRunImageJobs_AlreadyCancelledContext covers requirement 3's +// both-ignore-errors-settings case directly: an already-cancelled context +// makes storeImage's own early ctx.Err() check fire (add.go, before any +// ignoreErrors branching) for the job that needs no verification, and makes +// resolveAndVerify's pinDigest -> retry.Operation observe ctx.Err() before its +// first attempt for the job that does. logVerifyFailure treats that +// context.Canceled as an always-propagate case regardless of ignoreErrors, so +// both jobs fail and nothing is stored either way. +// +// This is deliberately not a job-triggers-cancellation-of-another-job test: +// per TestRunImageJobs_CancelledVerifyIsNotReportedAsSignatureFailure's doc, +// that cascade cannot happen under --ignore-errors=true in the current +// architecture, since storeImage's ignoreErrors branch swallows a job's own +// failure before it ever reaches gctx's cancel. Cancelling from outside the +// run (as a Ctrl-C would) is the realistic trigger for this case and needs no +// blocking handler or timing to be deterministic. +func TestRunImageJobs_AlreadyCancelledContext(t *testing.T) { + for _, ignoreErrors := range []bool{false, true} { + t.Run(fmt.Sprintf("ignoreErrors=%v", ignoreErrors), func(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + _, keyPath := seedSignedImage(t, host, "signed", "v1", remoteOpts...) + seedImage(t, host, "plain", "v1", remoteOpts...) + + s := newTestStore(t) + ro := defaultCliOpts() + ro.IgnoreErrors = ignoreErrors + jobs := []imageJob{ + {img: v1.Image{Name: host + "/signed:v1"}, needsPubKey: true, key: keyPath, excludeExtras: true}, + {img: v1.Image{Name: host + "/plain:v1"}, excludeExtras: true}, + } + + zl := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(zl.WithContext(context.Background())) + cancel() + + if err := runImageJobs(ctx, s, jobs, 2, defaultRootOpts(s.Root), ro, nil); err == nil { + t.Fatal("runImageJobs succeeded against an already-cancelled context") + } + if got := countArtifactsInStore(t, s); got != 0 { + t.Fatalf("store holds %d artifacts, want 0; an already-cancelled run must not store anything", got) + } + }) + } +} diff --git a/cmd/hauler/cli/store/testhelpers_test.go b/cmd/hauler/cli/store/testhelpers_test.go index 3e840fd..ca52688 100644 --- a/cmd/hauler/cli/store/testhelpers_test.go +++ b/cmd/hauler/cli/store/testhelpers_test.go @@ -6,11 +6,24 @@ package store // helpers like storeImage, storeFile, rewriteReference, etc. import ( + "bytes" "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" "io" + golog "log" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" + "sync" "testing" "github.com/google/go-containerregistry/pkg/name" @@ -25,6 +38,11 @@ import ( digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog" + ociempty "github.com/sigstore/cosign/v3/pkg/oci/empty" + ocimutate "github.com/sigstore/cosign/v3/pkg/oci/mutate" + ocistatic "github.com/sigstore/cosign/v3/pkg/oci/static" + "github.com/sigstore/sigstore/pkg/signature" + "github.com/sigstore/sigstore/pkg/signature/payload" "helm.sh/helm/v4/pkg/action" "hauler.dev/go/hauler/v2/internal/flags" @@ -32,6 +50,96 @@ import ( "hauler.dev/go/hauler/v2/pkg/store" ) +// newTestKeyPair generates a fresh ECDSA P-256 key pair and writes the public +// half in the PEM form cosign's --key expects, returning the private key and +// that path. Generated rather than vendored so the tests never depend on a +// fixture whose algorithm cosign might later stop accepting. +func newTestKeyPair(t *testing.T) (*ecdsa.PrivateKey, string) { + t.Helper() + + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + der, err := x509.MarshalPKIXPublicKey(&priv.PublicKey) + if err != nil { + t.Fatalf("marshaling public key: %v", err) + } + + path := filepath.Join(t.TempDir(), "cosign.pub") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der}), 0o600); err != nil { + t.Fatalf("writing public key: %v", err) + } + return priv, path +} + +// writeTestPubKey returns the path to a public key with no signed image behind +// it -- for tests that only need verification to fail closed. +func writeTestPubKey(t *testing.T) string { + t.Helper() + _, path := newTestKeyPair(t) + return path +} + +// seedSignedImage pushes a random image and a genuine cosign v2 signature for +// it, returning the image and the path to the public key that verifies it. +// +// The signature is built the way cosign itself builds one -- a simplesigning +// payload naming the image's digest, signed with ECDSA-P256, carried as the +// dev.cosignproject.cosign/signature annotation on a layer of the +// sha256-.sig manifest -- so cosign.Verifier's library path accepts it +// with IgnoreTlog set. seedCosignV2Artifacts pushes the same tags with random +// content and is its fail-closed counterpart. +func seedSignedImage(t *testing.T, host, repo, tag string, opts ...remote.Option) (gcrv1.Image, string) { + t.Helper() + + img := seedImage(t, host, repo, tag, opts...) + hash, err := img.Digest() + if err != nil { + t.Fatalf("seedSignedImage digest: %v", err) + } + + priv, keyPath := newTestKeyPair(t) + sv, err := signature.LoadECDSASignerVerifier(priv, crypto.SHA256) + if err != nil { + t.Fatalf("seedSignedImage LoadECDSASignerVerifier: %v", err) + } + + digestRef, err := name.NewDigest(host+"/"+repo+"@"+hash.String(), name.Insecure) + if err != nil { + t.Fatalf("seedSignedImage NewDigest: %v", err) + } + // The payload binds the signature to this exact digest; cosign rejects a + // signature whose payload names a different one, which is what makes the + // pinning assertions meaningful. + payloadBytes, err := (&payload.Cosign{Image: digestRef}).MarshalJSON() + if err != nil { + t.Fatalf("seedSignedImage marshaling payload: %v", err) + } + rawSig, err := sv.SignMessage(bytes.NewReader(payloadBytes)) + if err != nil { + t.Fatalf("seedSignedImage SignMessage: %v", err) + } + + ociSig, err := ocistatic.NewSignature(payloadBytes, base64.StdEncoding.EncodeToString(rawSig)) + if err != nil { + t.Fatalf("seedSignedImage ocistatic.NewSignature: %v", err) + } + sigs, err := ocimutate.AppendSignatures(ociempty.Signatures(), false, ociSig) + if err != nil { + t.Fatalf("seedSignedImage AppendSignatures: %v", err) + } + + sigRef, err := name.NewTag(host+"/"+repo+":"+strings.ReplaceAll(hash.String(), ":", "-")+".sig", name.Insecure) + if err != nil { + t.Fatalf("seedSignedImage NewTag (sig): %v", err) + } + if err := remote.Write(sigRef, sigs, opts...); err != nil { + t.Fatalf("seedSignedImage writing signature: %v", err) + } + return img, keyPath +} + // newTestStore creates a fresh store in a temp directory. Fatal on error. func newTestStore(t *testing.T) *store.Layout { t.Helper() @@ -57,6 +165,56 @@ func newTestRegistry(t *testing.T) (host string, remoteOpts []remote.Option) { return host, remoteOpts } +// recordingHandler records the path of every request it serves before +// delegating. Concurrent handlers append to the same slice, so every method +// takes mu. +type recordingHandler struct { + next http.Handler + mu sync.Mutex + paths []string +} + +func (h *recordingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() + h.paths = append(h.paths, r.Method+" "+r.URL.Path) + h.mu.Unlock() + h.next.ServeHTTP(w, r) +} + +func (h *recordingHandler) reset() { + h.mu.Lock() + defer h.mu.Unlock() + h.paths = nil +} + +func (h *recordingHandler) countContaining(sub string) int { + h.mu.Lock() + defer h.mu.Unlock() + n := 0 + for _, p := range h.paths { + if strings.Contains(p, sub) { + n++ + } + } + return n +} + +func (h *recordingHandler) snapshot() []string { + h.mu.Lock() + defer h.mu.Unlock() + return append([]string(nil), h.paths...) +} + +// newRecordingRegistry mirrors newTestRegistry but records request paths, so a +// test can assert how a tag was reached and not just what was stored. +func newRecordingRegistry(t *testing.T) (string, []remote.Option, *recordingHandler) { + t.Helper() + rec := &recordingHandler{next: registry.New(registry.Logger(golog.New(io.Discard, "", 0)))} + srv := httptest.NewServer(rec) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://"), []remote.Option{remote.WithTransport(srv.Client().Transport)}, rec +} + // seedImage pushes a random single-platform image to the test registry. // repo is a bare path like "myorg/myimage"... tag is the image tag string. // Pass the remoteOpts from newTestRegistry so writes use the correct transport. @@ -205,6 +363,45 @@ func assertArtifactKindInStore(t *testing.T, s *store.Layout, refSubstring, kind } } +// storedDigest returns the digest of the first indexed descriptor whose ref +// annotation contains refSubstring, or "" if there is none. +func storedDigest(t *testing.T, s *store.Layout, refSubstring string) string { + t.Helper() + found := "" + if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error { + if found == "" && strings.Contains(desc.Annotations[ocispec.AnnotationRefName], refSubstring) { + found = desc.Digest.String() + } + return nil + }); err != nil { + t.Fatalf("storedDigest walk: %v", err) + } + return found +} + +// lastAuditEntryFlags reads /audit.log (see pkg/audit.Append / +// resolveDir) and returns the "flags" object of its last JSON line -- only +// populated at audit level "verbose", since audit.Append omits Flags +// otherwise. Fails the test if the file is missing, empty, or malformed. +func lastAuditEntryFlags(t *testing.T, haulerDir string) map[string]any { + t.Helper() + data, err := os.ReadFile(filepath.Join(haulerDir, "audit.log")) + if err != nil { + t.Fatalf("reading audit.log: %v", err) + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + if len(lines) == 0 || lines[len(lines)-1] == "" { + t.Fatalf("audit.log has no entries") + } + var entry struct { + Flags map[string]any `json:"flags"` + } + if err := json.Unmarshal([]byte(lines[len(lines)-1]), &entry); err != nil { + t.Fatalf("unmarshaling last audit.log line: %v\nline: %s", err, lines[len(lines)-1]) + } + return entry.Flags +} + // countArtifactsInStore returns the number of descriptors in the store index. func countArtifactsInStore(t *testing.T, s *store.Layout) int { t.Helper() diff --git a/cmd/hauler/cli/store_info_check_test.go b/cmd/hauler/cli/store_info_check_test.go index c75e20d..d87ca50 100644 --- a/cmd/hauler/cli/store_info_check_test.go +++ b/cmd/hauler/cli/store_info_check_test.go @@ -158,7 +158,7 @@ func TestStoreInfoCheck_JSON_CorruptArtifactInPayload(t *testing.T) { ctx := context.Background() seedInfoCheckImage(t, host, "test/corrupt", "v1") - if _, err := s.AddImage(ctx, host+"/test/corrupt:v1", "", true); err != nil { + if _, err := s.AddImage(ctx, host+"/test/corrupt:v1", "", true, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -213,7 +213,7 @@ func TestStoreInfoCheck_JSON_NoCheckStillWorks(t *testing.T) { ctx := context.Background() seedInfoCheckImage(t, host, "test/plain", "v1") - if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true); err != nil { + if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true, ""); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/go.mod b/go.mod index 44210e7..2c363c0 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,12 @@ require ( github.com/containerd/errdefs v1.0.0 github.com/distribution/distribution/v3 v3.1.1 github.com/distribution/reference v0.6.0 + github.com/dustin/go-humanize v1.0.1 github.com/google/go-containerregistry v0.21.8 github.com/google/uuid v1.6.0 github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 + github.com/mattn/go-isatty v0.0.20 github.com/mholt/archives v0.1.5 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v1.1.4 @@ -20,10 +22,12 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.35.1 github.com/sigstore/cosign/v3 v3.1.2 + github.com/sigstore/sigstore v1.10.8 github.com/sirupsen/logrus v1.9.4 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 golang.org/x/sync v0.22.0 + golang.org/x/term v0.45.0 gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v4 v4.2.3 k8s.io/apimachinery v0.36.3 @@ -131,7 +135,6 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emicklei/proto v1.14.3 // indirect @@ -217,7 +220,6 @@ require ( github.com/lib/pq v1.12.3 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/miekg/pkcs11 v1.1.2 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect @@ -272,7 +274,6 @@ require ( github.com/sigstore/protobuf-specs v0.5.1 // indirect github.com/sigstore/rekor v1.5.3 // indirect github.com/sigstore/rekor-tiles/v2 v2.3.0 // indirect - github.com/sigstore/sigstore v1.10.8 // indirect github.com/sigstore/sigstore-go v1.2.1 // indirect github.com/sigstore/timestamp-authority/v2 v2.1.2 // indirect github.com/sorairolake/lzip-go v0.3.8 // indirect @@ -337,7 +338,6 @@ require ( golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/term v0.45.0 // indirect golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.284.0 // indirect diff --git a/internal/flags/add.go b/internal/flags/add.go index ae77f51..75270b3 100644 --- a/internal/flags/add.go +++ b/internal/flags/add.go @@ -3,6 +3,8 @@ package flags import ( "github.com/spf13/cobra" "helm.sh/helm/v4/pkg/action" + + "hauler.dev/go/hauler/v2/pkg/consts" ) type AddImageOpts struct { @@ -58,6 +60,8 @@ type AddChartOpts struct { Platform string Registry string KubeVersion string + Concurrency int + NoProgress bool } func (o *AddChartOpts) AddFlags(cmd *cobra.Command) { @@ -85,4 +89,6 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) { f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image, e.g. linux/amd64") f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one") f.StringVar(&o.KubeVersion, "kube-version", "v1.34.1", "(Optional) Override the kubernetes version for helm template rendering") + f.IntVarP(&o.Concurrency, "concurrency", "j", consts.DefaultConcurrency, "(Optional) Maximum number of charts and their discovered images to fetch and store concurrently (1 = serial; also via HAULER_CONCURRENCY, explicit flag wins)") + f.BoolVar(&o.NoProgress, "no-progress", false, "(Optional) Disable the live progress display") } diff --git a/internal/flags/cli.go b/internal/flags/cli.go index a85e3f6..4bfab7f 100644 --- a/internal/flags/cli.go +++ b/internal/flags/cli.go @@ -14,6 +14,6 @@ func AddRootFlags(cmd *cobra.Command, ro *CliRootOpts) { pf.StringVarP(&ro.LogLevel, "log-level", "l", "", "Set the logging level (i.e. info, debug, warn) (defaults info)") pf.StringVarP(&ro.HaulerDir, "haulerdir", "d", "", "Set the location of the hauler directory (default $HOME/.hauler)") - pf.BoolVar(&ro.IgnoreErrors, "ignore-errors", false, "Ignore/Bypass errors (i.e. warn on error) (defaults false)") + pf.BoolVar(&ro.IgnoreErrors, "ignore-errors", false, "Warn and continue instead of failing on errors, including storing images that failed verification (defaults false)") pf.StringVar(&ro.AuditLevel, "audit-level", "", "Set the audit logging level (none, standard, verbose) (defaults standard)") } diff --git a/internal/flags/concurrency.go b/internal/flags/concurrency.go new file mode 100644 index 0000000..1140082 --- /dev/null +++ b/internal/flags/concurrency.go @@ -0,0 +1,104 @@ +package flags + +import ( + "fmt" + "os" + "strconv" + + "hauler.dev/go/hauler/v2/pkg/consts" +) + +// ResolveConcurrency returns the effective --concurrency value for `store +// sync`, honoring explicit-flag > HAULER_CONCURRENCY env var > default +// precedence. Values < 1 are rejected outright, never clamped -- clamping +// would hide a typo'd --concurrency 0 or a bad env var behind "it just +// worked". +func ResolveConcurrency(flagChanged bool, flagValue int) (int, error) { + if flagChanged { + if flagValue < 1 { + return 0, fmt.Errorf("--concurrency must be >= 1, got %d", flagValue) + } + return flagValue, nil + } + + if v := os.Getenv(consts.HaulerConcurrency); v != "" { + n, err := strconv.Atoi(v) + if err != nil { + return 0, fmt.Errorf("invalid %s value %q: %w", consts.HaulerConcurrency, v, err) + } + if n < 1 { + return 0, fmt.Errorf("%s must be >= 1, got %d", consts.HaulerConcurrency, n) + } + return n, nil + } + + return consts.DefaultConcurrency, nil +} + +// BlobConcurrencyFor derives the OCI layout's blob-write concurrency +// ceiling from a resolved --concurrency value: max(16, 4*concurrency), +// capped at 32. The floor of consts.DefaultBlobConcurrency (16) matters: +// layer writes within a single image are bounded only by the shared +// blobSem (content.OCI.blobSem), not by --concurrency, so a naive +// 4*concurrency would make --concurrency 1 slower than today's behavior on +// images with more than 4 layers. The cap keeps a pathologically wide image +// or large --concurrency from opening unbounded sockets. +func BlobConcurrencyFor(concurrency int) int { + n := 4 * concurrency + if n < consts.DefaultBlobConcurrency { + n = consts.DefaultBlobConcurrency + } + if n > 32 { + n = 32 + } + return n +} + +// ResolveBlobConcurrency returns an explicitly-requested blob-write +// concurrency ceiling, honoring flag > HAULER_BLOB_CONCURRENCY precedence. +// It returns 0 when neither was supplied ("not specified"); the caller +// picks the fallback (SyncBlobConcurrency derives one; every other store +// subcommand leaves it to consts.DefaultBlobConcurrency). Unlike +// ResolveConcurrency, 0 here is a deliberate "auto" sentinel rather than an +// error, letting an explicit value bypass BlobConcurrencyFor's floor and +// cap entirely -- the only way to measure disk fan-out on a low-IOPS +// volume, since the floor of 16 otherwise keeps --concurrency 1 at 16 +// concurrent blob writes. Negative values and unparseable env values are +// still rejected outright, matching ResolveConcurrency's rule that a typo +// must surface rather than appear to work. +func ResolveBlobConcurrency(flagValue int) (int, error) { + if flagValue < 0 { + return 0, fmt.Errorf("--blob-concurrency must be >= 0, got %d", flagValue) + } + if flagValue > 0 { + return flagValue, nil + } + + v := os.Getenv(consts.HaulerBlobConcurrency) + if v == "" { + return 0, nil + } + n, err := strconv.Atoi(v) + if err != nil { + return 0, fmt.Errorf("invalid %s value %q: %w", consts.HaulerBlobConcurrency, v, err) + } + if n < 1 { + return 0, fmt.Errorf("%s must be >= 1, got %d", consts.HaulerBlobConcurrency, n) + } + return n, nil +} + +// SyncBlobConcurrency resolves the effective blob-write ceiling for `store +// sync`: an explicit --blob-concurrency (or HAULER_BLOB_CONCURRENCY) value +// wins outright, otherwise the value is derived from the already-resolved +// --concurrency via BlobConcurrencyFor. It never returns 0. +func SyncBlobConcurrency(blobFlagValue, concurrency int) (int, error) { + bc, err := ResolveBlobConcurrency(blobFlagValue) + if err != nil { + return 0, err + } + if bc == 0 { + bc = BlobConcurrencyFor(concurrency) + } + return bc, nil +} diff --git a/internal/flags/concurrency_test.go b/internal/flags/concurrency_test.go new file mode 100644 index 0000000..6b3f476 --- /dev/null +++ b/internal/flags/concurrency_test.go @@ -0,0 +1,298 @@ +package flags + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/cobra" + "helm.sh/helm/v4/pkg/action" + + "hauler.dev/go/hauler/v2/pkg/consts" +) + +func TestResolveConcurrency(t *testing.T) { + tests := []struct { + name string + flagChanged bool + flagValue int + env string // empty means unset + want int + wantErr bool + errContains string + }{ + { + name: "flag changed, valid value", + flagChanged: true, + flagValue: 8, + want: 8, + }, + { + name: "flag changed, value 1", + flagChanged: true, + flagValue: 1, + want: 1, + }, + { + name: "flag changed, zero is rejected", + flagChanged: true, + flagValue: 0, + wantErr: true, + errContains: "--concurrency must be >= 1", + }, + { + name: "flag changed, negative is rejected", + flagChanged: true, + flagValue: -3, + wantErr: true, + errContains: "--concurrency must be >= 1", + }, + { + name: "flag not changed, env set", + flagChanged: false, + flagValue: consts.DefaultConcurrency, + env: "6", + want: 6, + }, + { + name: "flag not changed, env invalid", + flagChanged: false, + flagValue: consts.DefaultConcurrency, + env: "not-a-number", + wantErr: true, + }, + { + name: "flag not changed, env zero is rejected", + flagChanged: false, + flagValue: consts.DefaultConcurrency, + env: "0", + wantErr: true, + errContains: consts.HaulerConcurrency, + }, + { + name: "flag not changed, env unset, default used", + flagChanged: false, + flagValue: consts.DefaultConcurrency, + want: consts.DefaultConcurrency, + }, + { + name: "explicit flag beats conflicting env", + flagChanged: true, + flagValue: 2, + env: "10", + want: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env != "" { + t.Setenv(consts.HaulerConcurrency, tt.env) + } + + got, err := ResolveConcurrency(tt.flagChanged, tt.flagValue) + if tt.wantErr { + if err == nil { + t.Fatalf("ResolveConcurrency() expected error, got nil (result %d)", got) + } + if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("ResolveConcurrency() error = %q, want substring %q", err.Error(), tt.errContains) + } + return + } + if err != nil { + t.Fatalf("ResolveConcurrency() unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("ResolveConcurrency() = %d, want %d", got, tt.want) + } + }) + } +} + +// TestAddChartOpts_ConcurrencyFlag proves `store add chart` registers +// --concurrency/-j with the same shorthand and default as `store sync`. +func TestAddChartOpts_ConcurrencyFlag(t *testing.T) { + o := &AddChartOpts{StoreRootOpts: &StoreRootOpts{}, ChartOpts: &action.ChartPathOptions{}} + cmd := &cobra.Command{Use: "chart"} + o.AddFlags(cmd) + + f := cmd.Flags().Lookup("concurrency") + if f == nil { + t.Fatal("expected --concurrency to be registered") + } + if f.Shorthand != "j" { + t.Errorf("shorthand = %q, want %q", f.Shorthand, "j") + } + if f.DefValue != fmt.Sprintf("%d", consts.DefaultConcurrency) { + t.Errorf("default = %q, want %d", f.DefValue, consts.DefaultConcurrency) + } + + if err := cmd.ParseFlags([]string{"-j", "9"}); err != nil { + t.Fatalf("ParseFlags: %v", err) + } + if o.Concurrency != 9 { + t.Errorf("o.Concurrency = %d, want 9", o.Concurrency) + } +} + +func TestBlobConcurrencyFor(t *testing.T) { + tests := []struct { + name string + concurrency int + want int + }{ + {name: "concurrency 1 floors at DefaultBlobConcurrency", concurrency: 1, want: 16}, + {name: "concurrency 4 stays at floor", concurrency: 4, want: 16}, + {name: "concurrency 8 hits the cap exactly", concurrency: 8, want: 32}, + {name: "concurrency 20 is capped at 32", concurrency: 20, want: 32}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := BlobConcurrencyFor(tt.concurrency); got != tt.want { + t.Errorf("BlobConcurrencyFor(%d) = %d, want %d", tt.concurrency, got, tt.want) + } + }) + } +} + +func TestResolveBlobConcurrency(t *testing.T) { + tests := []struct { + name string + flagValue int + env string // empty means unset + want int + wantErr bool + errContains string + }{ + { + name: "explicit flag wins", + flagValue: 4, + env: "9", + want: 4, + }, + { + name: "explicit flag below the floor is honored", + flagValue: 1, + want: 1, + }, + { + name: "explicit flag above the cap is honored", + flagValue: 64, + want: 64, + }, + { + name: "zero with no env means not specified", + flagValue: 0, + want: 0, + }, + { + name: "zero falls back to env", + flagValue: 0, + env: "6", + want: 6, + }, + { + name: "negative flag is rejected", + flagValue: -2, + wantErr: true, + errContains: "--blob-concurrency must be >= 0", + }, + { + name: "invalid env is rejected", + flagValue: 0, + env: "not-a-number", + wantErr: true, + errContains: "invalid HAULER_BLOB_CONCURRENCY", + }, + { + name: "env below one is rejected", + flagValue: 0, + env: "0", + wantErr: true, + errContains: "HAULER_BLOB_CONCURRENCY must be >= 1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env != "" { + t.Setenv(consts.HaulerBlobConcurrency, tt.env) + } + got, err := ResolveBlobConcurrency(tt.flagValue) + if tt.wantErr { + if err == nil { + t.Fatalf("expected an error, got nil") + } + if !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("error %q does not contain %q", err.Error(), tt.errContains) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("got %d, want %d", got, tt.want) + } + }) + } +} + +func TestSyncBlobConcurrency(t *testing.T) { + tests := []struct { + name string + blobFlag int + concurrency int + env string + want int + }{ + { + name: "explicit flag overrides the derived floor", + blobFlag: 1, + concurrency: 1, + want: 1, + }, + { + name: "unset derives from concurrency, floored at 16", + blobFlag: 0, + concurrency: 1, + want: 16, + }, + { + name: "unset derives 4x concurrency", + blobFlag: 0, + concurrency: 5, + want: 20, + }, + { + name: "unset derivation is capped at 32", + blobFlag: 0, + concurrency: 100, + want: 32, + }, + { + name: "env overrides the derivation", + blobFlag: 0, + concurrency: 5, + env: "2", + want: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env != "" { + t.Setenv(consts.HaulerBlobConcurrency, tt.env) + } + got, err := SyncBlobConcurrency(tt.blobFlag, tt.concurrency) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("got %d, want %d", got, tt.want) + } + }) + } +} diff --git a/internal/flags/ignore_errors.go b/internal/flags/ignore_errors.go new file mode 100644 index 0000000..509d45a --- /dev/null +++ b/internal/flags/ignore_errors.go @@ -0,0 +1,24 @@ +package flags + +import ( + "os" + + "hauler.dev/go/hauler/v2/pkg/consts" +) + +// ShouldIgnoreErrors reports whether the CLI should treat operation failures as +// non-fatal (log + continue) rather than aborting, based on the --ignore-errors +// flag or the HAULER_IGNORE_ERRORS environment variable. It is a pure read: it +// never mutates ro. This replaces the old pattern of writing os.Getenv's result +// back into ro.IgnoreErrors, which was a data race the moment callers ran +// concurrently (ro is a single *CliRootOpts shared across the whole CLI +// invocation). +func ShouldIgnoreErrors(ro *CliRootOpts) bool { + if ro == nil { + return false + } + if ro.IgnoreErrors { + return true + } + return os.Getenv(consts.HaulerIgnoreErrors) == "true" +} diff --git a/internal/flags/ignore_errors_test.go b/internal/flags/ignore_errors_test.go new file mode 100644 index 0000000..160eff6 --- /dev/null +++ b/internal/flags/ignore_errors_test.go @@ -0,0 +1,72 @@ +package flags + +import ( + "testing" + + "hauler.dev/go/hauler/v2/pkg/consts" +) + +func TestShouldIgnoreErrors(t *testing.T) { + tests := []struct { + name string + ro *CliRootOpts + env string // empty means unset + want bool + }{ + { + name: "flag true, env unset", + ro: &CliRootOpts{IgnoreErrors: true}, + env: "", + want: true, + }, + { + name: "flag false, env true", + ro: &CliRootOpts{IgnoreErrors: false}, + env: "true", + want: true, + }, + { + name: "flag false, env unset", + ro: &CliRootOpts{IgnoreErrors: false}, + env: "", + want: false, + }, + { + name: "flag true, env false (flag wins)", + ro: &CliRootOpts{IgnoreErrors: true}, + env: "false", + want: true, + }, + { + name: "nil opts", + ro: nil, + env: "", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env != "" { + t.Setenv(consts.HaulerIgnoreErrors, tt.env) + } + + got := ShouldIgnoreErrors(tt.ro) + if got != tt.want { + t.Fatalf("ShouldIgnoreErrors() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestShouldIgnoreErrors_DoesNotMutate(t *testing.T) { + ro := &CliRootOpts{IgnoreErrors: false} + t.Setenv(consts.HaulerIgnoreErrors, "true") + + if got := ShouldIgnoreErrors(ro); !got { + t.Fatalf("ShouldIgnoreErrors() = %v, want true", got) + } + if ro.IgnoreErrors { + t.Fatal("ShouldIgnoreErrors must not mutate ro.IgnoreErrors") + } +} diff --git a/internal/flags/no_progress_test.go b/internal/flags/no_progress_test.go new file mode 100644 index 0000000..1e4fd78 --- /dev/null +++ b/internal/flags/no_progress_test.go @@ -0,0 +1,82 @@ +package flags + +import ( + "testing" + + "github.com/spf13/cobra" + "helm.sh/helm/v4/pkg/action" +) + +// TestSyncOpts_NoProgressFlag proves --no-progress registered by +// SyncOpts.AddFlags sets o.NoProgress, and defaults to false when unset. +func TestSyncOpts_NoProgressFlag(t *testing.T) { + tests := []struct { + name string + args []string + want bool + }{ + { + name: "flag not passed defaults false", + args: []string{}, + want: false, + }, + { + name: "--no-progress sets true", + args: []string{"--no-progress"}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := &SyncOpts{StoreRootOpts: &StoreRootOpts{}} + cmd := &cobra.Command{Use: "sync"} + o.AddFlags(cmd) + + if err := cmd.ParseFlags(tt.args); err != nil { + t.Fatalf("ParseFlags(%v): %v", tt.args, err) + } + + if o.NoProgress != tt.want { + t.Errorf("o.NoProgress = %v, want %v", o.NoProgress, tt.want) + } + }) + } +} + +// TestAddChartOpts_NoProgressFlag proves --no-progress registered by +// AddChartOpts.AddFlags sets o.NoProgress, and defaults to false when unset. +func TestAddChartOpts_NoProgressFlag(t *testing.T) { + tests := []struct { + name string + args []string + want bool + }{ + { + name: "flag not passed defaults false", + args: []string{}, + want: false, + }, + { + name: "--no-progress sets true", + args: []string{"--no-progress"}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := &AddChartOpts{StoreRootOpts: &StoreRootOpts{}, ChartOpts: &action.ChartPathOptions{}} + cmd := &cobra.Command{Use: "chart"} + o.AddFlags(cmd) + + if err := cmd.ParseFlags(tt.args); err != nil { + t.Fatalf("ParseFlags(%v): %v", tt.args, err) + } + + if o.NoProgress != tt.want { + t.Errorf("o.NoProgress = %v, want %v", o.NoProgress, tt.want) + } + }) + } +} diff --git a/internal/flags/store.go b/internal/flags/store.go index e17be34..3bff356 100644 --- a/internal/flags/store.go +++ b/internal/flags/store.go @@ -22,6 +22,15 @@ type StoreRootOpts struct { StoreDir string Retries int TempOverride string + + // BlobConcurrency overrides the store's default blob-write concurrency + // ceiling (content.OCI.blobSem) when > 0, bound to the + // --blob-concurrency persistent flag (0 means "auto"). Populated by one + // of two idempotent paths: `store sync`'s PreRunE (derives a value from + // --concurrency when none was given) or Store() itself (consults + // HAULER_BLOB_CONCURRENCY, so subcommands with no PreRunE still honor + // the env var). + BlobConcurrency int } func (o *StoreRootOpts) AddFlags(cmd *cobra.Command) { @@ -29,6 +38,7 @@ func (o *StoreRootOpts) AddFlags(cmd *cobra.Command) { pf.StringVarP(&o.StoreDir, "store", "s", "", "Set the directory to use for the content store") pf.IntVarP(&o.Retries, "retries", "r", consts.DefaultRetries, "Set the number of retries for operations") pf.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directory determined by the OS") + pf.IntVar(&o.BlobConcurrency, "blob-concurrency", 0, fmt.Sprintf("(Optional) Override the maximum number of concurrent blob writes (0 auto-derives from --concurrency where set, otherwise defaults to %d)", consts.DefaultBlobConcurrency)) } func (o *StoreRootOpts) Store(ctx context.Context, ro *CliRootOpts) (*store.Layout, error) { @@ -79,7 +89,25 @@ func (o *StoreRootOpts) Store(ctx context.Context, ro *CliRootOpts) (*store.Layo return nil, err } - s, err := store.NewLayout(abs, store.WithHaulerDir(haulerDir)) + // Always resolve, never just "when unset": this picks up + // HAULER_BLOB_CONCURRENCY for subcommands with no PreRunE of their own + // (add, load, copy, serve, extract...) and validates whatever value is + // already present -- a `o.BlobConcurrency == 0` guard would let a + // typo'd negative value skip validation and fail a later `> 0` check + // silently. This stays idempotent for `store sync`, whose PreRunE has + // already resolved a non-zero value: ResolveBlobConcurrency returns a + // positive input unchanged. + bc, err := ResolveBlobConcurrency(o.BlobConcurrency) + if err != nil { + return nil, err + } + o.BlobConcurrency = bc + + opts := []store.Options{store.WithHaulerDir(haulerDir)} + if o.BlobConcurrency > 0 { + opts = append(opts, store.WithBlobConcurrency(o.BlobConcurrency)) + } + s, err := store.NewLayout(abs, opts...) if err != nil { return nil, err } diff --git a/internal/flags/store_blob_concurrency_test.go b/internal/flags/store_blob_concurrency_test.go new file mode 100644 index 0000000..f9f07fa --- /dev/null +++ b/internal/flags/store_blob_concurrency_test.go @@ -0,0 +1,133 @@ +package flags + +import ( + "testing" + + "github.com/spf13/cobra" + "hauler.dev/go/hauler/v2/pkg/consts" +) + +// TestAddFlagsRegistersBlobConcurrency verifies the flag is registered as a +// persistent flag on the parent command, so every store subcommand inherits +// it rather than only `store sync`. +func TestAddFlagsRegistersBlobConcurrency(t *testing.T) { + o := &StoreRootOpts{} + cmd := &cobra.Command{Use: "store"} + o.AddFlags(cmd) + + f := cmd.PersistentFlags().Lookup("blob-concurrency") + if f == nil { + t.Fatal("expected --blob-concurrency to be registered as a persistent flag") + } + if f.DefValue != "0" { + t.Fatalf("expected default value 0 (auto), got %q", f.DefValue) + } +} + +// TestStoreResolvesBlobConcurrencyFromEnv verifies that a subcommand which +// never runs sync's PreRunE -- `store add`, `store load`, `store copy` -- +// still picks up HAULER_BLOB_CONCURRENCY. +func TestStoreResolvesBlobConcurrencyFromEnv(t *testing.T) { + t.Setenv(consts.HaulerBlobConcurrency, "3") + t.Setenv(consts.HaulerStoreDir, t.TempDir()) + t.Setenv(consts.HaulerDir, t.TempDir()) + + o := &StoreRootOpts{} + if _, err := o.Store(t.Context(), &CliRootOpts{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if o.BlobConcurrency != 3 { + t.Fatalf("got BlobConcurrency %d, want 3", o.BlobConcurrency) + } +} + +// TestStoreLeavesExplicitBlobConcurrencyAlone verifies Store() does not +// overwrite a value already resolved by sync's PreRunE. +func TestStoreLeavesExplicitBlobConcurrencyAlone(t *testing.T) { + t.Setenv(consts.HaulerBlobConcurrency, "3") + t.Setenv(consts.HaulerStoreDir, t.TempDir()) + t.Setenv(consts.HaulerDir, t.TempDir()) + + o := &StoreRootOpts{BlobConcurrency: 20} + if _, err := o.Store(t.Context(), &CliRootOpts{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if o.BlobConcurrency != 20 { + t.Fatalf("got BlobConcurrency %d, want 20 (env must not override)", o.BlobConcurrency) + } +} + +// TestStoreRejectsNegativeBlobConcurrency verifies that a negative +// --blob-concurrency passed to a subcommand with no PreRunE of its own (add, +// load, copy, serve, extract) surfaces an error instead of being silently +// dropped. Before the fix, Store() only called ResolveBlobConcurrency when +// o.BlobConcurrency == 0, so a negative value skipped validation entirely +// and was never rejected. +func TestStoreRejectsNegativeBlobConcurrency(t *testing.T) { + t.Setenv(consts.HaulerStoreDir, t.TempDir()) + t.Setenv(consts.HaulerDir, t.TempDir()) + + o := &StoreRootOpts{BlobConcurrency: -5} + if _, err := o.Store(t.Context(), &CliRootOpts{}); err == nil { + t.Fatal("expected an error for negative --blob-concurrency, got nil") + } +} + +// TestAddChartConcurrencyDerivesBlobConcurrency mirrors the two-step +// composition `store add chart`'s PreRunE performs -- ResolveConcurrency +// then SyncBlobConcurrency, the same pairing `store sync`'s PreRunE already +// uses -- and pins that an explicit --blob-concurrency still wins over +// whatever --concurrency would otherwise derive. +func TestAddChartConcurrencyDerivesBlobConcurrency(t *testing.T) { + tests := []struct { + name string + flagChanged bool + concurrency int + blobConcurrency int + wantConcurrency int + wantBlob int + }{ + { + name: "default concurrency derives 4x, above the floor", + flagChanged: false, + concurrency: consts.DefaultConcurrency, + wantConcurrency: consts.DefaultConcurrency, + wantBlob: consts.DefaultConcurrency * 4, + }, + { + name: "explicit --concurrency derives above the floor", + flagChanged: true, + concurrency: 8, + wantConcurrency: 8, + wantBlob: 32, + }, + { + name: "explicit --blob-concurrency wins over the derived value", + flagChanged: true, + concurrency: 8, + blobConcurrency: 2, + wantConcurrency: 8, + wantBlob: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n, err := ResolveConcurrency(tt.flagChanged, tt.concurrency) + if err != nil { + t.Fatalf("ResolveConcurrency: %v", err) + } + if n != tt.wantConcurrency { + t.Errorf("ResolveConcurrency() = %d, want %d", n, tt.wantConcurrency) + } + + bc, err := SyncBlobConcurrency(tt.blobConcurrency, n) + if err != nil { + t.Fatalf("SyncBlobConcurrency: %v", err) + } + if bc != tt.wantBlob { + t.Errorf("SyncBlobConcurrency() = %d, want %d", bc, tt.wantBlob) + } + }) + } +} diff --git a/internal/flags/sync.go b/internal/flags/sync.go index 6a0a550..1e483c4 100644 --- a/internal/flags/sync.go +++ b/internal/flags/sync.go @@ -2,6 +2,8 @@ package flags import ( "github.com/spf13/cobra" + + "hauler.dev/go/hauler/v2/pkg/consts" ) type SyncOpts struct { @@ -21,6 +23,8 @@ type SyncOpts struct { Tlog bool ExcludeExtras bool DryRun bool + Concurrency int + NoProgress bool } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { @@ -41,4 +45,6 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.BoolVar(&o.Tlog, "use-tlog-verify", false, "(Optional) Allow transparency log verification (defaults to false)") f.BoolVar(&o.ExcludeExtras, "exclude-extras", false, "(Optional) Exclude cosign signatures, attestations, SBOMs, and OCI referrers when pulling images") f.BoolVar(&o.DryRun, "dry-run", false, "(Optional) Output product manifest content to stdout instead of processing it (requires --products)") + f.IntVarP(&o.Concurrency, "concurrency", "j", consts.DefaultConcurrency, "(Optional) Maximum number of artifacts to fetch and store concurrently (1 = serial; also via HAULER_CONCURRENCY, explicit flag wins)") + f.BoolVar(&o.NoProgress, "no-progress", false, "(Optional) Disable the live progress display") } diff --git a/pkg/artifacts/file/cache.go b/pkg/artifacts/file/cache.go new file mode 100644 index 0000000..7c44d66 --- /dev/null +++ b/pkg/artifacts/file/cache.go @@ -0,0 +1,89 @@ +package file + +import ( + "context" + "sync" + + gv1 "github.com/google/go-containerregistry/pkg/v1" + "golang.org/x/sync/singleflight" +) + +// LayerCache lets multiple File instances that share the same source Path +// (e.g. two manifest entries pointing at the same URL, one plain and one +// with a name override) skip redundant fetches of the underlying content. +// Scope it to the caller (typically one sync run) -- construct with +// NewLayerCache and attach via WithLayerCacheContext. +// +// Sharing the cached gv1.Layer is safe despite differing name overrides: +// compute() always rebuilds its own descriptor and overwrites the Title +// with its own client.Name(Path) afterward, so only the fetch and +// digest/diffID hashing (getter.Client.LayerFrom) is actually shared. +type LayerCache struct { + group singleflight.Group + mu sync.RWMutex + cache map[string]gv1.Layer +} + +// NewLayerCache returns an empty LayerCache. +func NewLayerCache() *LayerCache { + return &LayerCache{cache: make(map[string]gv1.Layer)} +} + +// getOrFetch returns the cached layer for path if present; otherwise it +// calls fetch, sharing the in-flight call across concurrent callers for the +// same path (singleflight) and caching only on success. A failed fetch is +// deliberately not cached: singleflight forgets the flight once it returns, +// so the next call -- a new job or a retry.Operation retry -- starts fresh +// rather than replaying a poisoned error. +func (c *LayerCache) getOrFetch(path string, fetch func() (gv1.Layer, error)) (gv1.Layer, error) { + c.mu.RLock() + l, ok := c.cache[path] + c.mu.RUnlock() + if ok { + return l, nil + } + + v, err, _ := c.group.Do(path, func() (interface{}, error) { + // Re-check under the flight: another flight for this path may have + // completed and cached a result while this goroutine queued behind it. + c.mu.RLock() + l, ok := c.cache[path] + c.mu.RUnlock() + if ok { + return l, nil + } + + l, err := fetch() + if err != nil { + return nil, err + } + + c.mu.Lock() + c.cache[path] = l + c.mu.Unlock() + return l, nil + }) + if err != nil { + return nil, err + } + return v.(gv1.Layer), nil +} + +// layerCacheKey is the context key WithLayerCacheContext/layerCacheFromContext use. +type layerCacheKey struct{} + +// WithLayerCacheContext attaches c to ctx so compute() picks it up via +// layerCacheFromContext -- the same ctx-attached side-channel idiom +// pkg/store uses for ImageStats (store.WithImageStats). +func WithLayerCacheContext(ctx context.Context, c *LayerCache) context.Context { + return context.WithValue(ctx, layerCacheKey{}, c) +} + +// layerCacheFromContext returns the *LayerCache attached via +// WithLayerCacheContext, or nil if none was attached -- the common case for +// callers not coordinating a batch of File instances that might share a +// Path (a single `store add file` call, existing unit tests, etc). +func layerCacheFromContext(ctx context.Context) *LayerCache { + c, _ := ctx.Value(layerCacheKey{}).(*LayerCache) + return c +} diff --git a/pkg/artifacts/file/cache_test.go b/pkg/artifacts/file/cache_test.go new file mode 100644 index 0000000..2ca571f --- /dev/null +++ b/pkg/artifacts/file/cache_test.go @@ -0,0 +1,218 @@ +package file_test + +import ( + "bytes" + "context" + "errors" + "io" + "net/url" + "sync" + "sync/atomic" + "testing" + "time" + + "hauler.dev/go/hauler/v2/pkg/artifacts" + "hauler.dev/go/hauler/v2/pkg/artifacts/file" + "hauler.dev/go/hauler/v2/pkg/getter" +) + +// countingGetter counts every Open call and, when failUntil > 0, fails the +// first failUntil calls before succeeding -- used to prove LayerCache +// doesn't permanently poison a path after a failed fetch. +type countingGetter struct { + data []byte + opens int32 + mu sync.Mutex + failCount int +} + +func (g *countingGetter) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + atomic.AddInt32(&g.opens, 1) + + g.mu.Lock() + shouldFail := g.failCount > 0 + if shouldFail { + g.failCount-- + } + g.mu.Unlock() + + if shouldFail { + return nil, errors.New("simulated transient fetch failure") + } + return io.NopCloser(bytes.NewReader(g.data)), nil +} + +func (g *countingGetter) Detect(u *url.URL) bool { return true } +func (g *countingGetter) Name(u *url.URL) string { return "shared" } +func (g *countingGetter) Config(u *url.URL) artifacts.Config { + return artifacts.ToConfig(struct { + Reference string `json:"reference"` + }{u.String()}, artifacts.WithConfigMediaType("application/vnd.test.config")) +} + +func newCountingClient(g *countingGetter, nameOverride string) *getter.Client { + return &getter.Client{ + Options: getter.ClientOptions{NameOverride: nameOverride}, + Getters: map[string]getter.Getter{"mock": g}, + } +} + +// TestLayerCache_DedupesFetchesAcrossFileInstances proves that two File +// instances sharing the same source Path -- e.g. two Files entries pointing +// at the identical URL, one plain and one with a name override, the shape +// testdata/hauler-manifest-pipeline.yaml uses -- fetch the underlying +// content exactly once when they share a *file.LayerCache via +// file.WithLayerCacheContext, even though each independently computes its +// own manifest. +func TestLayerCache_DedupesFetchesAcrossFileInstances(t *testing.T) { + g := &countingGetter{data: []byte("shared content")} + cache := file.NewLayerCache() + + baseCtx := file.WithLayerCacheContext(context.Background(), cache) + + f1 := file.NewFile("mock://shared/path", file.WithClient(newCountingClient(g, "")), file.WithContext(baseCtx)) + f2 := file.NewFile("mock://shared/path", file.WithClient(newCountingClient(g, "renamed.sh")), file.WithContext(baseCtx)) + + if _, err := f1.Layers(); err != nil { + t.Fatalf("f1.Layers(): %v", err) + } + if _, err := f2.Layers(); err != nil { + t.Fatalf("f2.Layers(): %v", err) + } + + if got := atomic.LoadInt32(&g.opens); got != 1 { + t.Errorf("expected exactly 1 Open call (layer.FromOpener opens once and reuses the digest as diffID) from a single shared fetch, got %d", got) + } + + // Each instance still computes its own correct Title annotation despite + // sharing the underlying fetched layer. + m1, err := f1.Manifest() + if err != nil { + t.Fatalf("f1.Manifest(): %v", err) + } + m2, err := f2.Manifest() + if err != nil { + t.Fatalf("f2.Manifest(): %v", err) + } + if got := m1.Layers[0].Annotations["org.opencontainers.image.title"]; got != "shared" { + t.Errorf("f1 title = %q, want %q", got, "shared") + } + if got := m2.Layers[0].Annotations["org.opencontainers.image.title"]; got != "renamed.sh" { + t.Errorf("f2 title = %q, want %q", got, "renamed.sh") + } +} + +// TestLayerCache_DoesNotPoisonPathAfterFailure proves a failed fetch is not +// cached forever: a second File instance for the same path (simulating a +// retry.Operation retry, which reuses compute()'s memoization only within a +// single File -- a fresh File, as a new job attempt would construct, must +// still be able to succeed once the underlying transient failure clears). +func TestLayerCache_DoesNotPoisonPathAfterFailure(t *testing.T) { + g := &countingGetter{data: []byte("recovered content"), failCount: 1} + cache := file.NewLayerCache() + baseCtx := file.WithLayerCacheContext(context.Background(), cache) + + f1 := file.NewFile("mock://flaky/path", file.WithClient(newCountingClient(g, "")), file.WithContext(baseCtx)) + if _, err := f1.Layers(); err == nil { + t.Fatal("expected f1.Layers() to fail on the simulated first attempt, got nil error") + } + + f2 := file.NewFile("mock://flaky/path", file.WithClient(newCountingClient(g, "")), file.WithContext(baseCtx)) + if _, err := f2.Layers(); err != nil { + t.Fatalf("expected f2.Layers() to succeed after the transient failure cleared, got: %v", err) + } + + if got := atomic.LoadInt32(&g.opens); got != 2 { + t.Errorf("expected exactly 2 Open calls (1 failed attempt + 1 for the succeeding attempt, since layer.FromOpener now opens once), got %d", got) + } +} + +// TestLayerCache_NilCacheInContext_FetchesNormally proves compute() falls +// back to a direct, uncached fetch when no LayerCache is attached to ctx -- +// the common case (a single `store add file` call, or any File built +// without file.WithLayerCacheContext). +func TestLayerCache_NilCacheInContext_FetchesNormally(t *testing.T) { + g := &countingGetter{data: []byte("content")} + + f := file.NewFile("mock://uncached/path", file.WithClient(newCountingClient(g, ""))) + if _, err := f.Layers(); err != nil { + t.Fatalf("Layers(): %v", err) + } + if got := atomic.LoadInt32(&g.opens); got != 1 { + t.Errorf("expected 1 Open call (layer.FromOpener opens once and reuses the digest as diffID), got %d", got) + } +} + +// TestLayerCache_DedupesConcurrentOverlappingFetches proves the dedup also +// holds when two File instances for the same path race concurrently (not +// just sequentially, as TestLayerCache_DedupesFetchesAcrossFileInstances +// exercises) -- the shape a --concurrency > 1 sync run produces when a +// manifest lists the same source twice. Uses a blocking getter to force +// genuine overlap: both goroutines are guaranteed to be inside compute() at +// the same time before either is allowed to finish. +func TestLayerCache_DedupesConcurrentOverlappingFetches(t *testing.T) { + g := &blockingGetter{release: make(chan struct{}), data: []byte("raced content")} + + cache := file.NewLayerCache() + baseCtx := file.WithLayerCacheContext(context.Background(), cache) + + var opens int32 + countingOpen := func(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + atomic.AddInt32(&opens, 1) + return g.Open(ctx, u) + } + countingGetterFn := &fnGetter{open: countingOpen, name: "raced"} + + newClient := func(nameOverride string) *getter.Client { + return &getter.Client{ + Options: getter.ClientOptions{NameOverride: nameOverride}, + Getters: map[string]getter.Getter{"mock": countingGetterFn}, + } + } + + f1 := file.NewFile("mock://raced/path", file.WithClient(newClient("")), file.WithContext(baseCtx)) + f2 := file.NewFile("mock://raced/path", file.WithClient(newClient("second.sh")), file.WithContext(baseCtx)) + + var wg sync.WaitGroup + errs := make(chan error, 2) + wg.Add(2) + go func() { defer wg.Done(); _, err := f1.Layers(); errs <- err }() + go func() { defer wg.Done(); _, err := f2.Layers(); errs <- err }() + + // Give both goroutines a chance to actually enter Open (and block on + // g.release) before releasing them -- proving they were genuinely + // in-flight concurrently, not accidentally serialized by the test. + time.Sleep(100 * time.Millisecond) + close(g.release) + + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("Layers(): %v", err) + } + } + + if got := atomic.LoadInt32(&opens); got != 1 { + t.Errorf("expected exactly 1 Open call (one shared fetch, opened once by layer.FromOpener) despite two concurrently-racing File instances, got %d", got) + } +} + +// fnGetter adapts a plain Open func into a getter.Getter for tests that need +// to wrap another getter's Open behavior (e.g. to count calls) without +// re-implementing Detect/Name/Config. +type fnGetter struct { + open func(context.Context, *url.URL) (io.ReadCloser, error) + name string +} + +func (g *fnGetter) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + return g.open(ctx, u) +} +func (g *fnGetter) Detect(u *url.URL) bool { return true } +func (g *fnGetter) Name(u *url.URL) string { return g.name } +func (g *fnGetter) Config(u *url.URL) artifacts.Config { + return artifacts.ToConfig(struct { + Reference string `json:"reference"` + }{u.String()}, artifacts.WithConfigMediaType("application/vnd.test.config")) +} diff --git a/pkg/artifacts/file/context_test.go b/pkg/artifacts/file/context_test.go new file mode 100644 index 0000000..328eaec --- /dev/null +++ b/pkg/artifacts/file/context_test.go @@ -0,0 +1,92 @@ +package file_test + +import ( + "bytes" + "context" + "io" + "net/url" + "testing" + "time" + + "hauler.dev/go/hauler/v2/pkg/artifacts" + "hauler.dev/go/hauler/v2/pkg/artifacts/file" + "hauler.dev/go/hauler/v2/pkg/getter" +) + +// blockingGetter's Open blocks on a channel controlled by the test until +// released, unless ctx is cancelled first -- letting tests distinguish +// "the real per-call ctx reached Open" from "compute() built the layer with +// some other, uncancellable ctx" (the context.TODO() bug this file guards +// against). +type blockingGetter struct { + release chan struct{} + data []byte +} + +func (g *blockingGetter) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + select { + case <-g.release: + return io.NopCloser(bytes.NewReader(g.data)), nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (g *blockingGetter) Detect(u *url.URL) bool { return true } +func (g *blockingGetter) Name(u *url.URL) string { return "blocked" } +func (g *blockingGetter) Config(u *url.URL) artifacts.Config { + return artifacts.ToConfig(struct { + Reference string `json:"reference"` + }{u.String()}, artifacts.WithConfigMediaType("application/vnd.test.config")) +} + +func newBlockingClient(g *blockingGetter) *getter.Client { + return &getter.Client{ + Options: getter.ClientOptions{}, + Getters: map[string]getter.Getter{"mock": g}, + } +} + +// TestFile_WithContext_CancellationAbortsInFlightFetch proves that the ctx +// passed via file.WithContext is the ctx that actually reaches the getter's +// Open call -- not some internal context.TODO() that can never be +// cancelled. Without this wiring, cancelling ctx while compute() is +// blocked inside Open would have no effect and this test would time out. +func TestFile_WithContext_CancellationAbortsInFlightFetch(t *testing.T) { + g := &blockingGetter{release: make(chan struct{})} + t.Cleanup(func() { close(g.release) }) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(50 * time.Millisecond) + cancel() + }() + + f := file.NewFile("mock://source", file.WithClient(newBlockingClient(g)), file.WithContext(ctx)) + + done := make(chan error, 1) + go func() { + _, err := f.Layers() + done <- err + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("expected an error from Layers() after ctx cancellation, got nil") + } + case <-time.After(5 * time.Second): + t.Fatal("Layers() did not return within 5s of ctx cancellation... file.WithContext's ctx is not reaching the getter") + } +} + +// TestFile_WithContext_DefaultsToBackground proves NewFile without +// file.WithContext still works end-to-end (the zero-value/default path), +// matching pre-existing behavior for every caller that never sets it. +func TestFile_WithContext_DefaultsToBackground(t *testing.T) { + f := file.NewFile(filename, file.WithClient(mc)) + + if _, err := f.Layers(); err != nil { + t.Fatalf("Layers() with default context: %v", err) + } +} diff --git a/pkg/artifacts/file/file.go b/pkg/artifacts/file/file.go index fdeea68..f0ec798 100644 --- a/pkg/artifacts/file/file.go +++ b/pkg/artifacts/file/file.go @@ -27,6 +27,13 @@ type File struct { blob gv1.Layer manifest *gv1.Manifest annotations map[string]string + + // ctx is used by compute() when fetching content. artifacts.OCI + // (Layers/Manifest/RawConfig) takes no context parameter, so a + // struct-stored ctx is the only way to thread real cancellation through + // to the getter. Defaults to context.Background() when unset via + // WithContext. + ctx context.Context } func NewFile(path string, opts ...Option) *File { @@ -35,6 +42,7 @@ func NewFile(path string, opts ...Option) *File { f := &File{ client: client, Path: path, + ctx: context.Background(), } for _, opt := range opts { @@ -75,13 +83,42 @@ func (f *File) Manifest() (*gv1.Manifest, error) { return f.manifest, nil } +// Size returns the total compressed byte size across every layer this file +// produces (always exactly one today, see Layers), computing the content if +// needed. compute() is memoized, so an already-added File pays no extra +// fetch cost. +func (f *File) Size() (int64, error) { + layers, err := f.Layers() + if err != nil { + return 0, err + } + var total int64 + for _, l := range layers { + sz, err := l.Size() + if err != nil { + return 0, err + } + total += sz + } + return total, nil +} + func (f *File) compute() error { if f.computed { return nil } - ctx := context.TODO() - blob, err := f.client.LayerFrom(ctx, f.Path) + fetch := func() (gv1.Layer, error) { + return f.client.LayerFrom(f.ctx, f.Path) + } + + var blob gv1.Layer + var err error + if cache := layerCacheFromContext(f.ctx); cache != nil { + blob, err = cache.getOrFetch(f.Path, fetch) + } else { + blob, err = fetch() + } if err != nil { return err } @@ -91,12 +128,18 @@ func (f *File) compute() error { return err } - // Manually preserve the Title annotation from the layer - // The layer was created with this annotation in getter.LayerFrom - if layer.Annotations == nil { - layer.Annotations = make(map[string]string) + // Manually preserve the Title annotation from the layer (set by + // getter.LayerFrom). Copy rather than mutate in place: Descriptor() + // returns the layer's own annotations map by reference, and blob may be + // shared across File instances with different name overrides via a + // LayerCache (cache.go) -- mutating in place would let whichever + // compute() runs last overwrite every other sharer's Title. + annotations := make(map[string]string, len(layer.Annotations)+1) + for k, v := range layer.Annotations { + annotations[k] = v } - layer.Annotations[ocispec.AnnotationTitle] = f.client.Name(f.Path) + annotations[ocispec.AnnotationTitle] = f.client.Name(f.Path) + layer.Annotations = annotations cfg := f.client.Config(f.Path) if cfg == nil { diff --git a/pkg/artifacts/file/options.go b/pkg/artifacts/file/options.go index fea2812..fb33ca6 100644 --- a/pkg/artifacts/file/options.go +++ b/pkg/artifacts/file/options.go @@ -1,6 +1,8 @@ package file import ( + "context" + "hauler.dev/go/hauler/v2/pkg/artifacts" "hauler.dev/go/hauler/v2/pkg/getter" ) @@ -13,6 +15,15 @@ func WithClient(c *getter.Client) Option { } } +// WithContext sets the context used by compute() when fetching content, so +// cancelling ctx aborts an in-flight fetch. See File.ctx for why this is an +// option rather than a parameter on the artifacts.OCI interface. +func WithContext(ctx context.Context) Option { + return func(f *File) { + f.ctx = ctx + } +} + func WithConfig(obj interface{}, mediaType string) Option { return func(f *File) { f.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType)) diff --git a/pkg/audit/audit.go b/pkg/audit/audit.go index f33800a..09d96ab 100644 --- a/pkg/audit/audit.go +++ b/pkg/audit/audit.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "strings" + "sync" "time" "github.com/google/uuid" @@ -133,7 +134,7 @@ func BuildGlobal(ro *flags.CliRootOpts, rso *flags.StoreRootOpts) GlobalEntry { g := GlobalEntry{} if ro != nil { g.HaulerDir = resolveDir(ro.HaulerDir) - g.IgnoreErrors = ro.IgnoreErrors + g.IgnoreErrors = flags.ShouldIgnoreErrors(ro) g.LogLevel = ro.LogLevel g.AuditLevel = ro.AuditLevel } @@ -199,7 +200,15 @@ func Append(haulerDir string, e Entry) error { return globalErr } +// appendMu serializes appendLine calls: os.OpenFile with O_APPEND is only +// atomic for a single write() syscall on POSIX, and concurrent `store sync` +// image jobs (runImageJobs) can each call this at once without it. +var appendMu sync.Mutex + func appendLine(dir string, v any) error { + appendMu.Lock() + defer appendMu.Unlock() + if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("audit: ensure dir: %w", err) } diff --git a/pkg/audit/audit_test.go b/pkg/audit/audit_test.go index 8a44945..4f7d3ee 100644 --- a/pkg/audit/audit_test.go +++ b/pkg/audit/audit_test.go @@ -6,6 +6,9 @@ import ( "path/filepath" "strings" "testing" + + "hauler.dev/go/hauler/v2/internal/flags" + "hauler.dev/go/hauler/v2/pkg/consts" ) func TestAppend(t *testing.T) { @@ -212,3 +215,20 @@ func TestResolveDir_Default(t *testing.T) { t.Error("resolveDir(\"\") returned empty string") } } + +func TestBuildGlobal_IgnoreErrorsReflectsEnvVar(t *testing.T) { + // BuildGlobal must record the *effective* ignore-errors setting (flag OR + // env var) via flags.ShouldIgnoreErrors, not just the raw ro.IgnoreErrors + // field — callers no longer mutate ro.IgnoreErrors to reflect the env var. + ro := &flags.CliRootOpts{IgnoreErrors: false} + t.Setenv(consts.HaulerIgnoreErrors, "true") + + g := BuildGlobal(ro, nil) + + if !g.IgnoreErrors { + t.Fatal("expected BuildGlobal to report IgnoreErrors=true when HAULER_IGNORE_ERRORS is set, even though ro.IgnoreErrors is false") + } + if ro.IgnoreErrors { + t.Fatal("expected BuildGlobal to not mutate ro.IgnoreErrors") + } +} diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 7b9c041..b5f20e6 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -87,12 +87,14 @@ const ( CollectionGroup = "collection.hauler.cattle.io" // environment variables - HaulerDir = "HAULER_DIR" - HaulerTempDir = "HAULER_TEMP_DIR" - HaulerStoreDir = "HAULER_STORE_DIR" - HaulerIgnoreErrors = "HAULER_IGNORE_ERRORS" - HaulerLogLevel = "HAULER_LOG_LEVEL" - HaulerAuditLevel = "HAULER_AUDIT_LEVEL" + HaulerDir = "HAULER_DIR" + HaulerTempDir = "HAULER_TEMP_DIR" + HaulerStoreDir = "HAULER_STORE_DIR" + HaulerIgnoreErrors = "HAULER_IGNORE_ERRORS" + HaulerConcurrency = "HAULER_CONCURRENCY" + HaulerBlobConcurrency = "HAULER_BLOB_CONCURRENCY" + HaulerLogLevel = "HAULER_LOG_LEVEL" + HaulerAuditLevel = "HAULER_AUDIT_LEVEL" // container files and directories ImageManifestFile = "manifest.json" @@ -116,7 +118,18 @@ const ( DefaultStoreInventoryName = "stores.json" DefaultRetries = 3 RetriesInterval = 5 - CustomTimeFormat = "2006-01-02 15:04:05" + // DefaultConcurrency bounds the number of images that `store sync` pulls + // and stores concurrently. See flags.ResolveConcurrency and + // flags.BlobConcurrencyFor for how this feeds into the per-store blob + // write ceiling (DefaultBlobConcurrency below). + DefaultConcurrency = 5 + // DefaultBlobConcurrency bounds the number of blob writes (layer + // downloads, config/manifest writes) that may be in flight at once across + // the whole process, regardless of how many images or errgroups are + // fanning out concurrently. See content.OCI.WriteBlob and + // content.ociPusher.Push, the two call sites that acquire against it. + DefaultBlobConcurrency = 16 + CustomTimeFormat = "2006-01-02 15:04:05" ) var FileExcludePattern = fmt.Sprintf(`^%s/[.\-_]`, DefaultNamespace) diff --git a/pkg/content/chart/chart.go b/pkg/content/chart/chart.go index c2d9124..0f5d63c 100644 --- a/pkg/content/chart/chart.go +++ b/pkg/content/chart/chart.go @@ -81,25 +81,7 @@ func NewChart(name string, opts *action.ChartPathOptions) (*Chart, error) { chartRef = opts.RepoURL + "/" + name } - // suppress helm downloader oci logs (stdout/stderr) - oldStdout := os.Stdout - oldStderr := os.Stderr - rOut, wOut, _ := os.Pipe() - rErr, wErr, _ := os.Pipe() - os.Stdout = wOut - os.Stderr = wErr - chartPath, err := client.ChartPathOptions.LocateChart(chartRef, settings) - - wOut.Close() - wErr.Close() - os.Stdout = oldStdout - os.Stderr = oldStderr - _, _ = io.Copy(io.Discard, rOut) - _, _ = io.Copy(io.Discard, rErr) - rOut.Close() - rErr.Close() - if err != nil { return nil, err } diff --git a/pkg/content/oci.go b/pkg/content/oci.go index bf7ecc7..4af5ade 100644 --- a/pkg/content/oci.go +++ b/pkg/content/oci.go @@ -1,21 +1,30 @@ package content import ( + "bytes" "context" "encoding/json" + "errors" "fmt" "io" "maps" "os" "path/filepath" + "runtime" + "slices" "sort" "strings" "sync" + "syscall" + "time" "github.com/google/go-containerregistry/pkg/name" + "golang.org/x/sync/semaphore" + "golang.org/x/sync/singleflight" ccontent "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/remotes" + "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -25,24 +34,129 @@ import ( var _ Target = (*OCI)(nil) +// indexCheckpointInterval bounds fsync frequency on the per-artifact save +// path: index.json is rewritten in full on every AddIndex, so fsyncing each +// one costs O(N^2) bytes across a sync (~5 GB for 5,000 artifacts) while +// holding o.mu. Coalescing trades a bounded power-loss/panic exposure window +// for that cost -- an interrupted sync is simply re-run. +const indexCheckpointInterval = 30 * time.Second + +// ErrDigestMismatch is returned by WriteBlob (and wrapped with details) when +// the content actually streamed from open() does not hash to the expected +// digest. Callers can retry: the final blob path is never touched on this +// error, so a fresh WriteBlob call will re-download cleanly. +var ErrDigestMismatch = errors.New("content: digest mismatch") + +// ctxReader wraps an io.Reader so Read returns ctx.Err() once ctx is done, +// instead of delegating. This makes an in-flight WriteBlob copy abort on +// cancellation, since the underlying reader (v1.Layer's Compressed(), which +// this package doesn't control) has no cancellation hook of its own. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + type OCI struct { root string index *ocispec.Index nameMap *sync.Map // map[string]ocispec.Descriptor + sf singleflight.Group + + // blobSem bounds blob writes in flight across this OCI's two write + // paths (WriteBlob, ociPusher.Push), scoped per-store. Acquire is + // ctx-aware, unlike errgroup.SetLimit, so it returns promptly on + // cancellation instead of leaving a goroutine parked. + blobSem *semaphore.Weighted + + // blobConcurrency is the ceiling blobSem was built with, retained so + // reporting can render "peak-inflight=N/ceiling" (semaphore.Weighted + // doesn't expose its own capacity). + blobConcurrency int + + // stats accumulates disk-contention counters for this store. See + // stats.go. + stats IOStats + + // mu guards index, index.json on disk, and nameMap descriptors' + // annotation maps. Exported methods are thin lock-then-Locked-variant + // wrappers; the *Locked methods assume the caller holds mu, letting + // internal chains (e.g. ociPusher.Push's load-modify-save) re-enter + // without double-locking (see Walk's doc comment for the related hazard). + mu sync.Mutex + + // lastDurableSave is when the index was last fsync'd, guarded by mu. + // The zero value makes the first save of a run durable, which gives an + // early checkpoint for free. + lastDurableSave time.Time + + // now is time.Now, replaced in tests so checkpoint-interval behavior + // can be exercised without sleeping. + now func() time.Time } -func NewOCI(root string) (*OCI, error) { +// lock acquires o.mu, recording time spent blocked into IOStats, so every +// caller measures index-serialization cost in one place. Unlock isn't +// wrapped since it never blocks. +func (o *OCI) lock() { + start := time.Now() + o.mu.Lock() + o.stats.IndexLockWaitNanos.Add(int64(time.Since(start))) +} + +// OCIOption configures an OCI store at construction time. +type OCIOption func(*OCI) + +// WithBlobConcurrency overrides consts.DefaultBlobConcurrency for this OCI's +// blobSem. n <= 0 is a no-op (keeps the default) rather than an error, so +// callers can pass a possibly-zero value unconditionally. +func WithBlobConcurrency(n int) OCIOption { + return func(o *OCI) { + if n > 0 { + o.blobSem = semaphore.NewWeighted(int64(n)) + o.blobConcurrency = n + } + } +} + +func NewOCI(root string, opts ...OCIOption) (*OCI, error) { o := &OCI{ - root: root, - nameMap: &sync.Map{}, + root: root, + nameMap: &sync.Map{}, + blobSem: semaphore.NewWeighted(consts.DefaultBlobConcurrency), + blobConcurrency: consts.DefaultBlobConcurrency, + now: time.Now, + } + for _, opt := range opts { + opt(o) } return o, nil } +// Stats returns this store's I/O contention counters. The returned pointer +// is live -- counters keep incrementing as work proceeds. Call Snapshot on +// it to take a stable reading. +func (o *OCI) Stats() *IOStats { + return &o.stats +} + +// BlobConcurrency returns the ceiling blobSem was constructed with. +func (o *OCI) BlobConcurrency() int { + return o.blobConcurrency +} + // AddIndex adds a descriptor to the index and updates it // // The descriptor must use AnnotationRefName to identify itself func (o *OCI) AddIndex(desc ocispec.Descriptor) error { + // Pure validation/parsing -- doesn't touch shared state -- stays outside + // the lock. if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok { return fmt.Errorf("descriptor must contain a reference from the annotation: %s", ocispec.AnnotationRefName) } @@ -52,19 +166,77 @@ func (o *OCI) AddIndex(desc ocispec.Descriptor) error { return err } - if strings.TrimSpace(key.String()) != "--" { - switch key.(type) { - case name.Digest: - o.nameMap.Store(fmt.Sprintf("%s-%s", key.Context().String(), desc.Annotations[consts.KindAnnotationName]), desc) - case name.Tag: - o.nameMap.Store(fmt.Sprintf("%s-%s", key.String(), desc.Annotations[consts.KindAnnotationName]), desc) + if strings.TrimSpace(key.String()) == "--" { + return nil + } + + var mapKey string + switch key.(type) { + case name.Digest: + mapKey = fmt.Sprintf("%s-%s", key.Context().String(), desc.Annotations[consts.KindAnnotationName]) + case name.Tag: + mapKey = fmt.Sprintf("%s-%s", key.String(), desc.Annotations[consts.KindAnnotationName]) + default: + return nil + } + + o.lock() + defer o.mu.Unlock() + + // Skip the write when the stored descriptor is already byte-identical: + // index.json rewrites aren't otherwise batched (O(N^2) bytes as the + // index grows), only their fsync is (see indexCheckpointInterval). + if existing, ok := o.nameMap.Load(mapKey); ok { + if descriptorsEqual(existing.(ocispec.Descriptor), desc) { + return nil } } - return o.SaveIndex() + + o.nameMap.Store(mapKey, desc) + return o.saveIndexCheckpointLocked() } -// LoadIndex will load the index from disk +// descriptorsEqual reports whether two descriptors are equal in every field +// AddIndex's callers in this codebase populate: MediaType, Digest, Size, +// URLs, ArtifactType, Platform, Data, and Annotations (compared by +// contents, not map identity). +func descriptorsEqual(a, b ocispec.Descriptor) bool { + if a.MediaType != b.MediaType || a.Digest != b.Digest || a.Size != b.Size || a.ArtifactType != b.ArtifactType { + return false + } + if !maps.Equal(a.Annotations, b.Annotations) { + return false + } + if !slices.Equal(a.URLs, b.URLs) { + return false + } + if !bytes.Equal(a.Data, b.Data) { + return false + } + if (a.Platform == nil) != (b.Platform == nil) { + return false + } + if a.Platform != nil { + pa, pb := a.Platform, b.Platform + if pa.Architecture != pb.Architecture || pa.OS != pb.OS || pa.OSVersion != pb.OSVersion || pa.Variant != pb.Variant { + return false + } + if !slices.Equal(pa.OSFeatures, pb.OSFeatures) { + return false + } + } + return true +} + +// LoadIndex will load the index from disk. func (o *OCI) LoadIndex() error { + o.lock() + defer o.mu.Unlock() + return o.loadIndexLocked() +} + +// loadIndexLocked is LoadIndex's implementation. Callers must hold o.mu. +func (o *OCI) loadIndexLocked() error { path := o.path(ocispec.ImageIndexFile) idx, err := os.Open(path) if err != nil { @@ -99,9 +271,8 @@ func (o *OCI) LoadIndex() error { kind = consts.KindAnnotationImage } - // Write the normalized kind back into a copy of the annotations map so - // that Walk() callers receive descriptors with dev.hauler/... values. - // We copy the map to avoid mutating the slice element's shared map. + // Write normalized kind into a copy of Annotations so Walk() callers + // see it, without mutating the slice element's shared map. normalized := make(map[string]string, len(desc.Annotations)+1) maps.Copy(normalized, desc.Annotations) normalized[consts.KindAnnotationName] = kind @@ -120,8 +291,24 @@ func (o *OCI) LoadIndex() error { return nil } -// SaveIndex will update the index on disk +// SaveIndex will update the index on disk. func (o *OCI) SaveIndex() error { + o.lock() + defer o.mu.Unlock() + return o.saveIndexLocked(true) +} + +// saveIndexLocked is SaveIndex's implementation. Callers must hold o.mu. +// +// The write is atomic: temp file (uniquely named via os.CreateTemp, since +// two hauler processes share no in-process mutex) in the same directory, +// then renamed into place. +// +// durable controls both the temp file's fsync and, after rename, an fsync of +// the containing directory (see syncDir); when false the write is still +// atomic but may not survive power loss until a later save catches up -- +// see indexCheckpointInterval. +func (o *OCI) saveIndexLocked(durable bool) error { var descs []ocispec.Descriptor o.nameMap.Range(func(name, desc interface{}) bool { n := desc.(ocispec.Descriptor).Annotations[ocispec.AnnotationRefName] @@ -154,7 +341,90 @@ func (o *OCI) SaveIndex() error { if err != nil { return err } - return os.WriteFile(o.path(ocispec.ImageIndexFile), data, 0644) + + indexPath := o.path(ocispec.ImageIndexFile) + dir := filepath.Dir(indexPath) + + tmp, err := os.CreateTemp(dir, "index-*.json") + if err != nil { + return err + } + tmpPath := tmp.Name() + // Unconditional cleanup: harmless ENOENT after a successful rename, and + // it's the cleanup path for every error branch below -- same idiom as + // writeBlobOnce. + defer os.Remove(tmpPath) + + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + // chmod before rename: CreateTemp files are 0600, but index.json must be + // readable by other consumers (e.g. `hauler store serve` as another user). + if err := tmp.Chmod(0644); err != nil { + tmp.Close() + return err + } + if durable { + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpPath, indexPath); err != nil { + return err + } + // Track the rename unconditionally -- it succeeded regardless of whether + // the durable branch below (which can still fail on syncDir) completes. + // IndexDurableWrites stays inside that branch since it must count only + // fsyncs that actually completed. + o.stats.IndexWrites.Add(1) + o.stats.IndexBytesWritten.Add(int64(len(data))) + if durable { + if err := syncDir(dir); err != nil { + return err + } + o.lastDurableSave = o.now() + o.stats.IndexDurableWrites.Add(1) + } + return nil +} + +// saveIndexCheckpointLocked saves the index, fsync'ing only when at least +// indexCheckpointInterval has elapsed since the last durable save. Used by +// the per-artifact callers (AddIndex, ociPusher.Push); callers that are +// explicit checkpoints call saveIndexLocked(true) directly. Callers must +// hold o.mu. +func (o *OCI) saveIndexCheckpointLocked() error { + return o.saveIndexLocked(o.now().Sub(o.lastDurableSave) >= indexCheckpointInterval) +} + +// syncDir fsyncs a directory so a rename into it survives power loss; +// fsyncing the file alone doesn't guarantee the directory entry pointing at +// it (containerd's local content store does the same after a blob rename). +// A no-op on Windows, which has no equivalent. EINVAL/ENOTSUP are tolerated +// (some NFS/SMB mounts, common for an air-gapped store root, don't support +// directory fsync) rather than turning a previously working command into a +// hard failure; every other error is still fatal. +func syncDir(dir string) error { + if runtime.GOOS == "windows" { + return nil + } + d, err := os.Open(dir) + if err != nil { + return err + } + if err := d.Sync(); err != nil { + d.Close() + if errors.Is(err, syscall.EINVAL) || errors.Is(err, syscall.ENOTSUP) { + return nil + } + return err + } + return d.Close() } // Resolve attempts to resolve the reference into a name and descriptor. @@ -169,7 +439,10 @@ func (o *OCI) SaveIndex() error { // // If the resolution fails, an error will be returned. func (o *OCI) Resolve(ctx context.Context, ref string) (ocispec.Descriptor, error) { - if err := o.LoadIndex(); err != nil { + o.lock() + defer o.mu.Unlock() + + if err := o.loadIndexLocked(); err != nil { return ocispec.Descriptor{}, err } d, ok := o.nameMap.Load(ref) @@ -184,7 +457,10 @@ func (o *OCI) Resolve(ctx context.Context, ref string) (ocispec.Descriptor, erro // All content fetched from the returned fetcher will be // from the namespace referred to by ref. func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { - if err := o.LoadIndex(); err != nil { + o.lock() + defer o.mu.Unlock() + + if err := o.loadIndexLocked(); err != nil { return nil, err } if _, ok := o.nameMap.Load(ref); !ok { @@ -193,6 +469,10 @@ func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) return o, nil } +// Fetch is intentionally lock-free: it only touches the filesystem and the +// immutable root field, never index/nameMap. A lock here would deadlock +// store.Layout.CleanUp, which calls Fetch from inside an OCI.Walk callback +// (see Walk's doc comment for the general hazard). func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { readerAt, err := o.blobReaderAt(desc) if err != nil { @@ -201,6 +481,7 @@ func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser return readerAt, nil } +// FetchManifest is intentionally lock-free -- see Fetch's doc comment. func (o *OCI) FetchManifest(ctx context.Context, manifest ocispec.Manifest) (io.ReadCloser, error) { readerAt, err := o.manifestBlobReaderAt(manifest) if err != nil { @@ -213,7 +494,10 @@ func (o *OCI) FetchManifest(ctx context.Context, manifest ocispec.Manifest) (io. // The returned Pusher should satisfy content.Ingester and concurrent attempts // to push the same blob using the Ingester API should result in ErrUnavailable. func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { - if err := o.LoadIndex(); err != nil { + o.lock() + defer o.mu.Unlock() + + if err := o.loadIndexLocked(); err != nil { return nil, err } @@ -232,24 +516,52 @@ func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { }, nil } +// Walk loads the index, snapshots nameMap under o.mu -- deep-copying each +// descriptor's annotations -- then releases the lock before invoking fn per +// entry. This is load-bearing: store.Layout.CopyAll's self-sync path +// re-enters Copy/Resolve/Fetcher/Pusher on this same OCI from inside a Walk +// callback (CleanUp does the same via Fetch), and a plain mutex held across +// fn would deadlock on that re-entrant Lock() from the same goroutine. +// +// The deep copy means in-place mutation of a callback's descriptor silently +// no-ops instead of corrupting nameMap's shared map; use UpdateAnnotations +// to persist changes. func (o *OCI) Walk(fn func(reference string, desc ocispec.Descriptor) error) error { - if err := o.LoadIndex(); err != nil { + o.lock() + if err := o.loadIndexLocked(); err != nil { + o.mu.Unlock() return err } - var errst []string + type entry struct { + key string + desc ocispec.Descriptor + } + var snapshot []entry o.nameMap.Range(func(key, value interface{}) bool { - if err := fn(key.(string), value.(ocispec.Descriptor)); err != nil { - errst = append(errst, err.Error()) - } + d := value.(ocispec.Descriptor) + cp := make(map[string]string, len(d.Annotations)) + maps.Copy(cp, d.Annotations) + d.Annotations = cp + snapshot = append(snapshot, entry{key: key.(string), desc: d}) return true }) + o.mu.Unlock() + + var errst []string + for _, e := range snapshot { + if err := fn(e.key, e.desc); err != nil { + errst = append(errst, err.Error()) + } + } if errst != nil { return fmt.Errorf("%s", strings.Join(errst, "; ")) } return nil } +// blobReaderAt, manifestBlobReaderAt, blobWriterAt, and ensureBlob are +// lock-free too -- see Fetch's doc comment. func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) { blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex()) if err != nil { @@ -282,6 +594,166 @@ func (o *OCI) ensureBlob(alg string, hex string) (string, error) { return filepath.Join(dir, hex), nil } +// WriteBlob atomically and verifiably writes a blob to the store's blob +// directory, deduplicating concurrent writers of the same digest. It is +// lock-free with respect to o.mu -- see Fetch's doc comment. open is a +// thunk, not a reader, since singleflight must not start the download until +// it wins the flight, and a retry needs a fresh reader. +// +// A file already at the final path with a matching size short-circuits +// WriteBlob without re-hashing, so re-syncs don't become O(store size) in +// disk reads. Otherwise content streams, deduplicated via singleflight, into +// a temp file hashed inline, then chmod'd, fsync'd, and renamed into place. +// On any error the temp file is removed and the final path untouched, so a +// failing writer can't corrupt a peer's completed blob and a retry +// re-downloads cleanly. +// +// singleflight.Do hands the flight winner's error to every waiter, even ones +// on a distinct, still-live ctx -- if that error is context.Canceled but +// this caller's ctx isn't done, WriteBlob retries once on the caller's own +// ctx rather than propagate a cancellation that wasn't its own. +func (o *OCI) WriteBlob(ctx context.Context, expected digest.Digest, size int64, open func() (io.ReadCloser, error)) error { + if err := ctx.Err(); err != nil { + return err + } + + dir := o.path(ocispec.ImageBlobsDir, expected.Algorithm().String()) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + blobPath := filepath.Join(dir, expected.Hex()) + + // Fast path: trust existing content by size alone, checked and returned + // before touching blobSem so a cache hit stays free regardless of + // semaphore saturation. + if info, err := os.Stat(blobPath); err == nil { + if size > 0 { + if info.Size() == size { + o.stats.BlobsCached.Add(1) + return nil + } + } else if info.Size() > 0 { + o.stats.BlobsCached.Add(1) + return nil + } + } + + err := o.writeBlobShared(ctx, dir, blobPath, expected, size, open) + if err != nil && errors.Is(err, context.Canceled) && ctx.Err() == nil { + // See WriteBlob's doc comment: retrying either hits the fast path or + // this goroutine becomes the new flight leader. + err = o.writeBlobShared(ctx, dir, blobPath, expected, size, open) + } + return err +} + +// writeBlobShared dedupes concurrent writers of expected via this OCI's +// singleflight.Group so only one actually streams content. +func (o *OCI) writeBlobShared(ctx context.Context, dir, blobPath string, expected digest.Digest, size int64, open func() (io.ReadCloser, error)) error { + _, err, _ := o.sf.Do(expected.String(), func() (interface{}, error) { + // Acquired inside the singleflight func, not around sf.Do: losers + // merely waiting on Do() must not hold a permit for someone else's + // write. + start := time.Now() + if err := o.blobSem.Acquire(ctx, 1); err != nil { + return nil, err + } + o.stats.addSemWait(time.Since(start)) + o.stats.enterBlob() + defer func() { + o.stats.exitBlob() + o.blobSem.Release(1) + }() + return nil, o.writeBlobOnce(ctx, dir, blobPath, expected, size, open) + }) + return err +} + +// writeBlobOnce performs the temp-file-then-rename write. Only ever invoked +// by the singleflight winner, which already holds a blobSem permit. +func (o *OCI) writeBlobOnce(ctx context.Context, dir, blobPath string, expected digest.Digest, size int64, open func() (io.ReadCloser, error)) (err error) { + // Re-check under the flight: a prior, already-completed flight may have + // written this blob while we were waiting to start. + if info, statErr := os.Stat(blobPath); statErr == nil { + if size > 0 && info.Size() == size { + o.stats.BlobsCached.Add(1) + return nil + } + if size <= 0 && info.Size() > 0 { + o.stats.BlobsCached.Add(1) + return nil + } + } + + if err := ctx.Err(); err != nil { + return err + } + + tmp, err := os.CreateTemp(dir, expected.Hex()+".tmp-*") + if err != nil { + return err + } + tmpPath := tmp.Name() + // Unconditional cleanup: harmless ENOENT after a successful rename, and + // it's the cleanup path for every error branch below. + defer os.Remove(tmpPath) + + rc, err := open() + if err != nil { + tmp.Close() + return err + } + + // See ctxReader's doc comment: this makes io.Copy below abort between + // chunks instead of running an already-cancelled download to completion. + cr := &ctxReader{ctx: ctx, r: rc} + + dg := digest.Canonical.Digester() + n, copyErr := io.Copy(io.MultiWriter(tmp, dg.Hash()), cr) + closeReadErr := rc.Close() + + if copyErr != nil { + tmp.Close() + return copyErr + } + if closeReadErr != nil { + tmp.Close() + return closeReadErr + } + if size > 0 && n != size { + tmp.Close() + return fmt.Errorf("content: short/long write for %s: wrote %d bytes, expected %d: %w", expected, n, size, ErrDigestMismatch) + } + + got := dg.Digest() + if got != expected { + tmp.Close() + return fmt.Errorf("content: digest mismatch for blob: expected %s, got %s (%d bytes): %w", expected, got, n, ErrDigestMismatch) + } + + // Commit: chmod 0600->0644 (see saveIndexLocked), fsync, then rename. + if err := tmp.Chmod(0644); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + // os.Rename silently replaces an existing target, which is correct: the + // content is digest-identical to whatever's already at blobPath. + if err := os.Rename(tmpPath, blobPath); err != nil { + return err + } + o.stats.BlobsWritten.Add(1) + o.stats.BlobBytesWritten.Add(n) + return nil +} + +// path and IndexExists are lock-free too -- see Fetch's doc comment. func (o *OCI) path(elem ...string) string { complete := []string{string(o.root)} return filepath.Join(append(complete, elem...)...) @@ -306,7 +778,11 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex, consts.DockerManifestSchema2, consts.DockerManifestListSchema2: // if the hash of the content matches that which was provided as the hash for the root, mark it if p.digest != "" && p.digest == d.Digest.String() { - if err := p.oci.LoadIndex(); err != nil { + // Single critical section (Locked variants, to avoid deadlocking + // on this same lock) so no other save can land in between. + p.oci.lock() + if err := p.oci.loadIndexLocked(); err != nil { + p.oci.mu.Unlock() return nil, err } // Use compound key format: "reference-kind"; normalize legacy values. @@ -323,7 +799,9 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr d.Annotations = normalizedAnnotations key := fmt.Sprintf("%s-%s", p.ref, kind) p.oci.nameMap.Store(key, d) - if err := p.oci.SaveIndex(); err != nil { + err := p.oci.saveIndexCheckpointLocked() + p.oci.mu.Unlock() + if err != nil { return nil, err } } @@ -335,23 +813,166 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr } if _, err := os.Stat(blobPath); err == nil { - // file already exists, discard (but validate digest) + // Already exists: discard but validate digest. Returned before + // touching blobSem -- same reasoning as WriteBlob's fast path. return NewIoContentWriter(nopCloser{io.Discard}, WithOutputHash(d.Digest.String())), nil } - f, err := os.Create(blobPath) - if err != nil { + // Shares WriteBlob's bound, but held for the writer's whole lifetime + // (Push through Close) since the caller streams via separate Write + // calls rather than one owned loop. + start := time.Now() + if err := p.oci.blobSem.Acquire(ctx, 1); err != nil { return nil, err } + p.oci.stats.addSemWait(time.Since(start)) + p.oci.stats.enterBlob() - w := NewIoContentWriter(f, WithOutputHash(d.Digest.String())) + w, err := newOCIBlobWriter(filepath.Dir(blobPath), blobPath, d.Digest.String()) + if err != nil { + p.oci.stats.exitBlob() + p.oci.blobSem.Release(1) + return nil, err + } + w.releaseSem = func() { + p.oci.stats.exitBlob() + p.oci.blobSem.Release(1) + } return w, nil } +// ociBlobWriter streams pushed content into a temp file and, on successful +// digest verification at Close, renames it into place -- same on-error +// invariant as content.OCI.WriteBlob (see its doc comment). +type ociBlobWriter struct { + tmp *os.File + tmpPath string + finalPath string + digester digest.Digester + status ccontent.Status + outputHash string + + // releaseSem releases the blobSem permit acquired by ociPusher.Push. nil + // when constructed outside Push (e.g. in tests). + releaseSem func() +} + +var _ ccontent.Writer = (*ociBlobWriter)(nil) + +func newOCIBlobWriter(dir, finalPath, outputHash string) (*ociBlobWriter, error) { + tmp, err := os.CreateTemp(dir, filepath.Base(finalPath)+".tmp-*") + if err != nil { + return nil, err + } + return &ociBlobWriter{ + tmp: tmp, + tmpPath: tmp.Name(), + finalPath: finalPath, + digester: digest.Canonical.Digester(), + outputHash: outputHash, + }, nil +} + +func (w *ociBlobWriter) Write(p []byte) (int, error) { + n, err := w.tmp.Write(p) + if n > 0 { + w.digester.Hash().Write(p[:n]) + } + return n, err +} + +// Close verifies the digest and, only on success, chmods, syncs, closes, and +// renames the temp file into place; on failure the temp file is left for +// the deferred os.Remove and the final path is untouched. +func (w *ociBlobWriter) Close() (err error) { + if w.releaseSem != nil { + defer w.releaseSem() // released on every path below, success or failure + } + defer os.Remove(w.tmpPath) // unconditional: harmless ENOENT after a successful rename + + if w.outputHash != "" { + if computed := w.digester.Digest().String(); computed != w.outputHash { + w.tmp.Close() + return fmt.Errorf("digest mismatch: expected %s, got %s", w.outputHash, computed) + } + } + + if err := w.tmp.Chmod(0644); err != nil { + w.tmp.Close() + return err + } + if err := w.tmp.Sync(); err != nil { + w.tmp.Close() + return err + } + if err := w.tmp.Close(); err != nil { + return err + } + + // os.Rename silently replaces an existing target, which is correct here: + // the content is digest-verified to match what's expected at finalPath. + return os.Rename(w.tmpPath, w.finalPath) +} + +func (w *ociBlobWriter) Digest() digest.Digest { + return w.digester.Digest() +} + +func (w *ociBlobWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...ccontent.Opt) error { + return nil +} + +func (w *ociBlobWriter) Status() (ccontent.Status, error) { + return w.status, nil +} + +func (w *ociBlobWriter) Truncate(size int64) error { + return fmt.Errorf("truncate not supported") +} + +// RemoveFromIndex removes ref from nameMap only; callers (e.g. +// store.Layout.RemoveArtifact) call SaveIndex separately afterward. func (o *OCI) RemoveFromIndex(ref string) { + o.lock() + defer o.mu.Unlock() o.nameMap.Delete(ref) } +// UpdateAnnotations locates every descriptor for which match returns true, +// replaces its annotations with a copy that has had apply run over it, and +// persists the index, returning the number matched. The whole pass runs as +// a single critical section so a concurrent UpdateAnnotations or Push can't +// interleave a save in between; when nothing matches, index.json is not +// re-saved. +func (o *OCI) UpdateAnnotations(match func(ocispec.Descriptor) bool, apply func(map[string]string)) (int, error) { + o.lock() + defer o.mu.Unlock() + + if err := o.loadIndexLocked(); err != nil { + return 0, err + } + + matched := 0 + o.nameMap.Range(func(key, value interface{}) bool { + d := value.(ocispec.Descriptor) + if !match(d) { + return true + } + cp := make(map[string]string, len(d.Annotations)) + maps.Copy(cp, d.Annotations) + apply(cp) + d.Annotations = cp + o.nameMap.Store(key, d) + matched++ + return true + }) + + if matched == 0 { + return 0, nil + } + return matched, o.saveIndexLocked(true) +} + // ResolvePath returns the absolute path for a given relative path within the OCI root func (o *OCI) ResolvePath(elem string) string { if elem == "" { diff --git a/pkg/content/oci_test.go b/pkg/content/oci_test.go index 404b978..5f8cd84 100644 --- a/pkg/content/oci_test.go +++ b/pkg/content/oci_test.go @@ -6,12 +6,19 @@ package content // legacy dev.cosignproject.cosign/... value that may be present on disk. import ( + "bytes" "context" "encoding/json" + "errors" + "fmt" + "io" "os" "path/filepath" "strings" + "sync" + "sync/atomic" "testing" + "time" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" @@ -279,3 +286,1419 @@ func TestPush_NormalizesLegacyKindInStoredDescriptor(t *testing.T) { desc.Annotations[consts.KindAnnotationName], legacyKind) } } + +// blobPathFor mirrors the layout convention used by OCI.ensureBlob, for +// assertions against the final on-disk blob path. +func blobPathFor(root string, d digest.Digest) string { + return filepath.Join(root, ocispec.ImageBlobsDir, d.Algorithm().String(), d.Hex()) +} + +// countTmpFiles returns the number of *.tmp-* files left behind in the +// sha256 blobs directory under root. +func countTmpFiles(t *testing.T, root string) int { + t.Helper() + dir := filepath.Join(root, ocispec.ImageBlobsDir, "sha256") + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("ReadDir %s: %v", dir, err) + } + count := 0 + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp-") { + count++ + } + } + return count +} + +func TestWriteBlob_WritesAndVerifiesNewBlob(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + data := []byte("hello world, this is blob content") + d := digest.FromBytes(data) + + err = o.WriteBlob(context.Background(), d, int64(len(data)), func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + }) + if err != nil { + t.Fatalf("WriteBlob: unexpected error: %v", err) + } + + blobPath := blobPathFor(dir, d) + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading written blob: %v", err) + } + if !bytes.Equal(got, data) { + t.Errorf("blob content = %q, want %q", got, data) + } + + if info, err := os.Stat(blobPath); err != nil { + t.Fatalf("stat blob: %v", err) + } else if info.Mode().Perm() != 0644 { + t.Errorf("blob mode = %o, want 0644", info.Mode().Perm()) + } + + if n := countTmpFiles(t, dir); n != 0 { + t.Errorf("left %d temp files behind, want 0", n) + } +} + +func TestWriteBlob_FastPath_SkipsOpenWhenExistingSizeMatches(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + data := []byte("existing correct content") + d := digest.FromBytes(data) + blobPath := blobPathFor(dir, d) + if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(blobPath, data, 0644); err != nil { + t.Fatalf("pre-writing blob: %v", err) + } + + var openCalled int32 + err = o.WriteBlob(context.Background(), d, int64(len(data)), func() (io.ReadCloser, error) { + atomic.AddInt32(&openCalled, 1) + return nil, errors.New("open should not have been called") + }) + if err != nil { + t.Fatalf("WriteBlob: unexpected error: %v", err) + } + if openCalled != 0 { + t.Errorf("open() was called %d times, want 0 (fast path should have skipped it)", openCalled) + } +} + +func TestWriteBlob_FastPath_SizeMismatchTriggersRewrite(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + correct := []byte("this is the correct, full-length blob content") + d := digest.FromBytes(correct) + blobPath := blobPathFor(dir, d) + if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + // Simulate a truncated/corrupt blob left behind by a crash: short content + // at the correct path. + if err := os.WriteFile(blobPath, []byte("short"), 0644); err != nil { + t.Fatalf("pre-writing truncated blob: %v", err) + } + + var openCalled int32 + err = o.WriteBlob(context.Background(), d, int64(len(correct)), func() (io.ReadCloser, error) { + atomic.AddInt32(&openCalled, 1) + return io.NopCloser(bytes.NewReader(correct)), nil + }) + if err != nil { + t.Fatalf("WriteBlob: unexpected error: %v", err) + } + if openCalled != 1 { + t.Errorf("open() was called %d times, want 1 (size mismatch should trigger rewrite)", openCalled) + } + + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading blob: %v", err) + } + if !bytes.Equal(got, correct) { + t.Errorf("blob content = %q, want %q", got, correct) + } +} + +func TestWriteBlob_DigestMismatch(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + actual := []byte("actual content that will be streamed") + wrongDigest := digest.FromBytes([]byte("this is not the actual content")) + + err = o.WriteBlob(context.Background(), wrongDigest, int64(len(actual)), func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(actual)), nil + }) + if err == nil { + t.Fatal("WriteBlob: expected digest mismatch error, got nil") + } + if !errors.Is(err, ErrDigestMismatch) { + t.Errorf("WriteBlob error %v does not wrap ErrDigestMismatch", err) + } + + blobPath := blobPathFor(dir, wrongDigest) + if _, statErr := os.Stat(blobPath); !os.IsNotExist(statErr) { + t.Errorf("final blob path exists after digest mismatch: %v", statErr) + } + + if n := countTmpFiles(t, dir); n != 0 { + t.Errorf("left %d temp files behind after digest mismatch, want 0", n) + } +} + +func TestWriteBlob_ConcurrentSameDigest_SingleflightDeduplicates(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + data := []byte("concurrently written content that many goroutines race to write") + d := digest.FromBytes(data) + + var opens int32 + const n = 16 + var wg sync.WaitGroup + errs := make([]error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + errs[i] = o.WriteBlob(context.Background(), d, int64(len(data)), func() (io.ReadCloser, error) { + atomic.AddInt32(&opens, 1) + return io.NopCloser(bytes.NewReader(data)), nil + }) + }(i) + } + wg.Wait() + + for i, e := range errs { + if e != nil { + t.Errorf("goroutine %d: WriteBlob error: %v", i, e) + } + } + + blobPath := blobPathFor(dir, d) + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading blob: %v", err) + } + if !bytes.Equal(got, data) { + t.Errorf("blob content = %q, want %q", got, data) + } + + if n := countTmpFiles(t, dir); n != 0 { + t.Errorf("left %d temp files behind, want 0", n) + } +} + +func TestWriteBlob_SeparateOCIInstancesDoNotShareFlights(t *testing.T) { + // Two Layout/OCI instances pointed at different roots must not share + // singleflight state -- each is expected to actually invoke open(). + dir1 := t.TempDir() + dir2 := t.TempDir() + o1, err := NewOCI(dir1) + if err != nil { + t.Fatalf("NewOCI 1: %v", err) + } + o2, err := NewOCI(dir2) + if err != nil { + t.Fatalf("NewOCI 2: %v", err) + } + + data := []byte("shared digest content across two independent stores") + d := digest.FromBytes(data) + + var opens int32 + openFn := func() (io.ReadCloser, error) { + atomic.AddInt32(&opens, 1) + return io.NopCloser(bytes.NewReader(data)), nil + } + + if err := o1.WriteBlob(context.Background(), d, int64(len(data)), openFn); err != nil { + t.Fatalf("o1.WriteBlob: %v", err) + } + if err := o2.WriteBlob(context.Background(), d, int64(len(data)), openFn); err != nil { + t.Fatalf("o2.WriteBlob: %v", err) + } + + if opens != 2 { + t.Errorf("open() called %d times across two independent stores, want 2", opens) + } +} + +// slowChunkedReader hands out data in small fixed-size chunks with a delay +// before each chunk, so that io.Copy has to call Read many times to drain it +// rather than draining it in one shot. This gives a concurrently-running +// context cancellation many chances to be observed by ctxReader between +// chunks, which is what TestWriteBlob_ContextCancellation_AbortsInFlightWrite +// needs to prove cancellation is prompt rather than "eventually noticed on +// the final EOF read". +type slowChunkedReader struct { + data []byte + chunkSize int + delay time.Duration + onFirstRead func() + once sync.Once +} + +func (r *slowChunkedReader) Read(p []byte) (int, error) { + r.once.Do(func() { + if r.onFirstRead != nil { + r.onFirstRead() + } + }) + if len(r.data) == 0 { + return 0, io.EOF + } + time.Sleep(r.delay) + n := r.chunkSize + if n > len(p) { + n = len(p) + } + if n > len(r.data) { + n = len(r.data) + } + copy(p, r.data[:n]) + r.data = r.data[n:] + return n, nil +} + +func (r *slowChunkedReader) Close() error { return nil } + +// TestWriteBlob_ContextCancellation_AbortsInFlightWrite proves the ctxReader +// wiring actually does something: without it, WriteBlob ignores ctx entirely +// and would run this ~256-chunk, ~750ms streamed write to completion even +// after cancel() fires. With it, the write must stop within a small fraction +// of that time, return an error matching context.Canceled, and leave no +// trace -- neither a final blob nor a leftover temp file -- at the digest's +// path. +func TestWriteBlob_ContextCancellation_AbortsInFlightWrite(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + data := bytes.Repeat([]byte("x"), 8*1024*1024) // 8MiB + d := digest.FromBytes(data) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + started := make(chan struct{}) + r := &slowChunkedReader{ + data: data, + chunkSize: 32 * 1024, // 256 chunks + delay: 3 * time.Millisecond, + onFirstRead: func() { close(started) }, + } + + errCh := make(chan error, 1) + writeStart := time.Now() + go func() { + errCh <- o.WriteBlob(ctx, d, int64(len(data)), func() (io.ReadCloser, error) { + return r, nil + }) + }() + + <-started + cancel() + + var writeErr error + select { + case writeErr = <-errCh: + case <-time.After(5 * time.Second): + t.Fatal("WriteBlob never returned after context cancellation") + } + elapsed := time.Since(writeStart) + + if !errors.Is(writeErr, context.Canceled) { + t.Fatalf("WriteBlob error = %v, want context.Canceled", writeErr) + } + // The uncancelled write would take ~256*3ms = ~768ms to finish streaming. + // A prompt abort should return well before that. + if elapsed > 500*time.Millisecond { + t.Errorf("WriteBlob took %s to abort after cancellation, want well under the ~768ms an uncancelled write would take", elapsed) + } + + blobPath := blobPathFor(dir, d) + if _, statErr := os.Stat(blobPath); !os.IsNotExist(statErr) { + t.Errorf("final blob path exists after cancellation: stat err = %v", statErr) + } + if n := countTmpFiles(t, dir); n != 0 { + t.Errorf("left %d temp files behind after cancellation, want 0", n) + } +} + +// TestWriteBlob_SemaphoreBoundsConcurrency writes 4x DefaultBlobConcurrency +// distinct digests concurrently -- distinct so none of them hit the fast path +// or dedupe through singleflight -- and has each one's open() track the +// concurrent-in-flight high-water mark. That watermark must never exceed +// consts.DefaultBlobConcurrency, which is only true if WriteBlob's blobSem +// acquire actually bounds the number of writers running at once. +func TestWriteBlob_SemaphoreBoundsConcurrency(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + const n = 4 * consts.DefaultBlobConcurrency + var inFlight int32 + var maxInFlight int32 + + var wg sync.WaitGroup + errs := make([]error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + data := []byte(fmt.Sprintf("distinct content #%d so every goroutine actually opens", i)) + d := digest.FromBytes(data) + errs[i] = o.WriteBlob(context.Background(), d, int64(len(data)), func() (io.ReadCloser, error) { + cur := atomic.AddInt32(&inFlight, 1) + for { + old := atomic.LoadInt32(&maxInFlight) + if cur <= old { + break + } + if atomic.CompareAndSwapInt32(&maxInFlight, old, cur) { + break + } + } + time.Sleep(20 * time.Millisecond) + atomic.AddInt32(&inFlight, -1) + return io.NopCloser(bytes.NewReader(data)), nil + }) + }() + } + wg.Wait() + + for i, e := range errs { + if e != nil { + t.Errorf("goroutine %d: WriteBlob error: %v", i, e) + } + } + + if maxInFlight > consts.DefaultBlobConcurrency { + t.Errorf("max concurrent opens = %d, want <= %d (consts.DefaultBlobConcurrency)", maxInFlight, consts.DefaultBlobConcurrency) + } +} + +// TestWriteBlob_FastPath_DoesNotAcquireSemaphore saturates blobSem completely +// (acquiring every permit and never releasing) and then calls WriteBlob for a +// digest that already exists on disk. If the fast path acquired a permit +// before returning, this call would block forever behind the saturated +// semaphore; instead it must return immediately without ever calling open(). +func TestWriteBlob_FastPath_DoesNotAcquireSemaphore(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + data := []byte("already present content, the fast path must not touch blobSem") + d := digest.FromBytes(data) + blobPath := blobPathFor(dir, d) + if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(blobPath, data, 0644); err != nil { + t.Fatalf("pre-writing blob: %v", err) + } + + for i := 0; i < consts.DefaultBlobConcurrency; i++ { + if err := o.blobSem.Acquire(context.Background(), 1); err != nil { + t.Fatalf("saturating blobSem: %v", err) + } + } + // Deliberately never released: any code path in this test that tries to + // acquire a permit will block for good. + + done := make(chan error, 1) + go func() { + done <- o.WriteBlob(context.Background(), d, int64(len(data)), func() (io.ReadCloser, error) { + return nil, errors.New("open should not have been called on the fast path") + }) + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("WriteBlob (fast path, saturated blobSem): unexpected error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("WriteBlob (fast path) blocked against a saturated blobSem -- the fast path is acquiring a permit it shouldn't") + } +} + +// gatedReader blocks its first Read call on proceed, then returns all of +// data in one shot once released. This lets a test deterministically control +// when the flight winner's io.Copy loop is unblocked relative to a context +// cancellation, without racing Go's scheduler. +type gatedReader struct { + data []byte + proceed <-chan struct{} + done bool +} + +func (g *gatedReader) Read(p []byte) (int, error) { + if g.done { + return 0, io.EOF + } + <-g.proceed + n := copy(p, g.data) + g.done = true + return n, nil +} + +func (g *gatedReader) Close() error { return nil } + +// TestWriteBlob_SingleflightWinnerCancellation_DoesNotFailIndependentWaiter +// is the acceptance test for the retry fix: goroutine 1 (its own, cancellable +// context) wins the singleflight flight for a shared digest and observes its +// own context's cancellation mid-copy. Goroutine 2 (an independent, never +// -cancelled context) joins the same flight as a waiter. Goroutine 1 must +// fail with context.Canceled; goroutine 2 must still succeed, proving the +// shared flight error was not blindly propagated to a caller whose own +// context was never cancelled. +func TestWriteBlob_SingleflightWinnerCancellation_DoesNotFailIndependentWaiter(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + data := bytes.Repeat([]byte("shared-digest-content"), 20) // small, fits in one io.Copy buffer read + d := digest.FromBytes(data) + + ctx1, cancel1 := context.WithCancel(context.Background()) + defer cancel1() + ctx2 := context.Background() // independent, never cancelled + + openCalled := make(chan struct{}) + proceed := make(chan struct{}) + open1 := func() (io.ReadCloser, error) { + close(openCalled) + return &gatedReader{data: data, proceed: proceed}, nil + } + open2 := func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + } + + errCh1 := make(chan error, 1) + go func() { + errCh1 <- o.WriteBlob(ctx1, d, int64(len(data)), open1) + }() + + select { + case <-openCalled: + case <-time.After(2 * time.Second): + t.Fatal("goroutine 1 never became the flight winner (open() was never called)") + } + + errCh2 := make(chan error, 1) + go func() { + // Give goroutine 1 time to register as the singleflight winner before + // we join as a waiter -- openCalled firing already guarantees this, + // since singleflight registers the flight before invoking the + // winner's function. + errCh2 <- o.WriteBlob(ctx2, d, int64(len(data)), open2) + }() + + // Let goroutine 2 actually reach o.sf.Do and join the in-flight call. + time.Sleep(100 * time.Millisecond) + + cancel1() + close(proceed) // unblock goroutine 1's gatedReader; its next Read observes ctx1 cancellation via ctxReader + + var err1, err2 error + select { + case err1 = <-errCh1: + case <-time.After(5 * time.Second): + t.Fatal("goroutine 1 (flight winner) never returned") + } + select { + case err2 = <-errCh2: + case <-time.After(5 * time.Second): + t.Fatal("goroutine 2 (independent waiter) never returned") + } + + if !errors.Is(err1, context.Canceled) { + t.Errorf("goroutine 1 (flight winner, own ctx cancelled) error = %v, want context.Canceled", err1) + } + if err2 != nil { + t.Errorf("goroutine 2 (independent waiter, own ctx never cancelled) error = %v, want nil", err2) + } + + blobPath := blobPathFor(dir, d) + got, readErr := os.ReadFile(blobPath) + if readErr != nil { + t.Fatalf("reading blob after retry: %v", readErr) + } + if !bytes.Equal(got, data) { + t.Errorf("blob content = %q, want %q", got, data) + } +} + +func TestOCIPusher_Push_NewBlob_AtomicRenameOnSuccess(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + data := []byte("manifest-or-layer content pushed via the docker resolver path") + d := digest.FromBytes(data) + + pusher, err := o.Pusher(context.Background(), "example.com/repo:tag") + if err != nil { + t.Fatalf("Pusher: %v", err) + } + + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageLayer, + Digest: d, + Size: int64(len(data)), + } + + w, err := pusher.Push(context.Background(), desc) + if err != nil { + t.Fatalf("Push: %v", err) + } + if _, err := w.Write(data); err != nil { + t.Fatalf("Write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: unexpected error: %v", err) + } + + blobPath := filepath.Join(dir, ocispec.ImageBlobsDir, d.Algorithm().String(), d.Hex()) + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading committed blob: %v", err) + } + if !bytes.Equal(got, data) { + t.Errorf("blob content = %q, want %q", got, data) + } + + if info, err := os.Stat(blobPath); err != nil { + t.Fatalf("stat blob: %v", err) + } else if info.Mode().Perm() != 0644 { + t.Errorf("blob mode = %o, want 0644", info.Mode().Perm()) + } + + assertNoTmpFiles(t, dir) +} + +func TestOCIPusher_Push_DigestMismatch_DoesNotRenameLeavesNoTmp(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + actual := []byte("the actual bytes streamed into the writer") + wrongDigest := digest.FromBytes([]byte("not the actual bytes")) + + pusher, err := o.Pusher(context.Background(), "example.com/repo:tag") + if err != nil { + t.Fatalf("Pusher: %v", err) + } + + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageLayer, + Digest: wrongDigest, + Size: int64(len(actual)), + } + + w, err := pusher.Push(context.Background(), desc) + if err != nil { + t.Fatalf("Push: %v", err) + } + if _, err := w.Write(actual); err != nil { + t.Fatalf("Write: %v", err) + } + + closeErr := w.Close() + if closeErr == nil { + t.Fatal("Close: expected digest mismatch error, got nil") + } + if !strings.Contains(closeErr.Error(), "digest mismatch") { + t.Errorf("Close error = %v, want it to mention digest mismatch", closeErr) + } + + blobPath := filepath.Join(dir, ocispec.ImageBlobsDir, wrongDigest.Algorithm().String(), wrongDigest.Hex()) + if _, statErr := os.Stat(blobPath); !os.IsNotExist(statErr) { + t.Errorf("final blob path exists after digest mismatch: stat err = %v", statErr) + } + + assertNoTmpFiles(t, dir) +} + +func TestOCIPusher_Push_ExistingBlob_ReturnsDiscardWriterAndLeavesBlobUntouched(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + existing := []byte("blob content that already exists on disk") + d := digest.FromBytes(existing) + blobPath := filepath.Join(dir, ocispec.ImageBlobsDir, d.Algorithm().String(), d.Hex()) + if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(blobPath, existing, 0644); err != nil { + t.Fatalf("pre-writing blob: %v", err) + } + + pusher, err := o.Pusher(context.Background(), "example.com/repo:tag") + if err != nil { + t.Fatalf("Pusher: %v", err) + } + + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageLayer, + Digest: d, + Size: int64(len(existing)), + } + + w, err := pusher.Push(context.Background(), desc) + if err != nil { + t.Fatalf("Push: %v", err) + } + // The docker resolver push path always writes the full content even if + // the pusher reports it already exists; the discard writer must consume + // it without error and without touching the existing blob. + if _, err := w.Write(existing); err != nil { + t.Fatalf("Write to discard writer: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: unexpected error: %v", err) + } + + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading blob: %v", err) + } + if !bytes.Equal(got, existing) { + t.Errorf("existing blob was modified: got %q, want %q", got, existing) + } +} + +// TestOCIPusher_Push_SharesBlobConcurrencyBound proves Push acquires against +// the same o.blobSem that content.OCI.WriteBlob does, per the plan's +// requirement that both write paths share a single process-wide ceiling. +// It saturates blobSem, confirms Push for a not-yet-existing blob blocks +// (returning ctx.Err() once ctx is cancelled while queued) rather than +// bypassing the bound, then releases one permit and confirms Push succeeds +// once a slot is free -- and that Close() releases its own permit in turn. +func TestOCIPusher_Push_SharesBlobConcurrencyBound(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + for i := 0; i < consts.DefaultBlobConcurrency; i++ { + if err := o.blobSem.Acquire(context.Background(), 1); err != nil { + t.Fatalf("saturating blobSem: %v", err) + } + } + + pusher, err := o.Pusher(context.Background(), "example.com/repo:tag") + if err != nil { + t.Fatalf("Pusher: %v", err) + } + + data := []byte("content pushed while blobSem is fully saturated") + d := digest.FromBytes(data) + desc := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageLayer, Digest: d, Size: int64(len(data))} + + // Push must block behind the saturated semaphore: prove it by cancelling + // a short-lived context while it's queued and confirming Push returns + // that cancellation rather than silently bypassing the bound. + shortCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if _, err := pusher.Push(shortCtx, desc); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Push against a saturated blobSem with a short-lived context: err = %v, want context.DeadlineExceeded", err) + } + + // Free exactly one permit, then Push (unbounded ctx this time) must + // succeed -- proving the earlier block/error was really about the + // semaphore, not something else broken about Push under saturation. + o.blobSem.Release(1) + + w, err := pusher.Push(context.Background(), desc) + if err != nil { + t.Fatalf("Push after freeing a permit: %v", err) + } + if _, err := w.Write(data); err != nil { + t.Fatalf("Write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + // Close must have released its own permit: acquiring one more should + // succeed immediately rather than blocking against the still-saturated + // remainder. + acquireDone := make(chan error, 1) + go func() { + acquireDone <- o.blobSem.Acquire(context.Background(), 1) + }() + select { + case err := <-acquireDone: + if err != nil { + t.Fatalf("Acquire after Close: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("blobSem permit was not released by Close") + } +} + +func assertNoTmpFiles(t *testing.T, root string) { + t.Helper() + dir := filepath.Join(root, ocispec.ImageBlobsDir, "sha256") + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return + } + t.Fatalf("ReadDir %s: %v", dir, err) + } + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp-") { + t.Errorf("leftover temp file: %s", e.Name()) + } + } +} + +// TestWithBlobConcurrency_OverridesPermitCount constructs an OCI with a +// custom blob concurrency of 2 and asserts exactly 2 permits can be acquired +// without blocking, while a 3rd blocks. +func TestWithBlobConcurrency_OverridesPermitCount(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir, WithBlobConcurrency(2)) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + ctx := context.Background() + if err := o.blobSem.Acquire(ctx, 1); err != nil { + t.Fatalf("acquire 1: %v", err) + } + if err := o.blobSem.Acquire(ctx, 1); err != nil { + t.Fatalf("acquire 2: %v", err) + } + + acquired := make(chan struct{}) + go func() { + o.blobSem.Acquire(context.Background(), 1) //nolint:errcheck + close(acquired) + }() + + select { + case <-acquired: + t.Fatal("3rd acquire succeeded immediately, want it to block against a 2-permit semaphore") + case <-time.After(100 * time.Millisecond): + // expected: still blocked + } +} + +// TestWithBlobConcurrency_ZeroOrNegativeIsNoOp asserts that WithBlobConcurrency +// with n <= 0 leaves the default consts.DefaultBlobConcurrency permit count in +// place, rather than constructing a semaphore with zero (permanently blocked) +// or negative (panicking) capacity. +func TestWithBlobConcurrency_ZeroOrNegativeIsNoOp(t *testing.T) { + dir := t.TempDir() + o, err := NewOCI(dir, WithBlobConcurrency(0)) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + // The default is 16 permits; acquiring one must succeed immediately. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := o.blobSem.Acquire(ctx, 1); err != nil { + t.Fatalf("acquire against default-capacity semaphore should not block/fail: %v", err) + } +} + +// TestNewOCI_NoOptions_BackwardCompatible asserts the existing no-variadic-arg +// call form still compiles and works after NewOCI gained an opts... parameter. +func TestNewOCI_NoOptions_BackwardCompatible(t *testing.T) { + dir := t.TempDir() + if _, err := NewOCI(dir); err != nil { + t.Fatalf("NewOCI(dir) with no options: %v", err) + } +} + +// digestForIndex returns a distinct, syntactically valid sha256 digest for +// index i, so concurrent test goroutines never collide on digest. +func digestForIndex(i int) string { + return fmt.Sprintf("sha256:%064x", i) +} + +// newTestOCI constructs an OCI against dir and loads its index, mirroring +// how store.NewLayout always calls LoadIndex once at construction time. +// AddIndex (both before and after this task's changes) relies on o.index +// already being non-nil -- it does not call LoadIndex itself -- so any test +// that calls AddIndex directly against a bare OCI (bypassing store.Layout) +// must load the index first, exactly like this helper does. +func newTestOCI(t *testing.T, dir string) *OCI { + t.Helper() + o, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + if err := o.LoadIndex(); err != nil { + t.Fatalf("LoadIndex: %v", err) + } + return o +} + +// refDescriptor builds a minimal, valid-for-AddIndex descriptor for index i: +// a distinct tagged reference and a distinct digest. +func refDescriptor(i int) ocispec.Descriptor { + return ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.Digest(digestForIndex(i)), + Size: int64(100 + i), + Annotations: map[string]string{ + ocispec.AnnotationRefName: fmt.Sprintf("example.com/repo%d:tag%d", i, i), + consts.KindAnnotationName: consts.KindAnnotationImage, + }, + } +} + +// -------------------------------------------------------------------------- +// TestOCI_ConcurrentAddIndex +// -------------------------------------------------------------------------- + +// TestOCI_ConcurrentAddIndex runs many goroutines each adding a distinct +// descriptor concurrently. It must not panic/race, and -- critically -- a +// *fresh* OCI opened against the same root directory afterward must see all +// entries via LoadIndex. Checking only the original OCI's in-memory nameMap +// would not catch entries lost on disk because one goroutine's nameMap.Range +// snapshot predated another goroutine's Store. +func TestOCI_ConcurrentAddIndex(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + + const n = 50 + var wg sync.WaitGroup + errs := make(chan error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + if err := o.AddIndex(refDescriptor(i)); err != nil { + errs <- fmt.Errorf("AddIndex(%d): %w", i, err) + } + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + + // Open a fresh OCI against the same root and confirm all 50 entries + // survived to disk. + fresh, err := NewOCI(dir) + if err != nil { + t.Fatalf("NewOCI (fresh): %v", err) + } + seen := make(map[string]bool) + if err := fresh.Walk(func(_ string, d ocispec.Descriptor) error { + seen[d.Annotations[ocispec.AnnotationRefName]] = true + return nil + }); err != nil { + t.Fatalf("Walk (fresh): %v", err) + } + if len(seen) != n { + t.Fatalf("fresh OCI sees %d entries on disk, want %d", len(seen), n) + } + for i := 0; i < n; i++ { + ref := fmt.Sprintf("example.com/repo%d:tag%d", i, i) + if !seen[ref] { + t.Errorf("entry %q missing from disk after concurrent AddIndex", ref) + } + } +} + +// -------------------------------------------------------------------------- +// TestOCI_ConcurrentAddIndexAndWalk +// -------------------------------------------------------------------------- + +// TestOCI_ConcurrentAddIndexAndWalk runs AddIndex and Walk concurrently. +// +// Without the locking fix in this task, Walk hands out the live descriptor +// (and its live, shared Annotations map) straight out of nameMap while +// AddIndex/SaveIndex concurrently mutate index/nameMap on another goroutine: +// this is `fatal error: concurrent map read and map write`, a hard crash of +// the whole test binary -- not a normal per-test failure. So a green run of +// this specific test (not just "go test reported no failures") is the +// signal that the fix is in place; a regression takes down the process. +func TestOCI_ConcurrentAddIndexAndWalk(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + + // Seed a few entries so Walk has something to range over from the start. + for i := 0; i < 5; i++ { + if err := o.AddIndex(refDescriptor(i)); err != nil { + t.Fatalf("seed AddIndex(%d): %v", i, err) + } + } + + const n = 50 + var wg sync.WaitGroup + + for i := 5; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _ = o.AddIndex(refDescriptor(i)) + }(i) + } + + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _ = o.Walk(func(_ string, d ocispec.Descriptor) error { + // Read the annotation the way a real Walk caller would; this is + // the read side of the concurrent read/write hazard. + _ = d.Annotations[ocispec.AnnotationRefName] + _ = d.Annotations[consts.KindAnnotationName] + return nil + }) + }() + } + + wg.Wait() +} + +// -------------------------------------------------------------------------- +// TestOCI_SaveIndexAtomic +// -------------------------------------------------------------------------- + +// TestOCI_SaveIndexAtomic exercises the on-disk atomicity of SaveIndex's +// write. AddIndex/SaveIndex run concurrently (serialized through the OCI's +// internal lock) while a separate reader repeatedly reads the raw index.json +// bytes directly off disk -- bypassing the OCI's lock entirely, the way a +// second hauler process (no shared in-process mutex) would. A non-atomic +// os.WriteFile would let that reader observe a truncated or partially +// written file; the temp-file+rename approach guarantees the reader only +// ever sees a complete prior version or a complete new version. +func TestOCI_SaveIndexAtomic(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + // Seed so index.json exists before the reader starts. + if err := o.AddIndex(refDescriptor(0)); err != nil { + t.Fatalf("seed AddIndex: %v", err) + } + + indexPath := o.path(ocispec.ImageIndexFile) + + stop := make(chan struct{}) + readErrs := make(chan error, 1) + var readCount int + go func() { + for { + select { + case <-stop: + readErrs <- nil + return + default: + } + data, err := os.ReadFile(indexPath) + if err != nil { + // A concurrent rename can transiently race an Open with ENOENT + // on some platforms; that's not the hazard under test (torn + // content), so tolerate it and keep polling. + continue + } + readCount++ + var idx ocispec.Index + if err := json.Unmarshal(data, &idx); err != nil { + readErrs <- fmt.Errorf("torn/invalid index.json read: %w (raw: %s)", err, string(data)) + return + } + } + }() + + const n = 50 + var wg sync.WaitGroup + for i := 1; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _ = o.AddIndex(refDescriptor(i)) + }(i) + } + wg.Wait() + + close(stop) + if err := <-readErrs; err != nil { + t.Fatal(err) + } + if readCount == 0 { + t.Skip("reader goroutine never observed the file in time; not a meaningful run") + } +} + +// -------------------------------------------------------------------------- +// TestOCI_WalkCallbackReenteringResolveDoesNotDeadlock +// -------------------------------------------------------------------------- + +// TestOCI_WalkCallbackReenteringResolveDoesNotDeadlock is the single most +// important test in this file. It reproduces, at the pkg/content level, the +// exact call shape of store.CopyAll's self-sync path +// (cmd/hauler/cli/store/sync.go:468 calls s.CopyAll(ctx, s.OCI, nil), and +// CopyAll's Walk callback calls l.Copy(ctx, reference, to, toRef) with +// to == l.OCI -- the same OCI instance Walk is iterating). Copy in turn +// calls Resolve, Fetcher, and Pusher on that same OCI. +// +// A naive `sync.Mutex` held across the entire Walk call would deadlock the +// very first time a callback calls back into Resolve/Pusher/LoadIndex on the +// same OCI -- which self-sync does on every normal `store sync` run, not +// just in some edge case. Walk must snapshot under the lock and release +// before invoking any callback. +// +// The whole test is wrapped in a short timeout so that a regression fails +// this test with a clear message instead of hanging `go test` (and CI) +// forever. +func TestOCI_WalkCallbackReenteringResolveDoesNotDeadlock(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + + for i := 0; i < 3; i++ { + if err := o.AddIndex(refDescriptor(i)); err != nil { + t.Fatalf("seed AddIndex(%d): %v", i, err) + } + } + + done := make(chan error, 1) + go func() { + done <- o.Walk(func(key string, d ocispec.Descriptor) error { + // Re-enter the same OCI instance from inside the Walk callback, + // exactly as store.Layout.Copy does during self-sync. + if _, err := o.Resolve(context.Background(), key); err != nil { + return fmt.Errorf("Resolve reentrant call: %w", err) + } + if _, err := o.Fetcher(context.Background(), key); err != nil { + return fmt.Errorf("Fetcher reentrant call: %w", err) + } + if _, err := o.Pusher(context.Background(), key); err != nil { + return fmt.Errorf("Pusher reentrant call: %w", err) + } + if err := o.LoadIndex(); err != nil { + return fmt.Errorf("LoadIndex reentrant call: %w", err) + } + return nil + }) + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("Walk with reentrant callback returned error: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Walk with a callback re-entering Resolve/Fetcher/Pusher/LoadIndex on the same OCI deadlocked (timed out after 5s) -- this is the exact shape of store.CopyAll's self-sync path") + } +} + +// -------------------------------------------------------------------------- +// TestOCI_UpdateAnnotations +// -------------------------------------------------------------------------- + +// TestOCI_UpdateAnnotations covers match+apply semantics: descriptors +// matching the predicate get their annotations replaced via apply, the +// number of matches is returned, and non-matching descriptors are untouched. +func TestOCI_UpdateAnnotations(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + for i := 0; i < 3; i++ { + if err := o.AddIndex(refDescriptor(i)); err != nil { + t.Fatalf("seed AddIndex(%d): %v", i, err) + } + } + + target := "example.com/repo1:tag1" + matched, err := o.UpdateAnnotations( + func(d ocispec.Descriptor) bool { + return d.Annotations[ocispec.AnnotationRefName] == target + }, + func(a map[string]string) { + a[ocispec.AnnotationRefName] = "example.com/repo1:renamed" + }, + ) + if err != nil { + t.Fatalf("UpdateAnnotations: %v", err) + } + if matched != 1 { + t.Fatalf("matched = %d, want 1", matched) + } + + var found bool + var untouchedCount int + if err := o.Walk(func(_ string, d ocispec.Descriptor) error { + ref := d.Annotations[ocispec.AnnotationRefName] + if ref == "example.com/repo1:renamed" { + found = true + } + if ref == "example.com/repo0:tag0" || ref == "example.com/repo2:tag2" { + untouchedCount++ + } + if ref == target { + t.Errorf("old reference %q still present after UpdateAnnotations", target) + } + return nil + }); err != nil { + t.Fatalf("Walk: %v", err) + } + if !found { + t.Error("renamed reference not found after UpdateAnnotations") + } + if untouchedCount != 2 { + t.Errorf("untouched entries = %d, want 2", untouchedCount) + } +} + +// TestOCI_UpdateAnnotationsNoMatchDoesNotWrite verifies that when no +// descriptor matches, UpdateAnnotations returns (0, nil) and does not touch +// index.json on disk at all -- not even a no-op rewrite. +func TestOCI_UpdateAnnotationsNoMatchDoesNotWrite(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + if err := o.AddIndex(refDescriptor(0)); err != nil { + t.Fatalf("seed AddIndex: %v", err) + } + + indexPath := o.path(ocispec.ImageIndexFile) + old := time.Now().Add(-1 * time.Hour).Truncate(time.Second) + if err := os.Chtimes(indexPath, old, old); err != nil { + t.Fatalf("Chtimes: %v", err) + } + + matched, err := o.UpdateAnnotations( + func(d ocispec.Descriptor) bool { return false }, + func(a map[string]string) { a["should-not-be-called"] = "true" }, + ) + if err != nil { + t.Fatalf("UpdateAnnotations: %v", err) + } + if matched != 0 { + t.Fatalf("matched = %d, want 0", matched) + } + + info, err := os.Stat(indexPath) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if !info.ModTime().Equal(old) { + t.Errorf("index.json mtime changed on a zero-match UpdateAnnotations call: got %v, want %v", info.ModTime(), old) + } +} + +// TestOCI_ConcurrentUpdateAnnotations runs many concurrent UpdateAnnotations +// calls, each targeting a distinct descriptor, and confirms no race/panic +// and that all renames land. +func TestOCI_ConcurrentUpdateAnnotations(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + const n = 20 + for i := 0; i < n; i++ { + if err := o.AddIndex(refDescriptor(i)); err != nil { + t.Fatalf("seed AddIndex(%d): %v", i, err) + } + } + + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + old := fmt.Sprintf("example.com/repo%d:tag%d", i, i) + _, err := o.UpdateAnnotations( + func(d ocispec.Descriptor) bool { + return d.Annotations[ocispec.AnnotationRefName] == old + }, + func(a map[string]string) { + a[ocispec.AnnotationRefName] = fmt.Sprintf("example.com/repo%d:renamed", i) + }, + ) + if err != nil { + t.Errorf("UpdateAnnotations(%d): %v", i, err) + } + }(i) + } + wg.Wait() + + renamed := make(map[string]bool) + if err := o.Walk(func(_ string, d ocispec.Descriptor) error { + renamed[d.Annotations[ocispec.AnnotationRefName]] = true + return nil + }); err != nil { + t.Fatalf("Walk: %v", err) + } + for i := 0; i < n; i++ { + want := fmt.Sprintf("example.com/repo%d:renamed", i) + if !renamed[want] { + t.Errorf("missing renamed entry %q after concurrent UpdateAnnotations", want) + } + } +} + +// -------------------------------------------------------------------------- +// TestOCI_AddIndexSkipsSaveWhenUnchanged +// -------------------------------------------------------------------------- + +// TestOCI_AddIndexSkipsSaveWhenUnchanged verifies the "cheap win" dedup: a +// second AddIndex call with a byte-identical descriptor must not rewrite +// index.json (checked via mtime, since content alone can't distinguish a +// skip from an identical rewrite). A subsequent AddIndex with a genuinely +// different descriptor (different Size) for the same key must still write. +func TestOCI_AddIndexSkipsSaveWhenUnchanged(t *testing.T) { + dir := t.TempDir() + o := newTestOCI(t, dir) + + desc := refDescriptor(0) + if err := o.AddIndex(desc); err != nil { + t.Fatalf("AddIndex (first): %v", err) + } + + indexPath := o.path(ocispec.ImageIndexFile) + old := time.Now().Add(-1 * time.Hour).Truncate(time.Second) + if err := os.Chtimes(indexPath, old, old); err != nil { + t.Fatalf("Chtimes: %v", err) + } + + // Re-add the exact same descriptor (must be treated as unchanged). + if err := o.AddIndex(desc); err != nil { + t.Fatalf("AddIndex (repeat, unchanged): %v", err) + } + info, err := os.Stat(indexPath) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if !info.ModTime().Equal(old) { + t.Errorf("index.json was rewritten for a byte-identical AddIndex: mtime got %v, want unchanged %v", info.ModTime(), old) + } + + // Now add a genuinely different descriptor for the same key (different + // Size) -- this must write. + changed := desc + changed.Size = desc.Size + 1 + if err := o.AddIndex(changed); err != nil { + t.Fatalf("AddIndex (changed): %v", err) + } + info2, err := os.Stat(indexPath) + if err != nil { + t.Fatalf("Stat (after changed): %v", err) + } + if info2.ModTime().Equal(old) { + t.Error("index.json mtime unchanged after a genuinely different AddIndex; expected a write") + } +} + +// addTestIndexEntry adds a uniquely-named descriptor through AddIndex. +func addTestIndexEntry(t *testing.T, o *OCI, i int) { + t.Helper() + desc := ocispec.Descriptor{ + MediaType: consts.OCIManifestSchema1, + Digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + Size: 1, + Annotations: map[string]string{ + ocispec.AnnotationRefName: fmt.Sprintf("hauler/test-%d:latest", i), + consts.KindAnnotationName: consts.KindAnnotationImage, + }, + } + if err := o.AddIndex(desc); err != nil { + t.Fatalf("AddIndex(%d): %v", i, err) + } +} + +func TestAddIndexCheckpointsOnInterval(t *testing.T) { + // newTestOCI calls LoadIndex, which is required before AddIndex can be + // called directly against a bare OCI -- see newTestOCI's doc comment in + // oci_concurrency_test.go for why o.index must be non-nil first. + o := newTestOCI(t, t.TempDir()) + + // Fixed clock: no time passes unless the test advances it. + clock := time.Now() + o.now = func() time.Time { return clock } + + for i := 0; i < 25; i++ { + addTestIndexEntry(t, o, i) + } + + st := o.Stats().Snapshot() + if st.IndexWrites != 25 { + t.Fatalf("IndexWrites = %d, want 25", st.IndexWrites) + } + if st.IndexDurableWrites != 1 { + t.Fatalf("IndexDurableWrites = %d, want 1 (only the first save of a run)", st.IndexDurableWrites) + } + + // Advance past the interval: the next save must be durable again. + clock = clock.Add(indexCheckpointInterval + time.Second) + addTestIndexEntry(t, o, 25) + + st = o.Stats().Snapshot() + if st.IndexDurableWrites != 2 { + t.Fatalf("IndexDurableWrites = %d after advancing the clock, want 2", st.IndexDurableWrites) + } +} + +func TestSaveIndexIsAlwaysDurable(t *testing.T) { + o := newTestOCI(t, t.TempDir()) + clock := time.Now() + o.now = func() time.Time { return clock } + + for i := 0; i < 3; i++ { + if err := o.SaveIndex(); err != nil { + t.Fatalf("SaveIndex: %v", err) + } + } + + st := o.Stats().Snapshot() + if st.IndexDurableWrites != 3 { + t.Fatalf("IndexDurableWrites = %d, want 3 (SaveIndex ignores the interval)", st.IndexDurableWrites) + } +} + +// TestCheckpointPathsProduceIdenticalIndex verifies the durable and +// non-durable paths differ only in fsync -- the bytes on disk must match. +func TestCheckpointPathsProduceIdenticalIndex(t *testing.T) { + build := func(durable bool) []byte { + o := newTestOCI(t, t.TempDir()) + addTestIndexEntry(t, o, 1) + o.lock() + if err := o.saveIndexLocked(durable); err != nil { + o.mu.Unlock() + t.Fatalf("saveIndexLocked(%v): %v", durable, err) + } + o.mu.Unlock() + + data, err := os.ReadFile(o.path(ocispec.ImageIndexFile)) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + return data + } + + if a, b := build(true), build(false); string(a) != string(b) { + t.Fatalf("durable and non-durable index bytes differ:\n durable: %s\n plain: %s", a, b) + } +} diff --git a/pkg/content/stats.go b/pkg/content/stats.go new file mode 100644 index 0000000..ac913e9 --- /dev/null +++ b/pkg/content/stats.go @@ -0,0 +1,84 @@ +package content + +import ( + "sync/atomic" + "time" +) + +// IOStats accumulates disk-contention counters for a single OCI store, so +// one pasted log line explains a field performance report -- the previous +// parallel-blob work was abandoned partly because regression reports were +// unreproducible with the reporter's contention invisible. Counters live on +// OCI rather than being context-attached (the store.WithImageStats idiom) +// because they measure store-level semaphore/mutex contention, not +// per-image state like ImageStats. All fields are atomic since every +// increment runs concurrently by design. +type IOStats struct { + BlobsWritten atomic.Int64 + BlobsCached atomic.Int64 + BlobBytesWritten atomic.Int64 + BlobSemWaitNanos atomic.Int64 + BlobPeakInFlight atomic.Int64 + IndexWrites atomic.Int64 + IndexDurableWrites atomic.Int64 + IndexBytesWritten atomic.Int64 + IndexLockWaitNanos atomic.Int64 + + // blobInFlight backs BlobPeakInFlight. It is never reported: the + // instantaneous value at the end of a run is always 0 and says nothing. + blobInFlight atomic.Int64 +} + +// IOStatsSnapshot is a plain-value copy of IOStats, so formatting code never +// touches atomics and can be unit-tested with a literal. +type IOStatsSnapshot struct { + BlobsWritten int64 + BlobsCached int64 + BlobBytesWritten int64 + BlobSemWait time.Duration + BlobPeakInFlight int64 + IndexWrites int64 + IndexDurableWrites int64 + IndexBytesWritten int64 + IndexLockWait time.Duration +} + +// Snapshot reads every counter. It is not atomic as a whole -- counters are +// read one at a time and may skew relative to each other if a sync is still +// running. Callers report it after all work has finished, where that does +// not matter. +func (s *IOStats) Snapshot() IOStatsSnapshot { + return IOStatsSnapshot{ + BlobsWritten: s.BlobsWritten.Load(), + BlobsCached: s.BlobsCached.Load(), + BlobBytesWritten: s.BlobBytesWritten.Load(), + BlobSemWait: time.Duration(s.BlobSemWaitNanos.Load()), + BlobPeakInFlight: s.BlobPeakInFlight.Load(), + IndexWrites: s.IndexWrites.Load(), + IndexDurableWrites: s.IndexDurableWrites.Load(), + IndexBytesWritten: s.IndexBytesWritten.Load(), + IndexLockWait: time.Duration(s.IndexLockWaitNanos.Load()), + } +} + +// enterBlob records that a blob write has just acquired a permit, updating +// the high-water mark with a CAS loop. Must be paired with exitBlob. +func (s *IOStats) enterBlob() { + cur := s.blobInFlight.Add(1) + for { + peak := s.BlobPeakInFlight.Load() + if cur <= peak || s.BlobPeakInFlight.CompareAndSwap(peak, cur) { + return + } + } +} + +// exitBlob records that a blob write has released its permit. +func (s *IOStats) exitBlob() { + s.blobInFlight.Add(-1) +} + +// addSemWait records time spent blocked acquiring blobSem. +func (s *IOStats) addSemWait(d time.Duration) { + s.BlobSemWaitNanos.Add(int64(d)) +} diff --git a/pkg/content/stats_test.go b/pkg/content/stats_test.go new file mode 100644 index 0000000..e8b3377 --- /dev/null +++ b/pkg/content/stats_test.go @@ -0,0 +1,148 @@ +package content + +import ( + "context" + "io" + "strings" + "sync" + "testing" + "time" + + "github.com/opencontainers/go-digest" +) + +// writeTestBlob writes content through WriteBlob and returns its digest. +func writeTestBlob(t *testing.T, o *OCI, body string) digest.Digest { + t.Helper() + dg := digest.FromString(body) + err := o.WriteBlob(context.Background(), dg, int64(len(body)), func() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(body)), nil + }) + if err != nil { + t.Fatalf("WriteBlob: %v", err) + } + return dg +} + +func TestIOStatsCountsWrittenAndCached(t *testing.T) { + o, err := NewOCI(t.TempDir()) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + + writeTestBlob(t, o, "hello world") + + st := o.Stats().Snapshot() + if st.BlobsWritten != 1 { + t.Fatalf("BlobsWritten = %d, want 1", st.BlobsWritten) + } + if st.BlobsCached != 0 { + t.Fatalf("BlobsCached = %d, want 0", st.BlobsCached) + } + if st.BlobBytesWritten != int64(len("hello world")) { + t.Fatalf("BlobBytesWritten = %d, want %d", st.BlobBytesWritten, len("hello world")) + } + + // Same digest again: must hit the os.Stat fast path, not rewrite. + writeTestBlob(t, o, "hello world") + + st = o.Stats().Snapshot() + if st.BlobsWritten != 1 { + t.Fatalf("BlobsWritten = %d after rewrite, want 1", st.BlobsWritten) + } + if st.BlobsCached != 1 { + t.Fatalf("BlobsCached = %d, want 1", st.BlobsCached) + } + if st.BlobBytesWritten != int64(len("hello world")) { + t.Fatalf("BlobBytesWritten = %d after cache hit, want unchanged", st.BlobBytesWritten) + } +} + +func TestIOStatsPeakInFlightNeverExceedsCeiling(t *testing.T) { + const ceiling = 3 + o, err := NewOCI(t.TempDir(), WithBlobConcurrency(ceiling)) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + if got := o.BlobConcurrency(); got != ceiling { + t.Fatalf("BlobConcurrency() = %d, want %d", got, ceiling) + } + + var wg sync.WaitGroup + for i := 0; i < 40; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + body := strings.Repeat("x", 1024) + string(rune('a'+i%26)) + strings.Repeat("y", i) + dg := digest.FromString(body) + _ = o.WriteBlob(context.Background(), dg, int64(len(body)), func() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(body)), nil + }) + }(i) + } + wg.Wait() + + st := o.Stats().Snapshot() + if st.BlobPeakInFlight > ceiling { + t.Fatalf("BlobPeakInFlight = %d, must never exceed the ceiling of %d", st.BlobPeakInFlight, ceiling) + } + if st.BlobPeakInFlight < 1 { + t.Fatalf("BlobPeakInFlight = %d, expected at least 1 concurrent write to be observed", st.BlobPeakInFlight) + } +} + +func TestIOStatsCountsIndexWrites(t *testing.T) { + o, err := NewOCI(t.TempDir()) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + // o.index is only populated by LoadIndex; SaveIndex on a bare OCI (no + // prior LoadIndex) is a nil-pointer panic by design -- see newTestOCI's + // doc comment in oci_concurrency_test.go. Mirror that same pattern here. + if err := o.LoadIndex(); err != nil { + t.Fatalf("LoadIndex: %v", err) + } + + if err := o.SaveIndex(); err != nil { + t.Fatalf("SaveIndex: %v", err) + } + + st := o.Stats().Snapshot() + if st.IndexWrites != 1 { + t.Fatalf("IndexWrites = %d, want 1", st.IndexWrites) + } + if st.IndexBytesWritten <= 0 { + t.Fatalf("IndexBytesWritten = %d, want > 0", st.IndexBytesWritten) + } +} + +func TestIOStatsRecordsLockWait(t *testing.T) { + o, err := NewOCI(t.TempDir()) + if err != nil { + t.Fatalf("NewOCI: %v", err) + } + // See TestIOStatsCountsIndexWrites: o.index must be populated before + // SaveIndex can run, so load it before we grab o.mu below. + if err := o.LoadIndex(); err != nil { + t.Fatalf("LoadIndex: %v", err) + } + + // Hold the mutex directly so the next lock() call must block on it. + o.mu.Lock() + + done := make(chan struct{}) + go func() { + defer close(done) + _ = o.SaveIndex() + }() + + // Give the goroutine time to reach lock() and block there. + time.Sleep(50 * time.Millisecond) + o.mu.Unlock() + <-done + + st := o.Stats().Snapshot() + if st.IndexLockWait < 10*time.Millisecond { + t.Fatalf("IndexLockWait = %v, want at least 10ms of recorded contention", st.IndexLockWait) + } +} diff --git a/pkg/content/types.go b/pkg/content/types.go index 9162a1a..35af144 100644 --- a/pkg/content/types.go +++ b/pkg/content/types.go @@ -49,15 +49,29 @@ func (w *IoContentWriter) Write(p []byte) (n int, err error) { return n, err } -// Close closes the writer and verifies the digest if configured +// Close closes the writer and verifies the digest if configured, always +// closing the underlying writer (even on mismatch) to avoid leaking file +// descriptors. If both a digest mismatch and a close error occur, the more +// informative mismatch is reported, with the close error appended. func (w *IoContentWriter) Close() error { + var mismatchErr error if w.outputHash != "" { computed := w.digester.Digest().String() if computed != w.outputHash { - return fmt.Errorf("digest mismatch: expected %s, got %s", w.outputHash, computed) + mismatchErr = fmt.Errorf("digest mismatch: expected %s, got %s", w.outputHash, computed) } } - return w.writer.Close() + + closeErr := w.writer.Close() + + switch { + case mismatchErr != nil && closeErr != nil: + return fmt.Errorf("%w (additionally, close failed: %v)", mismatchErr, closeErr) + case mismatchErr != nil: + return mismatchErr + default: + return closeErr + } } // Digest returns the current digest of written data diff --git a/pkg/content/types_test.go b/pkg/content/types_test.go new file mode 100644 index 0000000..e01a433 --- /dev/null +++ b/pkg/content/types_test.go @@ -0,0 +1,94 @@ +package content + +import ( + "errors" + "testing" +) + +// closeTrackingWriter records whether Close was called and can be made to +// return an error from Write and/or Close for testing IoContentWriter.Close's +// error-combination behavior. +type closeTrackingWriter struct { + closed bool + closeErr error + writeErr error + writeData []byte +} + +func (w *closeTrackingWriter) Write(p []byte) (int, error) { + if w.writeErr != nil { + return 0, w.writeErr + } + w.writeData = append(w.writeData, p...) + return len(p), nil +} + +func (w *closeTrackingWriter) Close() error { + w.closed = true + return w.closeErr +} + +// TestIoContentWriter_Close_DigestMismatchStillClosesUnderlyingWriter verifies +// that when the computed digest doesn't match the expected output hash, +// Close() still closes the underlying writer (no fd leak) even though it +// returns a digest-mismatch error. +func TestIoContentWriter_Close_DigestMismatchStillClosesUnderlyingWriter(t *testing.T) { + underlying := &closeTrackingWriter{} + w := NewIoContentWriter(underlying, WithOutputHash("sha256:doesnotmatch")) + + if _, err := w.Write([]byte("some data")); err != nil { + t.Fatalf("Write: %v", err) + } + + err := w.Close() + if err == nil { + t.Fatal("Close: expected digest mismatch error, got nil") + } + if !underlying.closed { + t.Error("Close: underlying writer was not closed on digest mismatch (fd leak)") + } +} + +// TestIoContentWriter_Close_NoMismatchClosesAndReturnsNil verifies the happy +// path still works: matching digest closes the underlying writer and returns +// no error. +func TestIoContentWriter_Close_NoMismatchClosesAndReturnsNil(t *testing.T) { + underlying := &closeTrackingWriter{} + w := NewIoContentWriter(underlying) + + data := []byte("some data") + if _, err := w.Write(data); err != nil { + t.Fatalf("Write: %v", err) + } + // No outputHash configured, so no verification should occur. + if err := w.Close(); err != nil { + t.Fatalf("Close: unexpected error: %v", err) + } + if !underlying.closed { + t.Error("Close: underlying writer was not closed") + } +} + +// TestIoContentWriter_Close_ReturnsCloseErrorWhenNoMismatch verifies that a +// close error from the underlying writer is surfaced when there is no digest +// mismatch to report. +func TestIoContentWriter_Close_ReturnsCloseErrorWhenNoMismatch(t *testing.T) { + wantErr := errors.New("boom: close failed") + underlying := &closeTrackingWriter{closeErr: wantErr} + w := NewIoContentWriter(underlying) + + if _, err := w.Write([]byte("data")); err != nil { + t.Fatalf("Write: %v", err) + } + + err := w.Close() + if err == nil { + t.Fatal("Close: expected close error, got nil") + } + if !errors.Is(err, wantErr) { + t.Errorf("Close: error %v does not wrap %v", err, wantErr) + } + if !underlying.closed { + t.Error("Close: underlying writer was not closed") + } +} diff --git a/pkg/cosign/cosign.go b/pkg/cosign/cosign.go deleted file mode 100644 index 1490e19..0000000 --- a/pkg/cosign/cosign.go +++ /dev/null @@ -1,58 +0,0 @@ -package cosign - -import ( - "context" - - "github.com/sigstore/cosign/v3/cmd/cosign/cli/options" - "github.com/sigstore/cosign/v3/cmd/cosign/cli/verify" - "hauler.dev/go/hauler/v2/internal/flags" - "hauler.dev/go/hauler/v2/pkg/log" - "hauler.dev/go/hauler/v2/pkg/retry" -) - -// VerifySignature verifies the digital signature of an image using Sigstore/Cosign. -func VerifySignature(ctx context.Context, keyPath string, useTlog bool, ref string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { - l := log.FromContext(ctx) - operation := func() error { - v := &verify.VerifyCommand{ - KeyRef: keyPath, - IgnoreTlog: true, // Ignore transparency log by default. - NewBundleFormat: true, - } - - if useTlog { - v.IgnoreTlog = false - } - - return log.CaptureOutput(l, true, func() error { - return v.Exec(ctx, []string{ref}) - }) - } - return retry.Operation(ctx, rso, ro, operation) -} - -// VerifyKeylessSignature verifies an image signature using keyless/OIDC identity. -func VerifyKeylessSignature(ctx context.Context, identity string, identityRegexp string, oidcIssuer string, oidcIssuerRegexp string, ghWorkflowRepository string, ref string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { - l := log.FromContext(ctx) - operation := func() error { - certVerifyOptions := options.CertVerifyOptions{ - CertOidcIssuer: oidcIssuer, - CertOidcIssuerRegexp: oidcIssuerRegexp, - CertIdentity: identity, - CertIdentityRegexp: identityRegexp, - CertGithubWorkflowRepository: ghWorkflowRepository, - } - - v := &verify.VerifyCommand{ - CertVerifyOptions: certVerifyOptions, - IgnoreTlog: false, // Use transparency log by default for keyless verification. - CertGithubWorkflowRepository: ghWorkflowRepository, - NewBundleFormat: true, - } - - return log.CaptureOutput(l, true, func() error { - return v.Exec(ctx, []string{ref}) - }) - } - return retry.Operation(ctx, rso, ro, operation) -} diff --git a/pkg/cosign/verifier.go b/pkg/cosign/verifier.go new file mode 100644 index 0000000..d3c7e3c --- /dev/null +++ b/pkg/cosign/verifier.go @@ -0,0 +1,381 @@ +package cosign + +import ( + "context" + "crypto" + "errors" + "fmt" + "sync" + + gname "github.com/google/go-containerregistry/pkg/name" + "github.com/sigstore/cosign/v3/cmd/cosign/cli/options" + "github.com/sigstore/cosign/v3/cmd/cosign/cli/verify" + cosignpkg "github.com/sigstore/cosign/v3/pkg/cosign" + + "hauler.dev/go/hauler/v2/internal/flags" + "hauler.dev/go/hauler/v2/pkg/retry" +) + +// defaultMaxWorkers bounds cosign's own per-image signature fan-out. Hauler +// already runs one Verify per image goroutine, so this only governs work +// within a single image's signature set. +const defaultMaxWorkers = 10 + +// Config is the fully-resolved verification input for one image. It is +// deliberately a comparable struct: Cache keys on it directly, so every image +// in a sync sharing a key -- 880 of them, in the Rancher case -- shares one +// Verifier and therefore one trust-material setup instead of 880. +type Config struct { + Key string + Tlog bool + + CertIdentity string + CertIdentityRegexp string + CertOidcIssuer string + CertOidcIssuerRegexp string + CertGithubWorkflowRepository string +} + +// Empty reports whether cfg requests no verification at all. +func (c Config) Empty() bool { return c == Config{} } + +// Keyless reports whether cfg verifies against a Fulcio identity rather than +// an explicit public key. +func (c Config) Keyless() bool { return c.Key == "" } + +// validate rejects a key paired with any Cert* constraint, because supplying a +// key makes every one of them dead weight: cosign reads them only where +// co.SigVerifier is nil. verifyInternal (pkg/cosign/verify.go:871) skips the +// whole ValidateAndUnpackCertWithIntermediates -> CheckCertificatePolicy -> +// validateCertExtensions path once a verifier is set, and the only other +// reader, CheckOpts.verificationOptions, belongs to the NewBundleFormat path +// this package pins off. Accepting the combination would let a user who passed +// --certificate-identity believe they had pinned who signed the image when the +// signature was checked against the key alone. Cosign's own Exec rejects the +// narrower Key+CertIdentity case as KeyAndIdentityParseError +// (cmd/cosign/cli/verify/verify.go:102); the extra fields fail identically, so +// they are guarded identically. +func (c Config) validate() error { + if c.Keyless() { + return nil + } + for _, f := range []struct{ name, value string }{ + {"CertIdentity", c.CertIdentity}, + {"CertIdentityRegexp", c.CertIdentityRegexp}, + {"CertOidcIssuer", c.CertOidcIssuer}, + {"CertOidcIssuerRegexp", c.CertOidcIssuerRegexp}, + {"CertGithubWorkflowRepository", c.CertGithubWorkflowRepository}, + } { + if f.value != "" { + return fmt.Errorf("%s is set alongside a verification key; identity constraints apply only to keyless verification and would be ignored", f.name) + } + } + return nil +} + +// Verifier verifies images against one Config. Both CheckOpts are built once +// and then treated as read-only, which is what makes concurrent Verify calls +// safe: sigstore's package-level TUF and Fulcio state is touched only by the +// setup helpers below, all of which run before the Verifier is published. +type Verifier struct { + // co drives classic tag-based verification; coBundle is its shallow copy + // with NewBundleFormat set -- see NewVerifier for why they cannot be one. + co *cosignpkg.CheckOpts + coBundle *cosignpkg.CheckOpts + closeSV func() + + // Retry settings, applied once per verification path by withRetry. + rso *flags.StoreRootOpts + ro *flags.CliRootOpts +} + +// NewVerifier builds the CheckOpts for cfg. It replicates the setup sequence in +// cosign v3.1.2's verify.VerifyCommand.Exec (cmd/cosign/cli/verify/verify.go:95-190) +// -- hauler cannot call Exec here because Exec prints, which forces the +// process-global output capture that serializes verification. +// +// ctx governs setup only in appearance: options.RegistryOptions.ClientOpts bakes +// it into co.RegistryClientOpts, which every later Verify reuses, so ctx also +// governs all registry I/O for the returned Verifier's whole lifetime. Pass a +// run-scoped ctx. A per-image or per-timeout ctx here cancels registry reads for +// every image sharing this Verifier the moment that one image's deadline fires. +func NewVerifier(ctx context.Context, cfg Config, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) (*Verifier, error) { + if err := cfg.validate(); err != nil { + return nil, err + } + + var identities []cosignpkg.Identity + if cfg.Keyless() { + certOpts := options.CertVerifyOptions{ + CertOidcIssuer: cfg.CertOidcIssuer, + CertOidcIssuerRegexp: cfg.CertOidcIssuerRegexp, + CertIdentity: cfg.CertIdentity, + CertIdentityRegexp: cfg.CertIdentityRegexp, + CertGithubWorkflowRepository: cfg.CertGithubWorkflowRepository, + } + var err error + if identities, err = certOpts.Identities(); err != nil { + return nil, fmt.Errorf("building identities: %w", err) + } + } + + regOpts := options.RegistryOptions{} + ociremoteOpts, err := regOpts.ClientOpts(ctx) + if err != nil { + return nil, fmt.Errorf("constructing registry client options: %w", err) + } + + // Keyless Fulcio certs expire ~10 minutes after issue, so the transparency + // log is mandatory there to prove the cert was valid at signing time. + // Keyed verification honors the caller's --tlog choice. + ignoreTlog := !cfg.Tlog + if cfg.Keyless() { + ignoreTlog = false + } + + co := &cosignpkg.CheckOpts{ + RegistryClientOpts: ociremoteOpts, + Identities: identities, + CertGithubWorkflowRepository: cfg.CertGithubWorkflowRepository, + IgnoreTlog: ignoreTlog, + MaxWorkers: defaultMaxWorkers, + // Must stay false: VerifyImageSignatures rejects a true value outright + // with "bundle support for image signatures is not yet implemented" + // (pkg/cosign/verify.go:645). + NewBundleFormat: false, + } + + // Mirrors cosign's unexported verifyOfflineWithKey (cli/verify/common.go:413): + // no trusted root is needed when a key is supplied and neither Rekor nor + // signed timestamps are consulted. + offlineWithKey := !cfg.Keyless() && co.IgnoreTlog && !co.UseSignedTimestamps + + // Order is load-bearing and copied from Exec: trust material first, then + // legacy clients, then the verifier -- LoadVerifierFromKeyOrCert validates + // a certificate chain against the trust material and must see it populated. + if err := verify.SetTrustedMaterial(ctx, "", "", "", "", "", offlineWithKey, co); err != nil { + return nil, fmt.Errorf("setting trusted material: %w", err) + } + + // The second and third arguments mirror cosign's unexported shouldVerifySCT + // and keylessVerification (cli/verify/common.go:387,397); both reduce to + // "no explicit key" given hauler never sets IgnoreSCT or a security key. + if err := verify.SetLegacyClientsAndKeys(ctx, co.IgnoreTlog, cfg.Keyless(), cfg.Keyless(), "", "", "", "", "", co); err != nil { + return nil, fmt.Errorf("setting up clients and keys: %w", err) + } + + sv, _, closeSV, err := verify.LoadVerifierFromKeyOrCert(ctx, cfg.Key, "", "", "", crypto.SHA256, false, false, co) + if err != nil { + return nil, fmt.Errorf("loading verifier from key opts: %w", err) + } + co.SigVerifier = sv + + // VerifyImageAttestations reads co.NewBundleFormat directly, and co's copy + // must stay false, so the bundle path needs a CheckOpts of its own. A + // shallow copy rather than a second setup run: repeating the sequence above + // would re-read the key file and, wherever offlineWithKey is false, refetch + // TUF metadata, for a path most runs never take. Sharing RegistryClientOpts, + // SigVerifier and TrustedMaterial by pointer is safe for the same reason one + // CheckOpts can serve concurrent Verify calls -- cosign's verification paths + // only read the CheckOpts they are handed; the one write on the bundle side + // lands on a copy VerifyNewBundle makes first (pkg/cosign/verify_bundle.go:30). + coBundle := *co + coBundle.NewBundleFormat = true + + return &Verifier{co: co, coBundle: &coBundle, closeSV: closeSV, rso: rso, ro: ro}, nil +} + +// Verify checks ref against v's config. ref should be a digest reference so the +// bytes verified are the bytes the caller goes on to store. +// +// The classic path handles tag-based signatures, which is every image in the +// workloads hauler targets. An image signed with the new sigstore bundle format +// has no .sig tag and so fails there with a not-found error; only that specific +// failure falls back to the bundle path. +func (v *Verifier) Verify(ctx context.Context, ref string) error { + r, err := gname.ParseReference(ref) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", ref, err) + } + + if err = v.verifyImage(ctx, r); err == nil { + return nil + } + if !fallbackEligible(err) { + return err + } + return v.verifyBundle(ctx, r) +} + +// verifyImage checks ref for a classic tag-based signature. +func (v *Verifier) verifyImage(ctx context.Context, r gname.Reference) error { + return v.withRetry(ctx, func() error { + _, _, err := cosignpkg.VerifyImageSignatures(ctx, r, v.co) + return err + }) +} + +// verifyBundle checks ref for a new-format sigstore bundle. In that format the +// signature is carried as an attestation, so VerifyImageAttestations is the +// entry point -- VerifyImageSignatures rejects NewBundleFormat outright +// (pkg/cosign/verify.go:645), and cosign's own Exec dispatches bundles the same +// way (cmd/cosign/cli/verify/verify.go:234). It returns its result rather than +// printing the verification report Exec prints, so unlike the CLI this path +// needs no log.CaptureOutput and does not serialize on its process-global +// mutex. The registry path it actually reaches -- +// verifyImageAttestationsSigstoreBundle -> GetBundles + VerifyNewBundle +// (verify.go:1905, 1713; verify_bundle.go:25) -- has cosign write no +// diagnostics of its own: the ui.Warnf calls near this code (verify.go:1795- +// 1806) belong to GetLocalBundles, the local-OCI-layout sibling this function +// never calls. +// +// Reached only for images with no classic signature, so it stays off the hot +// path for the workloads hauler actually syncs. +func (v *Verifier) verifyBundle(ctx context.Context, r gname.Reference) error { + return v.withRetry(ctx, func() error { + _, _, err := cosignpkg.VerifyImageAttestations(ctx, r, v.coBundle) + return err + }) +} + +// withRetry runs one verification call under the caller's --retries budget. +// +// The budget is applied per path rather than around Verify as a whole: a loop +// outside Verify would re-run the classic attempt on its way back to the bundle +// path, costing retries*retries attempts for every bundle-signed image. +// +// Only errors another attempt could plausibly clear consume a retry -- +// retryableVerifyErr decides. A terminal error leaves the loop by returning nil +// from the operation and travelling out through the captured variable, because +// retry.Operation has no other way to be told "stop, and this is not an +// exhausted budget". +func (v *Verifier) withRetry(ctx context.Context, call func() error) error { + var terminal error + exhausted := retry.Operation(ctx, v.rso, v.ro, func() error { + err := call() + if err != nil && !retryableVerifyErr(err) { + terminal = err + return nil + } + return err + }) + if terminal != nil { + return terminal + } + return exhausted +} + +// retryableVerifyErr reports whether a further attempt at err could succeed for +// some reason other than luck. +// +// ErrNoMatchingSignatures is excluded for exactly the reason fallbackEligible +// excludes it: signatures were found and none validated, so every extra attempt +// is another chance for a bad signature to pass. Leaving it retryable would +// reintroduce that hazard by way of --retries. +// +// ErrNoMatchingAttestations is the bundle path's terminal answer, and cosign +// spells two conditions with it: every bundle found failed verification +// (verify.go:1986), which carries the same hazard as ErrNoMatchingSignatures, +// and no bundle was found at all (verify.go:1756). The type does not +// distinguish them, and GetBundles reaches the second by skipping past +// per-referrer fetch failures (verify.go:1746-1751), so a transient blip can +// arrive here spelled as absence. Excluding it reads that ambiguity the failing +// way: the cost is an image a further attempt might have verified, against a +// failed bundle drawing --retries more chances to pass. +// +// The fallbackEligible errors are excluded because the classic signature is +// absent and will stay absent; Verify hands them to verifyBundle, which draws +// its own full budget from withRetry. +func retryableVerifyErr(err error) bool { + var noMatching *cosignpkg.ErrNoMatchingSignatures + var noAtts *cosignpkg.ErrNoMatchingAttestations + return !errors.As(err, &noMatching) && !errors.As(err, &noAtts) && !fallbackEligible(err) +} + +// fallbackEligible reports whether err means "this image carries no classic +// tag-based signature" -- the only condition under which retrying down the +// bundle path is sound. +// +// ErrNoMatchingSignatures is deliberately excluded. It means signatures were +// found and none of them validated; retrying that would give an image with a +// bad signature a second chance to pass, which is the one outcome this design +// must never permit. Every other error -- transport failures included -- also +// fails closed rather than falling back. +func fallbackEligible(err error) bool { + var tagNotFound *cosignpkg.ErrImageTagNotFound + var noSigs *cosignpkg.ErrNoSignaturesFound + return errors.As(err, &tagNotFound) || errors.As(err, &noSigs) +} + +// Close releases the signature verifier's resources. It must not be called +// until every in-flight Verify has returned. +func (v *Verifier) Close() { + if v.closeSV != nil { + v.closeSV() + } +} + +// Cache hands out one Verifier per distinct Config for the lifetime of a run. +// The retry options live here rather than in Get's arguments because only +// Config keys the map: a per-call value would be silently ignored for every +// image after the first that shares a Config. +// +// With several distinct configs, building one Verifier can overlap another +// Verifier's in-flight Verify calls. That is safe for the reason Verifier's doc +// gives, plus sigstore v1.10.8's pkg/tuf guarding its own singleton (sync.Once +// plus initMu, client.go:61-68). +type Cache struct { + mu sync.Mutex + m map[Config]*cacheEntry + rso *flags.StoreRootOpts + ro *flags.CliRootOpts +} + +type cacheEntry struct { + v *Verifier + err error +} + +// NewCache returns an empty Cache. rso and ro carry the --retries/--ignore-errors +// settings every Verifier it builds applies to verification. +func NewCache(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *Cache { + return &Cache{m: make(map[Config]*cacheEntry), rso: rso, ro: ro} +} + +// Get returns the Verifier for cfg, building it on first request. A build +// failure is cached too, so a bad key path fails fast for every image that +// shares it instead of re-reading and re-failing 880 times. +// +// ctx must be run-scoped, never per-image. Only the caller that loses no race +// -- whichever one finds cfg cold -- has its ctx handed to NewVerifier, and +// that ctx then lives inside the shared registry options every subsequent +// caller's Verify uses (see NewVerifier's doc). Passing a per-image timeout ctx +// therefore cancels registry I/O for all images sharing cfg when that one +// image's deadline fires, and which image that is depends on scheduling. +// +// The lock is held across NewVerifier rather than released around it: the whole +// point of the cache is that sigstore's trust-material setup runs once, and a +// double-checked scheme would let two goroutines both run it on a cold key. +func (c *Cache) Get(ctx context.Context, cfg Config) (*Verifier, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if e, ok := c.m[cfg]; ok { + return e.v, e.err + } + v, err := NewVerifier(ctx, cfg, c.rso, c.ro) + c.m[cfg] = &cacheEntry{v: v, err: err} + return v, err +} + +// Close releases every Verifier built by this Cache. It must not be called +// until every in-flight Verify has returned. +func (c *Cache) Close() { + c.mu.Lock() + defer c.mu.Unlock() + for _, e := range c.m { + if e.v != nil { + e.v.Close() + } + } +} diff --git a/pkg/cosign/verifier_test.go b/pkg/cosign/verifier_test.go new file mode 100644 index 0000000..c1f54aa --- /dev/null +++ b/pkg/cosign/verifier_test.go @@ -0,0 +1,672 @@ +package cosign + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io" + golog "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + gname "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/rs/zerolog" + cosignpkg "github.com/sigstore/cosign/v3/pkg/cosign" + ociremote "github.com/sigstore/cosign/v3/pkg/oci/remote" + + "hauler.dev/go/hauler/v2/internal/flags" +) + +// testOpts returns retry options for the constructors that plumb them to the +// fallback path. Retries must be non-zero: retry.Operation dereferences rso, so +// a nil would turn a test that wrongly falls back into a panic instead of a +// readable failure, and the default 3 would add two 5-second waits to it. +func testOpts() (*flags.StoreRootOpts, *flags.CliRootOpts) { + return &flags.StoreRootOpts{Retries: 1}, &flags.CliRootOpts{} +} + +// writeTestPubKey writes a fresh ECDSA P-256 public key in PEM form and returns +// its path. Generated rather than vendored so the test never depends on a +// fixture whose algorithm cosign might later stop accepting. +func writeTestPubKey(t *testing.T) string { + t.Helper() + + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + der, err := x509.MarshalPKIXPublicKey(&priv.PublicKey) + if err != nil { + t.Fatalf("marshaling public key: %v", err) + } + + path := filepath.Join(t.TempDir(), "cosign.pub") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der}), 0o600); err != nil { + t.Fatalf("writing public key: %v", err) + } + return path +} + +func TestConfigPredicates(t *testing.T) { + tests := []struct { + name string + cfg Config + wantEmpty bool + wantKeyless bool + }{ + {name: "zero value", cfg: Config{}, wantEmpty: true, wantKeyless: true}, + {name: "key only", cfg: Config{Key: "/tmp/cosign.pub"}, wantEmpty: false, wantKeyless: false}, + {name: "key with tlog", cfg: Config{Key: "/tmp/cosign.pub", Tlog: true}, wantEmpty: false, wantKeyless: false}, + {name: "identity only", cfg: Config{CertIdentity: "me@example.com"}, wantEmpty: false, wantKeyless: true}, + {name: "issuer regexp only", cfg: Config{CertOidcIssuerRegexp: ".*"}, wantEmpty: false, wantKeyless: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.cfg.Empty(); got != tt.wantEmpty { + t.Errorf("Empty() = %v, want %v", got, tt.wantEmpty) + } + if got := tt.cfg.Keyless(); got != tt.wantKeyless { + t.Errorf("Keyless() = %v, want %v", got, tt.wantKeyless) + } + }) + } +} + +func TestCacheBuildsOneVerifierPerConfig(t *testing.T) { + keyPath := writeTestPubKey(t) + cfg := Config{Key: keyPath} + + c := NewCache(testOpts()) + defer c.Close() + + first, err := c.Get(context.Background(), cfg) + if err != nil { + t.Fatalf("first Get: %v", err) + } + second, err := c.Get(context.Background(), cfg) + if err != nil { + t.Fatalf("second Get: %v", err) + } + if first != second { + t.Fatal("Cache.Get returned distinct Verifiers for an identical Config; " + + "trust material would be rebuilt per image") + } + if len(c.m) != 1 { + t.Fatalf("two Gets on one Config produced %d cache entries, want 1", len(c.m)) + } + + other, err := c.Get(context.Background(), Config{Key: writeTestPubKey(t)}) + if err != nil { + t.Fatalf("second-key Get: %v", err) + } + if other == first { + t.Fatal("Cache.Get shared a Verifier across configs that differ in Key") + } + + // Tlog: true alongside a key makes offlineWithKey false, so this build + // reaches cosign.TrustedRoot() -- a TUF fetch (cli/verify/common.go:191). + // Online it succeeds and SetLegacyClientsAndKeys then returns early at + // common.go:140 on co.TrustedMaterial != nil; offline SetTrustedMaterial + // only warns, leaving TrustedMaterial nil, and the build instead fails at + // GetRekorPubs. Asserting on the entry count rather than on the returned + // pointer holds either way: a failed build is still cached under its own + // Config, so the count rises iff the cache key covers Tlog. + _, _ = c.Get(context.Background(), Config{Key: keyPath, Tlog: true}) + if len(c.m) != 3 { + t.Fatalf("a Config differing only in Tlog produced %d cache entries, want 3; the key is too coarse", len(c.m)) + } +} + +func TestCacheGetIsConcurrencySafe(t *testing.T) { + keyPath := writeTestPubKey(t) + c := NewCache(testOpts()) + defer c.Close() + + var wg sync.WaitGroup + got := make([]*Verifier, 16) + for i := range got { + wg.Add(1) + go func() { + defer wg.Done() + v, err := c.Get(context.Background(), Config{Key: keyPath}) + if err != nil { + t.Errorf("Get: %v", err) + return + } + got[i] = v + }() + } + wg.Wait() + + for i, v := range got { + if v != got[0] { + t.Fatalf("goroutine %d got a different Verifier; the cache raced", i) + } + } +} + +// A build failure must be cached, or a bad key path is re-read and re-fails +// once per image in a sync. +func TestCacheCachesBuildFailure(t *testing.T) { + c := NewCache(testOpts()) + defer c.Close() + + cfg := Config{Key: filepath.Join(t.TempDir(), "missing.pub")} + + first, firstErr := c.Get(context.Background(), cfg) + if firstErr == nil { + t.Fatal("Get succeeded for a nonexistent key path") + } + if first != nil { + t.Fatal("Get returned a non-nil Verifier alongside an error") + } + + second, secondErr := c.Get(context.Background(), cfg) + if secondErr != firstErr { + t.Fatalf("Get rebuilt after a failure: first %v, second %v", firstErr, secondErr) + } + if second != nil { + t.Fatal("Get returned a non-nil Verifier alongside a cached error") + } +} + +// A keyless Config must name both a subject and an issuer; either alone leaves +// the other unconstrained. The assertions pin the "building identities" wrapper +// rather than just err != nil so that a reordering of NewVerifier's setup -- +// which would make these fail for some unrelated reason instead -- is caught. +func TestKeylessVerificationRequiresFullIdentity(t *testing.T) { + tests := []struct { + name string + cfg Config + }{ + {name: "subject without issuer", cfg: Config{CertIdentity: "me@example.com"}}, + {name: "subject regexp without issuer", cfg: Config{CertIdentityRegexp: ".*@example.com"}}, + {name: "issuer without subject", cfg: Config{CertOidcIssuer: "https://accounts.example.com"}}, + {name: "issuer regexp without subject", cfg: Config{CertOidcIssuerRegexp: "https://.*"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rso, ro := testOpts() + _, err := NewVerifier(context.Background(), tt.cfg, rso, ro) + if err == nil { + t.Fatal("NewVerifier accepted a keyless Config with a half-specified identity") + } + if !strings.Contains(err.Error(), "building identities") { + t.Fatalf("failed somewhere other than identity construction: %v", err) + } + }) + } +} + +// A key plus any cert constraint must not verify against the key alone: the +// user asked to pin who signed it, and cosign reads none of the Cert* fields +// once co.SigVerifier is set. One subtest per guarded field, so a copy-paste +// slip that drops one from validate's list fails on that field's name. +func TestKeyWithCertConstraintIsRejected(t *testing.T) { + keyPath := writeTestPubKey(t) + + tests := []struct { + name string + field string + cfg Config + }{ + {name: "identity", field: "CertIdentity", cfg: Config{Key: keyPath, CertIdentity: "me@example.com"}}, + {name: "identity regexp", field: "CertIdentityRegexp", cfg: Config{Key: keyPath, CertIdentityRegexp: ".*"}}, + {name: "issuer", field: "CertOidcIssuer", cfg: Config{Key: keyPath, CertOidcIssuer: "https://accounts.example.com"}}, + {name: "issuer regexp", field: "CertOidcIssuerRegexp", cfg: Config{Key: keyPath, CertOidcIssuerRegexp: "https://.*"}}, + {name: "github workflow repository", field: "CertGithubWorkflowRepository", cfg: Config{Key: keyPath, CertGithubWorkflowRepository: "example/repo"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rso, ro := testOpts() + v, err := NewVerifier(context.Background(), tt.cfg, rso, ro) + if err == nil { + t.Fatalf("NewVerifier built a Verifier that silently ignores %s", tt.field) + } + if v != nil { + t.Fatal("NewVerifier returned a Verifier alongside an error") + } + if !strings.Contains(err.Error(), tt.field) { + t.Fatalf("error does not name the ignored field %s: %v", tt.field, err) + } + }) + } +} + +// The guard must not be so wide that it rejects the configs hauler actually +// builds: sync's key branch leaves every Cert* field zero. +func TestKeyAloneIsAccepted(t *testing.T) { + rso, ro := testOpts() + if _, err := NewVerifier(context.Background(), Config{Key: writeTestPubKey(t)}, rso, ro); err != nil { + t.Fatalf("NewVerifier rejected a plain keyed Config: %v", err) + } +} + +// newSigTestRegistry starts an in-process OCI registry and returns its host +// plus the remote options that reach it. Its request log is discarded: the +// verification failures under test each drive a dozen requests, and cosign's +// own probes make the noise larger than the assertions. +func newSigTestRegistry(t *testing.T) (string, []remote.Option) { + t.Helper() + srv := httptest.NewServer(registry.New(registry.Logger(golog.New(io.Discard, "", 0)))) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://"), []remote.Option{remote.WithTransport(srv.Client().Transport)} +} + +// seedSigTestImage pushes a random image to repo:latest. When withSigTag is +// set it also pushes a random image at the cosign v2 signature tag for that +// digest -- a manifest whose layers parse as signatures but carry no +// dev.cosignproject.cosign/signature annotation, so every one of them fails +// verification. That is the "found signatures, none valid" condition. +func seedSigTestImage(t *testing.T, host, repo string, withSigTag bool, ropts []remote.Option) { + t.Helper() + + img, err := random.Image(64, 1) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + ref, err := gname.NewTag(host+"/"+repo+":latest", gname.Insecure) + if err != nil { + t.Fatalf("NewTag: %v", err) + } + if err := remote.Write(ref, img, ropts...); err != nil { + t.Fatalf("writing image: %v", err) + } + if !withSigTag { + return + } + + hash, err := img.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + sigImg, err := random.Image(64, 1) + if err != nil { + t.Fatalf("random.Image (sig): %v", err) + } + sigRef, err := gname.NewTag(host+"/"+repo+":"+strings.ReplaceAll(hash.String(), ":", "-")+".sig", gname.Insecure) + if err != nil { + t.Fatalf("NewTag (sig): %v", err) + } + if err := remote.Write(sigRef, sigImg, ropts...); err != nil { + t.Fatalf("writing signature manifest: %v", err) + } +} + +// realVerifyError returns the error cosign's own VerifyImageSignatures produces +// for ref. The sentinels this test discriminates on hold an unexported err +// field, so a zero-value &ErrNoMatchingSignatures{} panics the moment anything +// formats it -- including fmt.Errorf("%w", ...), which calls Error() eagerly. +// Driving the real code path is also what keeps the test honest: it fails if +// cosign ever stops returning these types for these conditions, which a +// hand-built value would silently hide. +func realVerifyError(t *testing.T, ref string, ropts []remote.Option) error { + t.Helper() + + rso, ro := testOpts() + v, err := NewVerifier(context.Background(), Config{Key: writeTestPubKey(t)}, rso, ro) + if err != nil { + t.Fatalf("NewVerifier: %v", err) + } + t.Cleanup(v.Close) + + // WithMoreRemoteOptions, not WithRemoteOptions: the latter overwrites ROpt + // wholesale and would discard the remote.WithContext binding ClientOpts + // installed, leaving cosign's registry reads outside the run's context. + v.co.RegistryClientOpts = append(v.co.RegistryClientOpts, ociremote.WithMoreRemoteOptions(ropts...)) + + r, err := gname.ParseReference(ref, gname.Insecure) + if err != nil { + t.Fatalf("ParseReference(%q): %v", ref, err) + } + _, _, err = cosignpkg.VerifyImageSignatures(context.Background(), r, v.co) + if err == nil { + t.Fatalf("VerifyImageSignatures(%q) succeeded against an unsigned fixture", ref) + } + return err +} + +// The whole fallback design turns on this discrimination. "No classic +// signature exists" is fail-safe and may retry down the bundle path; "the +// signatures that exist did not validate" must not, or a bad signature gets a +// second chance to pass. +func TestFallbackEligibleRejectsInvalidSignatures(t *testing.T) { + host, ropts := newSigTestRegistry(t) + seedSigTestImage(t, host, "unsigned", false, ropts) + seedSigTestImage(t, host, "badsig", true, ropts) + + noSigs := realVerifyError(t, host+"/unsigned:latest", ropts) + tagNotFound := realVerifyError(t, host+"/absent:latest", ropts) + noMatching := realVerifyError(t, host+"/badsig:latest", ropts) + + // Guard the fixtures: if a seeding change turned one of these into some + // other error, the table below would still pass while testing nothing. + for _, f := range []struct { + name string + err error + want any + }{ + {"unsigned image", noSigs, new(*cosignpkg.ErrNoSignaturesFound)}, + {"absent tag", tagNotFound, new(*cosignpkg.ErrImageTagNotFound)}, + {"unverifiable signature", noMatching, new(*cosignpkg.ErrNoMatchingSignatures)}, + } { + if !errors.As(f.err, f.want) { + t.Fatalf("%s fixture produced %T (%v), want %T", f.name, f.err, f.err, f.want) + } + } + + tests := []struct { + name string + err error + want bool + }{ + {name: "no signatures found falls back", err: noSigs, want: true}, + {name: "missing signature tag falls back", err: tagNotFound, want: true}, + {name: "no MATCHING signatures must not fall back", err: noMatching, want: false}, + {name: "wrapped no-matching must not fall back", err: fmt.Errorf("verifying: %w", noMatching), want: false}, + {name: "wrapped no-signatures still falls back", err: fmt.Errorf("verifying: %w", noSigs), want: true}, + {name: "arbitrary transport error must not fall back", err: errors.New("connection refused"), want: false}, + {name: "nil must not fall back", err: nil, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := fallbackEligible(tt.err); got != tt.want { + t.Fatalf("fallbackEligible(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + +// Verify must surface a failed-validation error as-is. If it instead fell +// through to the bundle path, the returned error would be retry.Operation's +// "operation unsuccessful after N attempts" wrapper, which is not an +// *ErrNoMatchingSignatures. +func TestVerifyDoesNotFallBackOnInvalidSignature(t *testing.T) { + host, ropts := newSigTestRegistry(t) + seedSigTestImage(t, host, "badsig", true, ropts) + + rso, ro := testOpts() + v, err := NewVerifier(context.Background(), Config{Key: writeTestPubKey(t)}, rso, ro) + if err != nil { + t.Fatalf("NewVerifier: %v", err) + } + defer v.Close() + v.co.RegistryClientOpts = append(v.co.RegistryClientOpts, ociremote.WithMoreRemoteOptions(ropts...)) + + err = v.Verify(context.Background(), host+"/badsig:latest") + if err == nil { + t.Fatal("Verify accepted an image whose only signature failed validation") + } + var noMatching *cosignpkg.ErrNoMatchingSignatures + if !errors.As(err, &noMatching) { + t.Fatalf("Verify replaced the validation failure with %T (%v); it fell back", err, err) + } +} + +// discardCtx carries a no-op logger, since retry.Operation logs every failed +// attempt through log.FromContext. +func discardCtx() context.Context { + return zerolog.New(io.Discard).WithContext(context.Background()) +} + +// countingTransport counts the requests whose path contains match, optionally +// failing them with status and running on afterwards. +type countingTransport struct { + base http.RoundTripper + match string + status int + on func() + n atomic.Int32 +} + +func (t *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if !strings.Contains(req.URL.Path, t.match) { + return t.base.RoundTrip(req) + } + t.n.Add(1) + if t.on != nil { + t.on() + } + if t.status == 0 { + return t.base.RoundTrip(req) + } + return &http.Response{ + StatusCode: t.status, + Status: http.StatusText(t.status), + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader("")), + Request: req, + }, nil +} + +// newCountingVerifier builds a keyed Verifier whose registry traffic runs +// through ct, on both the classic and the bundle path. Both need the append: +// NewVerifier takes coBundle's shallow copy before returning, so the slice +// header is already snapshotted and an append to one does not reach the other. +// The two may still write the same backing slot, which is harmless -- the value +// appended is identical. +// +// WithRemoteOptions, which replaces ROpt wholesale, is the only thing that +// works here: options.RegistryOptions.ClientOpts ends its list with +// remote.Reuse(puller) (cli/options/registry.go:169), and a reused Puller +// captured its transport when it was built, so a remote.WithTransport appended +// afterwards via WithMoreRemoteOptions is silently ignored. Dropping the +// inherited remote.WithContext along with the Reuse is deliberate: requests +// then run on context.Background, which is what lets +// TestVerifyRetriesRegistryFailures cancel the run context mid-request without +// aborting the request itself. +func newCountingVerifier(t *testing.T, retries int, ct *countingTransport) *Verifier { + t.Helper() + + v, err := NewVerifier(context.Background(), Config{Key: writeTestPubKey(t)}, + &flags.StoreRootOpts{Retries: retries}, &flags.CliRootOpts{}) + if err != nil { + t.Fatalf("NewVerifier: %v", err) + } + t.Cleanup(v.Close) + route := ociremote.WithRemoteOptions( + remote.WithTransport(ct), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + ) + v.co.RegistryClientOpts = append(v.co.RegistryClientOpts, route) + v.coBundle.RegistryClientOpts = append(v.coBundle.RegistryClientOpts, route) + return v +} + +// A signature that was found and failed to validate must cost exactly one +// attempt however large --retries is, or the retry loop hands a bad signature +// N chances to pass -- the hazard fallbackEligible exists to prevent, arriving +// by a second route. +// +// Comparing two retry budgets against the same fixture, rather than asserting +// an absolute count, keeps this independent of how many round trips cosign +// makes per attempt. +func TestVerifyDoesNotRetryInvalidSignature(t *testing.T) { + host, ropts := newSigTestRegistry(t) + seedSigTestImage(t, host, "badsig", true, ropts) + + sigFetches := func(retries int) int32 { + ct := &countingTransport{base: http.DefaultTransport, match: ".sig"} + err := newCountingVerifier(t, retries, ct).Verify(discardCtx(), host+"/badsig:latest") + var noMatching *cosignpkg.ErrNoMatchingSignatures + if !errors.As(err, &noMatching) { + t.Fatalf("Verify(retries=%d) returned %T (%v), want *ErrNoMatchingSignatures", retries, err, err) + } + return ct.n.Load() + } + + one, many := sigFetches(1), sigFetches(5) + if one == 0 { + t.Fatal("fixture never fetched the signature manifest; the assertion below would be vacuous") + } + if many != one { + t.Fatalf("--retries=5 cost %d signature-manifest fetches against --retries=1's %d: a failed validation is being retried", many, one) + } +} + +// The bundle CheckOpts must be the classic one with a single flag flipped. +// Rebuilding it instead would re-read the key and, keyless, refetch TUF; and +// the two must stay distinct objects, since a NewBundleFormat that leaked back +// into v.co would make VerifyImageSignatures reject every classic image +// outright (pkg/cosign/verify.go:645). +// +// reflect.DeepEqual is exact here despite CheckOpts holding func-valued +// options: its Slice and Ptr cases short-circuit on identical backing pointers, +// which a shallow copy guarantees, and the one func field cosign would trip +// over -- ClaimVerifier -- is nil on both, since only cosign's CLI commands +// ever set it. +func TestBundleCheckOptsIsClassicPlusFlag(t *testing.T) { + rso, ro := testOpts() + v, err := NewVerifier(context.Background(), Config{Key: writeTestPubKey(t)}, rso, ro) + if err != nil { + t.Fatalf("NewVerifier: %v", err) + } + defer v.Close() + + if v.co.NewBundleFormat { + t.Fatal("classic CheckOpts has NewBundleFormat set; VerifyImageSignatures rejects it outright") + } + if !v.coBundle.NewBundleFormat { + t.Fatal("bundle CheckOpts has NewBundleFormat clear; VerifyImageAttestations would take the tag-based branch") + } + if v.co == v.coBundle { + t.Fatal("both paths share one CheckOpts; the flag cannot differ") + } + + probe := *v.co + probe.NewBundleFormat = true + if !reflect.DeepEqual(&probe, v.coBundle) { + t.Fatal("bundle CheckOpts differs from the classic one in more than NewBundleFormat; " + + "it is no longer a shallow copy and the two paths may disagree on trust material") + } +} + +// The fallback must land in cosign's library bundle path. ErrNoMatchingAttestations +// is the proof: nothing VerifyImageSignatures reaches constructs that type, so +// the classic path cannot have returned it. The referrers count +// then pins that the request went out on the CheckOpts NewVerifier built: a +// fallback that rebuilt its own registry options, or shelled back out to the +// CLI, would reach the registry by a transport this test never sees. +func TestVerifyFallsBackToLibraryBundlePath(t *testing.T) { + host, ropts := newSigTestRegistry(t) + seedSigTestImage(t, host, "unsigned", false, ropts) + + ct := &countingTransport{base: http.DefaultTransport, match: "/referrers/"} + err := newCountingVerifier(t, 1, ct).Verify(discardCtx(), host+"/unsigned:latest") + + var noAtts *cosignpkg.ErrNoMatchingAttestations + if !errors.As(err, &noAtts) { + t.Fatalf("Verify returned %T (%v), want *ErrNoMatchingAttestations from the bundle path", err, err) + } + if ct.n.Load() == 0 { + t.Fatal("no referrers request reached this Verifier's transport; the bundle path used registry options of its own") + } +} + +// The bundle path needs its own --retries budget. Nothing else supplies one now +// that it is a bare library call, and without it a transient registry failure +// during bundle verification silently drops a signed image. +// +// The cancel-from-inside-the-request trick is explained on +// TestVerifyRetriesRegistryFailures. Failing only the referrers request leaves +// the classic attempt intact, so the run reaches verifyBundle by the ordinary +// route: no classic signature exists, which is fallback-eligible. +// +// 403 is the status that surfaces. go-containerregistry accepts 400, 404 and +// 406 on this endpoint as "no Referrers API here" and silently retries the +// fallback tag scheme (remote/referrers.go:64), and it retries 5xx itself, +// which would make this pass without hauler's loop existing at all. +func TestVerifyBundleRetriesRegistryFailures(t *testing.T) { + host, ropts := newSigTestRegistry(t) + seedSigTestImage(t, host, "unsigned", false, ropts) + + ctx, cancel := context.WithCancel(discardCtx()) + defer cancel() + + ct := &countingTransport{base: http.DefaultTransport, match: "/referrers/", status: http.StatusForbidden, on: cancel} + err := newCountingVerifier(t, 3, ct).Verify(ctx, host+"/unsigned:latest") + + if !errors.Is(err, context.Canceled) { + t.Fatalf("Verify returned %T (%v), want context.Canceled from retry.Operation's backoff; the bundle path is not being retried", err, err) + } + if ct.n.Load() == 0 { + t.Fatal("no referrers request reached the transport") + } +} + +// A bundle verification that ends in ErrNoMatchingAttestations must cost one +// attempt however large --retries is. Cosign spells both "no bundle exists" and +// "every bundle found failed to verify" with that one error, and the second is +// the hazard: retrying it hands a bad bundle N chances to pass, the same hazard +// fallbackEligible guards on the classic side. +func TestVerifyBundleDoesNotRetryMissingBundles(t *testing.T) { + host, ropts := newSigTestRegistry(t) + seedSigTestImage(t, host, "unsigned", false, ropts) + + referrerFetches := func(retries int) int32 { + ct := &countingTransport{base: http.DefaultTransport, match: "/referrers/"} + err := newCountingVerifier(t, retries, ct).Verify(discardCtx(), host+"/unsigned:latest") + var noAtts *cosignpkg.ErrNoMatchingAttestations + if !errors.As(err, &noAtts) { + t.Fatalf("Verify(retries=%d) returned %T (%v), want *ErrNoMatchingAttestations", retries, err, err) + } + return ct.n.Load() + } + + one, many := referrerFetches(1), referrerFetches(3) + if one == 0 { + t.Fatal("fixture never issued a referrers request; the assertion below would be vacuous") + } + if many != one { + t.Fatalf("--retries=3 cost %d referrers requests against --retries=1's %d: a terminal bundle result is being retried", many, one) + } +} + +// --retries must keep covering verification. The library path replaced a CLI +// call that retried internally, so dropping the loop would silently turn +// --retries into a no-op for signatures. +// +// Cancelling the run context from inside the first failing request is what +// makes a second attempt observable without waiting out consts.RetriesInterval: +// VerifyImageSignatures never sees this context -- its registry options carry +// the one NewVerifier was built with -- so a context.Canceled can only have +// come from retry.Operation's backoff, which is reached only when the error was +// judged retryable and a further attempt was pending. +func TestVerifyRetriesRegistryFailures(t *testing.T) { + host, _ := newSigTestRegistry(t) + + ctx, cancel := context.WithCancel(discardCtx()) + defer cancel() + + // 400, not 500: go-containerregistry retries 5xx itself, which would make + // this pass without hauler's loop existing at all. + ct := &countingTransport{base: http.DefaultTransport, match: "/manifests/", status: http.StatusBadRequest, on: cancel} + err := newCountingVerifier(t, 5, ct).Verify(ctx, host+"/anything:latest") + + if !errors.Is(err, context.Canceled) { + t.Fatalf("Verify returned %T (%v), want context.Canceled from retry.Operation's backoff; a registry failure is not being retried", err, err) + } + if ct.n.Load() == 0 { + t.Fatal("no manifest request reached the transport") + } +} diff --git a/pkg/getter/https.go b/pkg/getter/https.go index 19f54e5..943426a 100644 --- a/pkg/getter/https.go +++ b/pkg/getter/https.go @@ -46,7 +46,11 @@ func (h Http) Name(u *url.URL) string { } func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { - resp, err := http.Get(u.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } diff --git a/pkg/getter/https_test.go b/pkg/getter/https_test.go new file mode 100644 index 0000000..f0b3c3d --- /dev/null +++ b/pkg/getter/https_test.go @@ -0,0 +1,62 @@ +package getter_test + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "hauler.dev/go/hauler/v2/pkg/getter" +) + +// TestHttp_Open_HonorsContextCancellation proves that Http.Open actually +// wires the ctx argument into the outgoing HTTP request (via +// http.NewRequestWithContext), rather than accepting-but-ignoring it. A +// handler that never writes a response blocks http.Get indefinitely; if Open +// used http.Get instead of a context-aware request, cancelling ctx would +// never unblock the call and this test would hang until killed by the test +// binary's own timeout rather than returning promptly. +func TestHttp_Open_HonorsContextCancellation(t *testing.T) { + blockCh := make(chan struct{}) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Never respond until the test cleans up... simulates a hung/slow + // server so the only way Open returns early is via ctx cancellation. + <-blockCh + })) + // Cleanup runs LIFO: unblock the handler goroutine first so srv.Close + // (registered second, run first) doesn't itself hang waiting for the + // in-flight handler to return. + t.Cleanup(srv.Close) + t.Cleanup(func() { close(blockCh) }) + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("url.Parse: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() + + h := getter.NewHttp() + + done := make(chan error, 1) + go func() { + _, err := h.Open(ctx, u) + done <- err + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("expected an error from Open after ctx cancellation, got nil") + } + case <-time.After(5 * time.Second): + t.Fatal("Open did not return within 5s of ctx cancellation... ctx is not wired into the request") + } +} diff --git a/pkg/layer/layer.go b/pkg/layer/layer.go index 76fb927..532d23d 100644 --- a/pkg/layer/layer.go +++ b/pkg/layer/layer.go @@ -19,27 +19,20 @@ func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) { annotations: make(map[string]string, 1), } - layer.uncompressedOpener = opener - layer.compressedOpener = func() (io.ReadCloser, error) { - rc, err := opener() - if err != nil { - return nil, err - } - - return rc, nil - } + // This package never compresses: Compressed() and Uncompressed() both + // read from the same opener, so digest and diffID are always equal by + // construction and share one hash computation. A future Option adding + // real compression would break that equality and need distinct openers. + layer.opener = opener for _, opt := range opts { opt(layer) } - if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil { - return nil, err - } - - if layer.diffID, _, err = compute(layer.compressedOpener); err != nil { + if layer.digest, layer.size, err = compute(layer.opener); err != nil { return nil, err } + layer.diffID = layer.digest return layer, nil } @@ -71,14 +64,13 @@ func WithAnnotations(annotations map[string]string) Option { } type layer struct { - digest v1.Hash - diffID v1.Hash - size int64 - compressedOpener Opener - uncompressedOpener Opener - mediaType string - annotations map[string]string - urls []string + digest v1.Hash + diffID v1.Hash + size int64 + opener Opener + mediaType string + annotations map[string]string + urls []string } func (l layer) Descriptor() (*v1.Descriptor, error) { @@ -111,11 +103,11 @@ func (l layer) DiffID() (v1.Hash, error) { } func (l layer) Compressed() (io.ReadCloser, error) { - return l.compressedOpener() + return l.opener() } func (l layer) Uncompressed() (io.ReadCloser, error) { - return l.uncompressedOpener() + return l.opener() } func (l layer) Size() (int64, error) { diff --git a/pkg/layer/layer_test.go b/pkg/layer/layer_test.go new file mode 100644 index 0000000..8a778ce --- /dev/null +++ b/pkg/layer/layer_test.go @@ -0,0 +1,43 @@ +package layer + +import ( + "bytes" + "io" + "sync/atomic" + "testing" +) + +// TestFromOpener_OpensExactlyOnce proves FromOpener no longer pays for a +// redundant fetch: compressedOpener is a literal passthrough of +// uncompressedOpener (see FromOpener below), so digest and diffID are +// identical by construction and must be derived from a single read of the +// source, not two. +func TestFromOpener_OpensExactlyOnce(t *testing.T) { + var opens int32 + data := []byte("hello world") + opener := func() (io.ReadCloser, error) { + atomic.AddInt32(&opens, 1) + return io.NopCloser(bytes.NewReader(data)), nil + } + + l, err := FromOpener(opener) + if err != nil { + t.Fatalf("FromOpener: %v", err) + } + + if got := atomic.LoadInt32(&opens); got != 1 { + t.Errorf("expected opener to be invoked exactly once, got %d", got) + } + + digest, err := l.Digest() + if err != nil { + t.Fatalf("Digest: %v", err) + } + diffID, err := l.DiffID() + if err != nil { + t.Fatalf("DiffID: %v", err) + } + if digest != diffID { + t.Errorf("expected digest %v to equal diffID %v (compressedOpener is a passthrough)", digest, diffID) + } +} diff --git a/pkg/log/context.go b/pkg/log/context.go new file mode 100644 index 0000000..7b3c96c --- /dev/null +++ b/pkg/log/context.go @@ -0,0 +1,28 @@ +package log + +import "context" + +// baseLoggerKey is the context key WithBaseLogger/BaseFromContext use to +// stash an "unadorned" logger. +type baseLoggerKey struct{} + +// WithBaseLogger attaches base to ctx as the logger retrievable via +// BaseFromContext -- an "unadorned" logger without whatever contextual +// fields (e.g. a per-job "image=..." field) the ambient FromContext(ctx) +// logger carries. Use it for log lines that already name their subject +// inline (e.g. a sync job's "✓ added ..." line) so they don't also +// carry a duplicating structured field. +func WithBaseLogger(ctx context.Context, base Logger) context.Context { + return context.WithValue(ctx, baseLoggerKey{}, base) +} + +// BaseFromContext returns the Logger attached via WithBaseLogger, falling +// back to FromContext(ctx) if none was attached -- a no-op for callers that +// never had a per-job field to begin with (store add image's single-job +// path, SyncCmd's per-product-manifest loop, etc.). +func BaseFromContext(ctx context.Context) Logger { + if l, ok := ctx.Value(baseLoggerKey{}).(Logger); ok { + return l + } + return FromContext(ctx) +} diff --git a/pkg/log/context_test.go b/pkg/log/context_test.go new file mode 100644 index 0000000..64ca6a1 --- /dev/null +++ b/pkg/log/context_test.go @@ -0,0 +1,66 @@ +package log + +// context_test.go covers WithBaseLogger/BaseFromContext, the seam sync.go's +// runImageJobs uses to stash an "unadorned" logger (one without the per-job +// "image=..." field attached to jctx) for lines that already name their own +// subject inline -- see cmd/hauler/cli/store/add.go's storeImage completion +// line for the motivating bug: that field duplicated the ref the message +// already spelled out. + +import ( + "context" + "strings" + "testing" +) + +// TestBaseFromContext_FallsBackToFromContext proves that when no logger was +// ever attached via WithBaseLogger, BaseFromContext behaves identically to +// FromContext. This is what makes the fix a no-op for every caller that +// never had a per-job field to begin with (store add image's single-job +// path, SyncCmd's per-product-manifest loop, etc.) -- they never call +// WithBaseLogger, so BaseFromContext(ctx) == FromContext(ctx) for them. +func TestBaseFromContext_FallsBackToFromContext(t *testing.T) { + var buf strings.Builder + base := NewLogger(&buf) + ctx := base.WithContext(context.Background()) + + BaseFromContext(ctx).Infof("hello") + + if got := buf.String(); !strings.Contains(got, "hello") { + t.Errorf("BaseFromContext fallback did not write through the ambient logger; buf = %q", got) + } +} + +// TestBaseFromContext_StripsFieldsWhileFromContextKeepsThem proves the core +// behavior: given a ctx whose ambient (FromContext) logger carries an +// "image" field, and a base logger (without that field) attached via +// WithBaseLogger, BaseFromContext(ctx) returns the field-less logger while +// FromContext(ctx) on the very same ctx still carries the field. This is +// what lets retry.Operation's attempt-warning lines (which call +// log.FromContext(ctx) themselves, untouched by this change) keep their +// "image=" attribution while storeImage's own ref-naming lines drop the +// duplicate via BaseFromContext. +func TestBaseFromContext_StripsFieldsWhileFromContextKeepsThem(t *testing.T) { + var buf strings.Builder + base := NewLogger(&buf) + withField := base.With(Fields{"image": "example.com/repo:tag"}) + + ctx := withField.WithContext(context.Background()) + ctx = WithBaseLogger(ctx, base) + + BaseFromContext(ctx).Infof("unadorned line") + FromContext(ctx).Infof("adorned line") + + out := buf.String() + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if len(lines) != 2 { + t.Fatalf("expected 2 log lines, got %d: %q", len(lines), out) + } + + if strings.Contains(lines[0], "image=") { + t.Errorf("expected BaseFromContext's line to carry no image field, got %q", lines[0]) + } + if !strings.Contains(lines[1], "image=example.com/repo:tag") { + t.Errorf("expected FromContext's line to retain the image field, got %q", lines[1]) + } +} diff --git a/pkg/log/gating.go b/pkg/log/gating.go new file mode 100644 index 0000000..6cf07a7 --- /dev/null +++ b/pkg/log/gating.go @@ -0,0 +1,63 @@ +package log + +import ( + "os" + "strings" + + "github.com/mattn/go-isatty" +) + +// isDumbSink reports whether isTerminal, together with the environment's +// NO_COLOR and TERM=dumb signals, indicates output that should not carry +// ANSI escape codes -- either from the live progress Renderer or from +// zerolog.ConsoleWriter's coloring. This is the single source of truth both +// ShouldShowProgress and ShouldUseColor build on. +func isDumbSink(isTerminal bool) bool { + if !isTerminal { + return true + } + if os.Getenv("NO_COLOR") != "" { + return true + } + if strings.EqualFold(os.Getenv("TERM"), "dumb") { + return true + } + return false +} + +// shouldShowProgress decides whether the live progress Renderer should be +// used, given the resolved terminal-ness of stdout. It is pure in its +// parameters (isTerminal bypasses the real isatty syscall for tests) but +// still reads NO_COLOR/TERM from the environment via isDumbSink, since those +// aren't plumbed through as CLI inputs. +// +// Rules, in order, first match wins: +// 1. noProgress -> false +// 2. logLevel == "debug" (case-insensitive) -> false: interleaving a live +// region with verbose debug output is unreadable. +// 3. isDumbSink(isTerminal) (covers !isTerminal, NO_COLOR, TERM=dumb) -> false +// 4. otherwise -> true +func shouldShowProgress(isTerminal bool, noProgress bool, logLevel string) bool { + if noProgress { + return false + } + if strings.EqualFold(logLevel, "debug") { + return false + } + return !isDumbSink(isTerminal) +} + +// ShouldShowProgress is the real-world wrapper around shouldShowProgress, +// resolving terminal-ness via isatty.IsTerminal(os.Stdout.Fd()). +func ShouldShowProgress(noProgress bool, logLevel string) bool { + return shouldShowProgress(isatty.IsTerminal(os.Stdout.Fd()), noProgress, logLevel) +} + +// ShouldUseColor reports whether zerolog.ConsoleWriter should emit ANSI +// color codes, based on the same real-world signals ShouldShowProgress +// consults (isatty on os.Stdout, NO_COLOR, TERM=dumb) -- deliberately NOT +// gated on --no-progress or --log-level debug, since neither of those makes +// colored output undesirable on its own. +func ShouldUseColor() bool { + return !isDumbSink(isatty.IsTerminal(os.Stdout.Fd())) +} diff --git a/pkg/log/gating_test.go b/pkg/log/gating_test.go new file mode 100644 index 0000000..7abe1d0 --- /dev/null +++ b/pkg/log/gating_test.go @@ -0,0 +1,198 @@ +package log + +import "testing" + +// TestShouldShowProgress covers every disabling branch of shouldShowProgress +// plus the "would otherwise be true" baseline. isTerminal is passed directly +// so these tests bypass the real isatty check; NO_COLOR/TERM are read from +// the environment by shouldShowProgress itself, so tests that exercise those +// branches set them via t.Setenv and leave isTerminal true so only the +// branch under test can produce a false result. +func TestShouldShowProgress(t *testing.T) { + tests := []struct { + name string + isTerminal bool + noProgress bool + logLevel string + noColor string + term string + want bool + }{ + { + name: "baseline: terminal, not disabled, non-debug level", + isTerminal: true, + noProgress: false, + logLevel: "info", + term: "xterm-256color", + want: true, + }, + { + name: "noProgress disables regardless of terminal", + isTerminal: true, + noProgress: true, + logLevel: "info", + term: "xterm-256color", + want: false, + }, + { + name: "non-terminal disables", + isTerminal: false, + noProgress: false, + logLevel: "info", + term: "xterm-256color", + want: false, + }, + { + name: "debug log level disables", + isTerminal: true, + noProgress: false, + logLevel: "debug", + term: "xterm-256color", + want: false, + }, + { + name: "debug log level disables regardless of case", + isTerminal: true, + noProgress: false, + logLevel: "DEBUG", + term: "xterm-256color", + want: false, + }, + { + name: "empty log level does not disable", + isTerminal: true, + noProgress: false, + logLevel: "", + term: "xterm-256color", + want: true, + }, + { + name: "NO_COLOR set disables", + isTerminal: true, + noProgress: false, + logLevel: "info", + noColor: "1", + term: "xterm-256color", + want: false, + }, + { + name: "TERM=dumb disables", + isTerminal: true, + noProgress: false, + logLevel: "info", + term: "dumb", + want: false, + }, + { + name: "TERM=DUMB disables regardless of case", + isTerminal: true, + noProgress: false, + logLevel: "info", + term: "DUMB", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("NO_COLOR", tt.noColor) + t.Setenv("TERM", tt.term) + + if got := shouldShowProgress(tt.isTerminal, tt.noProgress, tt.logLevel); got != tt.want { + t.Errorf("shouldShowProgress(%v, %v, %q) = %v, want %v", tt.isTerminal, tt.noProgress, tt.logLevel, got, tt.want) + } + }) + } +} + +// TestShouldShowProgress_RealWrapperNonTerminal proves the exported +// ShouldShowProgress wrapper delegates its isTerminal check to the real +// isatty call. go test's stdout is never a terminal, so it must always +// return false here regardless of the other inputs. +func TestShouldShowProgress_RealWrapperNonTerminal(t *testing.T) { + if got := ShouldShowProgress(false, "info"); got != false { + t.Errorf("ShouldShowProgress(false, \"info\") under go test (non-terminal stdout) = %v, want false", got) + } +} + +// TestIsDumbSink covers every branch of the shared isTerminal/NO_COLOR/ +// TERM=dumb predicate that both shouldShowProgress and ShouldUseColor build +// on. isTerminal is passed directly to bypass the real isatty check; +// NO_COLOR/TERM are read from the environment by isDumbSink itself. +func TestIsDumbSink(t *testing.T) { + tests := []struct { + name string + isTerminal bool + noColor string + term string + want bool + }{ + { + name: "terminal, no NO_COLOR, non-dumb TERM is not a dumb sink", + isTerminal: true, + term: "xterm-256color", + want: false, + }, + { + name: "non-terminal is a dumb sink", + isTerminal: false, + term: "xterm-256color", + want: true, + }, + { + name: "NO_COLOR set is a dumb sink even on a terminal", + isTerminal: true, + noColor: "1", + term: "xterm-256color", + want: true, + }, + { + name: "TERM=dumb is a dumb sink even on a terminal", + isTerminal: true, + term: "dumb", + want: true, + }, + { + name: "TERM=DUMB is a dumb sink regardless of case", + isTerminal: true, + term: "DUMB", + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("NO_COLOR", tt.noColor) + t.Setenv("TERM", tt.term) + + if got := isDumbSink(tt.isTerminal); got != tt.want { + t.Errorf("isDumbSink(%v) = %v, want %v", tt.isTerminal, got, tt.want) + } + }) + } +} + +// TestShouldUseColor_RealWrapperNonTerminal is the ShouldUseColor analogue +// of TestShouldShowProgress_RealWrapperNonTerminal: go test's real stdout is +// never a terminal, so ShouldUseColor must always report false here. +func TestShouldUseColor_RealWrapperNonTerminal(t *testing.T) { + if got := ShouldUseColor(); got != false { + t.Errorf("ShouldUseColor() under go test (non-terminal stdout) = %v, want false", got) + } +} + +// TestShouldUseColor_DebugLevelStillUsesColor proves colors are not gated on +// log level -- unlike shouldShowProgress, ShouldUseColor has no notion of +// logLevel at all, so debug-level logging on a real terminal should still be +// colored. This is exercised indirectly: ShouldUseColor takes no logLevel +// parameter, so there is nothing for a "debug" value to disable. This test +// documents that omission is intentional by asserting the function's +// isDumbSink-only behavior via a terminal=true case. +func TestShouldUseColor_TerminalWithoutDumbSignalsIsTrue(t *testing.T) { + t.Setenv("NO_COLOR", "") + t.Setenv("TERM", "xterm-256color") + + if got := isDumbSink(true); got != false { + t.Errorf("isDumbSink(true) with no NO_COLOR/TERM=dumb = %v, want false (i.e. ShouldUseColor would be true on a real non-dumb terminal)", got) + } +} diff --git a/pkg/log/log.go b/pkg/log/log.go index 2d634cf..f45eaa4 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -3,7 +3,6 @@ package log import ( "context" "io" - "os" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -33,7 +32,22 @@ type Fields map[string]string // NewLogger returns a new Logger func NewLogger(out io.Writer) Logger { zerolog.TimeFieldFormat = consts.CustomTimeFormat - output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: consts.CustomTimeFormat} + // out is bound once here, captured in the ConsoleWriter's closure, so + // this logger keeps writing to its original writer even after + // CaptureOutput swaps the process-global os.Stdout/os.Stderr during a + // Helm chart traversal -- why CaptureOutput never captures hauler's own + // log lines. + // + // NoColor is decided from the real os.Stdout, not out, matching + // ShouldShowProgress: the live-region path wraps a Renderer around the + // real os.Stdout, whose terminal-ness ShouldShowProgress has already + // confirmed. out itself is often a *bytes.Buffer (tests) or the + // Renderer wrapper, neither a terminal, so isatty on out would say no. + output := zerolog.ConsoleWriter{ + Out: zerolog.SyncWriter(out), + TimeFormat: consts.CustomTimeFormat, + NoColor: !ShouldUseColor(), + } l := log.Output(output) return &logger{ zl: l.With().Timestamp().Logger(), diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go new file mode 100644 index 0000000..fa18fb4 --- /dev/null +++ b/pkg/log/log_test.go @@ -0,0 +1,41 @@ +package log + +import ( + "bytes" + "strings" + "testing" +) + +// TestNewLogger_WritesToProvidedWriter proves NewLogger honors its out +// io.Writer parameter instead of hardcoding os.Stdout. A logger built over a +// *bytes.Buffer must write its log lines into that buffer, not to the real +// process stdout. +func TestNewLogger_WritesToProvidedWriter(t *testing.T) { + var buf bytes.Buffer + l := NewLogger(&buf) + + l.Infof("hello %s", "world") + + got := buf.String() + if !strings.Contains(got, "hello world") { + t.Errorf("NewLogger(&buf).Infof did not write into the provided buffer; buf = %q", got) + } +} + +// TestNewLogger_NoAnsiEscapesUnderNonTerminal is the direct regression test +// for the bug where zerolog.ConsoleWriter emitted ANSI color codes +// unconditionally, regardless of whether the destination was a terminal. +// go test's real os.Stdout is never a terminal, so ShouldUseColor() must +// evaluate false here and NewLogger's ConsoleWriter must set NoColor, +// leaving zero "\x1b[" bytes in the output. +func TestNewLogger_NoAnsiEscapesUnderNonTerminal(t *testing.T) { + var buf bytes.Buffer + l := NewLogger(&buf) + + l.Infof("hello") + + got := buf.Bytes() + if bytes.Contains(got, []byte("\x1b[")) { + t.Errorf("NewLogger(&buf).Infof produced ANSI escape codes under go test's non-terminal stdout; buf = %q", got) + } +} diff --git a/pkg/log/logcapture.go b/pkg/log/logcapture.go index b36d10a..7b730de 100644 --- a/pkg/log/logcapture.go +++ b/pkg/log/logcapture.go @@ -42,8 +42,23 @@ func logStream(reader io.Reader, customWriter *CustomWriter, wg *sync.WaitGroup) } } +// captureMu serializes the entire body of CaptureOutput, not just the fd swap: +// os.Stdout/os.Stderr are process-global, so overlapping captures would route +// each other's output to the wrong logger and restore the wrong originals. +// +// Its scope is that swap and whatever prints beneath it. The only non-test +// caller left is the Helm chart traversal in runChartJobs +// (cmd/hauler/cli/store/add.go), whose downloader prints from transitive +// dependencies. Signature verification reaches sigstore through +// cosign.Verifier, which returns its result rather than printing the report +// cosign's CLI prints, so it needs no capture -- see Verifier.verifyBundle. +var captureMu sync.Mutex + // CaptureOutput redirects stdout and stderr to custom loggers and executes the provided function func CaptureOutput(logger Logger, debug bool, fn func() error) error { + captureMu.Lock() + defer captureMu.Unlock() + // Create pipes for capturing stdout and stderr stdoutReader, stdoutWriter, err := os.Pipe() if err != nil { diff --git a/pkg/log/logcapture_test.go b/pkg/log/logcapture_test.go new file mode 100644 index 0000000..48420bd --- /dev/null +++ b/pkg/log/logcapture_test.go @@ -0,0 +1,58 @@ +package log + +import ( + "context" + "fmt" + "io" + "os" + "sync" + "testing" + + "github.com/rs/zerolog" +) + +func discardLogger() Logger { + l := zerolog.New(io.Discard) + ctx := l.WithContext(context.Background()) + return FromContext(ctx) +} + +// TestCaptureOutput_Concurrent covers why captureMu spans the whole function +// body rather than just the redirect/restore lines: os.Stdout/os.Stderr are +// process-global, so two overlapping captures can interleave their swaps and +// leave the globals pointing at a closed pipe instead of the original files. +func TestCaptureOutput_Concurrent(t *testing.T) { + origStdout := os.Stdout + origStderr := os.Stderr + + logger := discardLogger() + + const goroutines = 5 + var wg sync.WaitGroup + wg.Add(goroutines) + + errs := make([]error, goroutines) + for i := 0; i < goroutines; i++ { + go func(i int) { + defer wg.Done() + errs[i] = CaptureOutput(logger, false, func() error { + fmt.Println("hello") + return nil + }) + }(i) + } + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Fatalf("CaptureOutput goroutine %d returned error: %v", i, err) + } + } + + if os.Stdout != origStdout { + t.Fatal("os.Stdout was not restored to its original value after concurrent CaptureOutput calls") + } + if os.Stderr != origStderr { + t.Fatal("os.Stderr was not restored to its original value after concurrent CaptureOutput calls") + } +} diff --git a/pkg/log/progress.go b/pkg/log/progress.go new file mode 100644 index 0000000..02eda6d --- /dev/null +++ b/pkg/log/progress.go @@ -0,0 +1,322 @@ +package log + +import ( + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + "golang.org/x/term" + + "hauler.dev/go/hauler/v2/pkg/consts" +) + +// spinnerFrames is the braille-dot spinner animation, advanced roughly every +// spinnerInterval by the Renderer's ticker goroutine. +var spinnerFrames = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} + +const spinnerInterval = 120 * time.Millisecond + +// fallbackWidth is used when out is not an *os.File (e.g. tests writing to a +// *bytes.Buffer) or when the terminal width can't be determined. +const fallbackWidth = 80 + +// heightMargin is reserved off the raw detected terminal height before it is +// used as a row cap. Writing to a terminal's last row triggers auto-scroll, +// which invalidates eraseLocked's "move cursor up N-1 rows" math and +// corrupts scrollback -- load-bearing, not a tunable optimization. It does +// not help if the cursor was already near the bottom when hauler started. +const heightMargin = 1 + +// Renderer renders one live row per in-flight job to a terminal, erasing and +// redrawing the whole live region as jobs begin, finish, and log lines are +// emitted through it. A job's row disappears the moment it finishes, but +// whatever it already logged (e.g. a "✓ added ..." line) stays behind in +// permanent scrollback. +// +// Locking: mu guards both the terminal (out) and all status state +// (in-flight names, spinner frame, cached width/height, drawn row count). +// Compose as log.NewLogger(renderer) so NewLogger's zerolog.SyncWriter wraps +// the Renderer, giving strict nesting SyncWriter.mu -> Renderer.mu on the +// log-write path. The spinner ticker writes to the terminal via Renderer.mu +// directly, bypassing SyncWriter entirely, so Renderer.mu is what actually +// prevents a tick-driven redraw from interleaving with an in-progress +// log-line write. Wrapping a second SyncWriter around the real os.Stdout +// inside the Renderer would create an unordered second lock on the same fd. +type Renderer struct { + out io.Writer + + mu sync.Mutex + started bool + stopped bool + inFlight []string + spinnerFrame int + width int + height int // cached effective terminal height; 0 == no cap + drawnRows int // rows currently occupying screen space from the last draw + + done chan struct{} + wg sync.WaitGroup +} + +// NewRenderer returns a Renderer that writes to out. +func NewRenderer(out io.Writer) *Renderer { + return &Renderer{out: out} +} + +// Start begins a new progress session and launches the spinner ticker +// goroutine. It caches the terminal width and height for the lifetime of the +// session (mid-run resize is out of scope). A call while a session is +// already live (started and not yet stopped) is a no-op: reassigning r.done +// here would orphan the still-running spinner goroutine on the old channel, +// which it holds in its own stack frame. Stop closes only the newest r.done, +// so that goroutine would never observe a close and r.wg.Wait() in Stop +// would block forever. +func (r *Renderer) Start() { + r.mu.Lock() + if r.started && !r.stopped { + r.mu.Unlock() + return + } + r.inFlight = nil + r.spinnerFrame = 0 + r.width, r.height = r.detectSize() + r.drawnRows = 0 + r.started = true + r.stopped = false + done := make(chan struct{}) + r.done = done + r.mu.Unlock() + + r.wg.Add(1) + go r.run(done) +} + +// Began records name as in-flight and redraws the live region. +func (r *Renderer) Began(name string) { + r.mu.Lock() + defer r.mu.Unlock() + r.inFlight = append(r.inFlight, name) + r.redrawLocked() +} + +// Finished removes name from the in-flight set and redraws the live region. +// Call it regardless of the job's success or failure so failed jobs are +// removed from the display too. +func (r *Renderer) Finished(name string) { + r.mu.Lock() + defer r.mu.Unlock() + for i, n := range r.inFlight { + if n == name { + r.inFlight = append(r.inFlight[:i], r.inFlight[i+1:]...) + break + } + } + r.redrawLocked() +} + +// Write implements io.Writer and is the sink log.NewLogger writes through. +// It erases the current live region, writes the log payload untouched, then +// redraws the region on the fresh line(s) left by the payload's trailing +// newline. +func (r *Renderer) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + + r.eraseLocked() + if _, err := r.out.Write(p); err != nil { + return 0, err + } + r.drawRowsLocked() + + return len(p), nil +} + +// Stop synchronously halts and joins the spinner goroutine, then erases the +// live region. Safe to call even if Start was never called; a single call is +// sufficient (double-Stop is not required to be safe). +func (r *Renderer) Stop() { + r.mu.Lock() + if !r.started || r.stopped { + r.mu.Unlock() + return + } + r.stopped = true + close(r.done) + r.mu.Unlock() + + r.wg.Wait() + + r.mu.Lock() + r.eraseLocked() + r.mu.Unlock() +} + +// run advances the spinner frame and redraws roughly every spinnerInterval +// until done closes. done is passed in rather than read from r.done because +// r.done is only ever written under r.mu (by Start), and this goroutine +// otherwise has no synchronized way to observe it. +func (r *Renderer) run(done chan struct{}) { + defer r.wg.Done() + + ticker := time.NewTicker(spinnerInterval) + defer ticker.Stop() + + for { + select { + case <-done: + return + case <-ticker.C: + r.mu.Lock() + r.spinnerFrame = (r.spinnerFrame + 1) % len(spinnerFrames) + r.redrawLocked() + r.mu.Unlock() + } + } +} + +// eraseLocked erases the currently-drawn live region. Callers must hold mu. +func (r *Renderer) eraseLocked() { + if r.drawnRows == 0 { + return + } + if r.drawnRows > 1 { + fmt.Fprintf(r.out, "\x1b[%dA", r.drawnRows-1) + } + fmt.Fprint(r.out, "\r\x1b[0J") + r.drawnRows = 0 +} + +// drawRowsLocked writes the current set of rows (see buildRowsLocked). +// Callers must hold mu. +func (r *Renderer) drawRowsLocked() { + lines := r.buildRowsLocked() + if len(lines) == 0 { + return + } + fmt.Fprint(r.out, strings.Join(lines, "\n")) + r.drawnRows = len(lines) +} + +// redrawLocked is eraseLocked + drawRowsLocked, used by Began/Finished/the +// spinner tick; Write uses the two halves separately. Callers must hold mu. +func (r *Renderer) redrawLocked() { + r.eraseLocked() + r.drawRowsLocked() +} + +// buildRowsLocked returns this frame's rows, truncated to width and capped +// to height (with a "+K more" summary row when capped). Callers must hold +// mu. +func (r *Renderer) buildRowsLocked() []string { + if len(r.inFlight) == 0 { + return nil + } + + frame := spinnerFrames[r.spinnerFrame] + + if r.height <= 0 || len(r.inFlight) <= r.height { + rows := make([]string, len(r.inFlight)) + for i, ref := range r.inFlight { + rows[i] = r.formatRowLocked(frame, ref) + } + return rows + } + + shown := r.height - 1 + rows := make([]string, 0, r.height) + for _, ref := range r.inFlight[:shown] { + rows = append(rows, r.formatRowLocked(frame, ref)) + } + remaining := len(r.inFlight) - shown + rows = append(rows, fmt.Sprintf(" +%d more", remaining)) + return rows +} + +// ellipsis marks a truncated ref. It is a 3-byte UTF-8 character, so +// truncation math below sizes against len(ellipsis) rather than assuming +// one byte of budget. +const ellipsis = "…" + +// levelWidth is the fixed width of zerolog.ConsoleWriter's rendered level +// field ("DBG", "INF", "WRN", "ERR", ...) -- always 3 characters. +const levelWidth = 3 + +// alignmentWidth pads a progress row to line up under the log *message* +// column rather than the timestamp; it mirrors one rendered log prefix +// (CustomTimeFormat + space + 3-char level + space) and must move if +// CustomTimeFormat's width changes. +var alignmentWidth = len(consts.CustomTimeFormat) + 1 + levelWidth + 1 + +// alignmentPrefix is alignmentWidth worth of an alternating dot-space guide +// pattern, precomputed once at package init to avoid per-frame allocation. +var alignmentPrefix = buildAlignmentPrefix() + +func buildAlignmentPrefix() string { + var b strings.Builder + b.Grow(alignmentWidth) + for i := 0; i < alignmentWidth; i++ { + if i%2 == 0 { + b.WriteByte('.') + } else { + b.WriteByte(' ') + } + } + return b.String() +} + +// formatRowLocked renders " adding ", +// truncating ref (never the fixed prefix) with a trailing ellipsis so the +// row fits within r.width. Callers must hold mu. +func (r *Renderer) formatRowLocked(frame, ref string) string { + prefix := alignmentPrefix + frame + " adding " + + budget := r.width - len(prefix) + if budget < 1 { + budget = 1 + } + if len(ref) > budget { + keep := budget - len(ellipsis) + if keep < 0 { + keep = 0 + } + ref = ref[:keep] + ellipsis + } + + return prefix + ref +} + +// detectSize queries width/height via golang.org/x/term when out is an +// *os.File, else falls back to fallbackWidth/0 (e.g. *bytes.Buffer in +// tests, or a GetSize error). Height's fallback is deliberately 0 (no cap) +// rather than a guessed size, so an unknown height never artificially +// limits rows; see heightMargin for the one-row reduction applied to a +// detected height. +func (r *Renderer) detectSize() (width, height int) { + width = fallbackWidth + height = 0 + + f, ok := r.out.(*os.File) + if !ok { + return width, height + } + + w, h, err := term.GetSize(int(f.Fd())) + if err != nil { + return width, height + } + if w > 0 { + width = w + } + if h > 0 { + if effective := h - heightMargin; effective >= 1 { + height = effective + } + // else: terminal too small after the margin; leave height at 0 (no + // cap) rather than a degenerate zero/negative-row region. + } + + return width, height +} diff --git a/pkg/log/progress_test.go b/pkg/log/progress_test.go new file mode 100644 index 0000000..854cc38 --- /dev/null +++ b/pkg/log/progress_test.go @@ -0,0 +1,473 @@ +package log + +import ( + "bytes" + "runtime" + "strings" + "sync" + "testing" + "time" + + "hauler.dev/go/hauler/v2/pkg/consts" +) + +// wantAlignmentWidth independently recomputes the expected leading-space +// padding on a progress row from the same pieces that make up a rendered log +// line prefix -- consts.CustomTimeFormat, a separating space, the 3-char +// level field, and a second separating space. It deliberately does not +// reference the package's own alignmentWidth var: the point is to catch a +// regression in that var's derivation, not merely to confirm formatRowLocked +// is internally consistent with whatever alignmentWidth happens to hold. +func wantAlignmentWidth() int { + const wantLevelWidth = 3 + return len(consts.CustomTimeFormat) + 1 + wantLevelWidth + 1 +} + +// eraseMarker is the tail of every erase sequence eraseLocked writes +// ("\r\x1b[0J", optionally preceded by a "\x1b[NA" cursor-up when more than +// one row was previously drawn). Tests locate the last occurrence of this +// marker to find the region currently visible "on screen" at the point a +// buffer snapshot was taken. +const eraseMarker = "\r\x1b[0J" + +// lastDrawnBlock returns everything written after the last erase marker in +// out -- i.e. whatever the Renderer currently has on screen at the moment +// out was captured. An empty result means the live region is empty (no +// in-flight jobs). +func lastDrawnBlock(out string) string { + idx := strings.LastIndex(out, eraseMarker) + if idx == -1 { + return out + } + return out[idx+len(eraseMarker):] +} + +// safeBuffer is a mutex-guarded io.Writer + String() buffer, deliberately +// using a lock independent of Renderer.mu. The Renderer's background +// spinner goroutine writes to it (via ticks) for the lifetime of the +// session, so any test that reads its contents while a Renderer is still +// running needs its own synchronization to stay race-free under +// `go test -race`; a bare *bytes.Buffer would not be safe for that. +type safeBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *safeBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *safeBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// TestAlignmentPrefix_DottedGuidePattern verifies that the alignment prefix +// uses a subtle alternating dot-space pattern, has the correct width, is +// ASCII-only, and is precomputed (not dynamically allocated per frame). +func TestAlignmentPrefix_DottedGuidePattern(t *testing.T) { + // Verify correct width + if got, want := len(alignmentPrefix), wantAlignmentWidth(); got != want { + t.Errorf("alignmentPrefix width = %d, want %d", got, want) + } + + // Verify alternating dot-space pattern + for i := 0; i < len(alignmentPrefix); i++ { + want := byte('.') + if i%2 == 1 { + want = byte(' ') + } + if got := alignmentPrefix[i]; got != want { + t.Errorf("alignmentPrefix[%d] = %q, want %q", i, got, want) + } + } + + // Verify ASCII-only (all bytes < 128) + for i := 0; i < len(alignmentPrefix); i++ { + if alignmentPrefix[i] >= 128 { + t.Errorf("alignmentPrefix[%d] = %d (non-ASCII), want < 128", i, alignmentPrefix[i]) + } + } + + // Verify the pattern is visually distinct from blank spaces + if alignmentPrefix == strings.Repeat(" ", len(alignmentPrefix)) { + t.Errorf("alignmentPrefix should not be all spaces (defeats visual guide purpose)") + } + + // Document the expected pattern for 24-character width (current default) + if wantAlignmentWidth() == 24 { + const want = ". . . . . . . . . . . . " + if alignmentPrefix != want { + t.Errorf("alignmentPrefix = %q, want %q", alignmentPrefix, want) + } + } +} + +// TestRenderer_MultipleBegan_ProducesDistinctRows proves that each +// concurrent in-flight job gets its own row, in insertion order. +func TestRenderer_MultipleBegan_ProducesDistinctRows(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + defer r.Stop() + + r.Began("image-a") + r.Began("image-b") + r.Began("image-c") + + block := lastDrawnBlock(buf.String()) + rows := strings.Split(block, "\n") + if len(rows) != 3 { + t.Fatalf("got %d rows, want 3: %q", len(rows), block) + } + for i, want := range []string{"image-a", "image-b", "image-c"} { + if !strings.Contains(rows[i], want) { + t.Errorf("row %d = %q, want to contain %q", i, rows[i], want) + } + } +} + +// TestRenderer_Finished_RemovesRowAndShrinksRegion proves Finished removes +// exactly the matching row and that the erase math (cursor-up count) for the +// resulting redraw reflects the smaller, post-shrink row count. +func TestRenderer_Finished_RemovesRowAndShrinksRegion(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + defer r.Stop() + + r.Began("image-a") + r.Began("image-b") + + beforeBlock := lastDrawnBlock(buf.String()) + if got := len(strings.Split(beforeBlock, "\n")); got != 2 { + t.Fatalf("setup: got %d rows before Finished, want 2: %q", got, beforeBlock) + } + + snapshot := buf.String() + r.Finished("image-a") + after := buf.String() + newBytes := after[len(snapshot):] + + // 2 rows were on screen prior to this redraw, so the erase must move + // the cursor up drawnRows-1 == 1 row before clearing. + if !strings.Contains(newBytes, "\x1b[1A") { + t.Errorf("expected cursor-up-1 escape reflecting shrink from 2 rows to 1, got %q", newBytes) + } + + rows := strings.Split(lastDrawnBlock(after), "\n") + if len(rows) != 1 { + t.Fatalf("got %d rows after Finished, want 1: %q", len(rows), lastDrawnBlock(after)) + } + if strings.Contains(rows[0], "image-a") { + t.Errorf("expected image-a to be removed, got row %q", rows[0]) + } + if !strings.Contains(rows[0], "image-b") { + t.Errorf("expected image-b to remain, got row %q", rows[0]) + } +} + +// TestRenderer_Write_GraduatesCompletedJobToScrollback proves a completion +// line written via Write appears untouched in scrollback, ahead of the +// region's final redraw with that job's row removed. +func TestRenderer_Write_GraduatesCompletedJobToScrollback(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + defer r.Stop() + + r.Began("image-a") + r.Began("image-b") + + const completionLine = "✓ added image-a (1 layer, 1 B, 0.1s)\n" + if _, err := r.Write([]byte(completionLine)); err != nil { + t.Fatalf("Write: %v", err) + } + r.Finished("image-a") + + out := buf.String() + if !strings.Contains(out, completionLine) { + t.Fatalf("expected completion line to appear untouched in scrollback, got %q", out) + } + + completionIdx := strings.Index(out, completionLine) + lastMarkerIdx := strings.LastIndex(out, eraseMarker) + if completionIdx > lastMarkerIdx { + t.Fatalf("expected completion line to precede the final region redraw") + } + + rows := strings.Split(lastDrawnBlock(out), "\n") + if len(rows) != 1 { + t.Fatalf("got %d rows after graduation, want 1: %q", len(rows), lastDrawnBlock(out)) + } + if strings.Contains(rows[0], "image-a") { + t.Errorf("expected image-a's row to be gone after graduation, got %q", rows[0]) + } + if !strings.Contains(rows[0], "image-b") { + t.Errorf("expected image-b's row to remain, got %q", rows[0]) + } +} + +// TestRenderer_Began_AfterFinished_RegrowsRegion proves the live region +// shrinks to empty when the last in-flight job finishes and grows again when +// a new job begins. +func TestRenderer_Began_AfterFinished_RegrowsRegion(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + defer r.Stop() + + r.Began("image-a") + r.Finished("image-a") + + if block := lastDrawnBlock(buf.String()); block != "" { + t.Fatalf("expected empty region after last job finished, got %q", block) + } + + r.Began("image-c") + rows := strings.Split(lastDrawnBlock(buf.String()), "\n") + if len(rows) != 1 || !strings.Contains(rows[0], "image-c") { + t.Errorf("expected region to regrow with image-c, got %q", lastDrawnBlock(buf.String())) + } +} + +// TestRenderer_HeightCap_ShowsSummaryRow proves that once the number of +// in-flight jobs exceeds the (overridden, for determinism) terminal height, +// the region caps at height rows total: height-1 job rows plus one "+K more" +// summary row accounting for the rest. +func TestRenderer_HeightCap_ShowsSummaryRow(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + defer r.Stop() + + r.mu.Lock() + r.height = 3 + r.mu.Unlock() + + names := []string{"image-a", "image-b", "image-c", "image-d", "image-e"} + for _, n := range names { + r.Began(n) + } + + block := lastDrawnBlock(buf.String()) + rows := strings.Split(block, "\n") + if len(rows) != 3 { + t.Fatalf("got %d rows, want height cap of 3: %q", len(rows), block) + } + for i, want := range names[:2] { + if !strings.Contains(rows[i], want) { + t.Errorf("row %d = %q, want to contain %q", i, rows[i], want) + } + } + last := rows[len(rows)-1] + if !strings.Contains(last, "+3 more") { + t.Errorf("expected final row to summarize the remaining 3 jobs, got %q", last) + } +} + +// TestRenderer_TruncatesLongRefToWidth proves a single very long ref is +// truncated with a trailing ellipsis, the fixed " adding " prefix is +// never truncated, and the total row never exceeds the (overridden, for +// determinism) width. +func TestRenderer_TruncatesLongRefToWidth(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + defer r.Stop() + + r.mu.Lock() + r.width = 60 + r.mu.Unlock() + + longRef := "registry.example.com/some/very/long/path/to/an/image/name:v1.2.3-extra-long-tag" + r.Began(longRef) + + rows := strings.Split(lastDrawnBlock(buf.String()), "\n") + if len(rows) != 1 { + t.Fatalf("got %d rows, want 1: %q", len(rows), lastDrawnBlock(buf.String())) + } + row := rows[0] + + if len(row) > 60 { + t.Errorf("row length %d exceeds width 60: %q", len(row), row) + } + if !strings.HasSuffix(row, "…") { + t.Errorf("expected truncated row to end with an ellipsis, got %q", row) + } + if !strings.HasPrefix(row, alignmentPrefix) { + t.Errorf("row doesn't start with alignment guide of width %d, got %q", wantAlignmentWidth(), row) + } + if len(alignmentPrefix) != wantAlignmentWidth() { + t.Errorf("alignment guide has width %d, want %d", len(alignmentPrefix), wantAlignmentWidth()) + } + + const prefix = " adding " + idx := strings.Index(row, prefix) + if idx == -1 { + t.Fatalf("expected row to contain the fixed %q prefix, got %q", prefix, row) + } + if !strings.HasPrefix(row[idx+len(prefix):], longRef[:10]) { + t.Errorf("expected truncation to preserve the start of the ref, got %q", row) + } +} + +// TestRenderer_TruncatesRealWorldLongRefAtDefaultWidth exercises +// formatRowLocked with a real production ref reported in a live sync run -- +// rgcrprod.azurecr.us/rancher/hardened-addon-resizer:1.8.23-build20260413 +// (73 characters) -- against the package's actual default width +// (fallbackWidth, 80 columns; r.width is left untouched here, unlike the +// other truncation test above which overrides it for determinism, since a +// *safeBuffer isn't an *os.File and so already resolves to fallbackWidth). +// Confirms the row renders without panicking or garbling, fits within the +// 80-column budget, and reports whether truncation actually triggered at +// this length. +func TestRenderer_TruncatesRealWorldLongRefAtDefaultWidth(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + defer r.Stop() + + const longRef = "rgcrprod.azurecr.us/rancher/hardened-addon-resizer:1.8.23-build20260413" + if got := len(longRef); got != 71 { + t.Fatalf("test setup: longRef is %d characters, want 71", got) + } + + r.Began(longRef) + + rows := strings.Split(lastDrawnBlock(buf.String()), "\n") + if len(rows) != 1 { + t.Fatalf("got %d rows, want 1: %q", len(rows), lastDrawnBlock(buf.String())) + } + row := rows[0] + + if len(row) > fallbackWidth { + t.Errorf("row length %d exceeds default width %d: %q", len(row), fallbackWidth, row) + } + if strings.Contains(row, "\x1b[") { + t.Errorf("expected the row itself to carry no escape codes, got %q", row) + } + if !strings.HasPrefix(row, alignmentPrefix) { + t.Errorf("row doesn't start with alignment guide of width %d, got %q", wantAlignmentWidth(), row) + } + if len(alignmentPrefix) != wantAlignmentWidth() { + t.Errorf("alignment guide has width %d, want %d", len(alignmentPrefix), wantAlignmentWidth()) + } + + truncated := strings.HasSuffix(row, "…") + t.Logf("real-world ref %q (%d chars) at default width %d: row = %q (truncated = %v)", longRef, len(longRef), fallbackWidth, row, truncated) + + if truncated { + if !strings.HasPrefix(row, alignmentPrefix+"⠋ adding "+longRef[:10]) { + t.Errorf("expected truncation to preserve the start of the ref, got %q", row) + } + } else { + if !strings.Contains(row, longRef) { + t.Errorf("expected the full ref to appear untruncated, got %q", row) + } + } +} + +// TestRenderer_Stop_ErasesFinalRegion proves Stop leaves no dangling rows: +// the bytes it writes erase whatever region was last drawn, with no +// replacement draw following. +func TestRenderer_Stop_ErasesFinalRegion(t *testing.T) { + buf := &safeBuffer{} + r := NewRenderer(buf) + r.Start() + r.Began("image-a") + r.Began("image-b") + + beforeStop := buf.String() + r.Stop() + out := buf.String() + newBytes := out[len(beforeStop):] + + if !strings.Contains(newBytes, eraseMarker) { + t.Fatalf("expected Stop to write a final erase sequence, got %q", newBytes) + } + // 2 rows were on screen, so the erase must move the cursor up 1 row. + if !strings.Contains(newBytes, "\x1b[1A") { + t.Errorf("expected Stop's erase to move the cursor up 1 row (2 rows were drawn), got %q", newBytes) + } + if block := lastDrawnBlock(out); block != "" { + t.Errorf("expected no rows to remain drawn after Stop, got %q", block) + } +} + +// TestRenderer_StopWithoutStart proves Stop is safe to call even when Start +// was never called. +func TestRenderer_StopWithoutStart(t *testing.T) { + var buf bytes.Buffer + r := NewRenderer(&buf) + r.Stop() +} + +// TestRenderer_StopJoinsSpinnerGoroutine proves Stop synchronously halts and +// joins the spinner ticker goroutine rather than leaking it. Run with +// -race -count=2 per the task's verification requirements. +func TestRenderer_StopJoinsSpinnerGoroutine(t *testing.T) { + before := runtime.NumGoroutine() + + for i := 0; i < 20; i++ { + var buf bytes.Buffer + r := NewRenderer(&buf) + r.Start() + r.Began("x") + r.Stop() + } + + deadline := time.Now().Add(2 * time.Second) + var after int + for { + after = runtime.NumGoroutine() + if after <= before+1 || time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + + if after > before+1 { + t.Errorf("goroutine count grew from %d to %d after 20 Start/Stop cycles; spinner goroutines appear leaked", before, after) + } +} + +// TestRenderer_DoubleStartIsIdempotent proves a second Start without an +// intervening Stop is a no-op: it must not replace r.done and must not +// launch a second spinner goroutine. Either would strand the first +// goroutine on the channel it already holds -- Stop only ever closes the +// current r.done, so the orphaned goroutine would never see a close and +// Stop's r.wg.Wait() would hang forever. A following Stop must still join +// cleanly. +func TestRenderer_DoubleStartIsIdempotent(t *testing.T) { + before := runtime.NumGoroutine() + + var buf bytes.Buffer + r := NewRenderer(&buf) + r.Start() + firstDone := r.done + + r.Start() + if r.done != firstDone { + t.Errorf("second Start replaced r.done; Start is not idempotent") + } + + r.Began("x") + r.Stop() + + deadline := time.Now().Add(2 * time.Second) + var after int + for { + after = runtime.NumGoroutine() + if after <= before || time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + if after > before { + t.Errorf("goroutine count grew from %d to %d after double Start + Stop; second Start leaked a spinner goroutine", before, after) + } +} diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go index 520b9bb..5c55bcd 100644 --- a/pkg/retry/retry.go +++ b/pkg/retry/retry.go @@ -3,7 +3,6 @@ package retry import ( "context" "fmt" - "os" "strings" "time" @@ -16,25 +15,28 @@ import ( func Operation(ctx context.Context, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, operation func() error) error { l := log.FromContext(ctx) - if !ro.IgnoreErrors { - if os.Getenv(consts.HaulerIgnoreErrors) == "true" { - ro.IgnoreErrors = true - } - } + ignoreErrors := flags.ShouldIgnoreErrors(ro) retries := rso.Retries if retries <= 0 { retries = consts.DefaultRetries } + var lastErr error + for attempt := 1; attempt <= retries; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + err := operation() if err == nil { return nil } + lastErr = err isTlogErr := strings.HasPrefix(err.Error(), "function execution failed: no matching signatures: rekor client not provided for online verification") - if ro.IgnoreErrors { + if ignoreErrors { if isTlogErr { l.Warnf("warning (attempt %d/%d)... failed tlog verification", attempt, retries) } else { @@ -49,9 +51,15 @@ func Operation(ctx context.Context, rso *flags.StoreRootOpts, ro *flags.CliRootO } if attempt < retries { - time.Sleep(time.Second * consts.RetriesInterval) + timer := time.NewTimer(time.Second * consts.RetriesInterval) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } } } - return fmt.Errorf("operation unsuccessful after %d attempts", retries) + return fmt.Errorf("operation unsuccessful after %d attempts: %w", retries, lastErr) } diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go index 4ac3b3d..3c7800c 100644 --- a/pkg/retry/retry_test.go +++ b/pkg/retry/retry_test.go @@ -2,9 +2,12 @@ package retry import ( "context" + "errors" "fmt" "io" + "strings" "testing" + "time" "github.com/rs/zerolog" @@ -54,7 +57,7 @@ func TestOperation_ExhaustsRetries(t *testing.T) { if callCount != 1 { t.Fatalf("expected 1 call, got %d", callCount) } - want := fmt.Sprintf("operation unsuccessful after %d attempts", 1) + want := fmt.Sprintf("operation unsuccessful after %d attempts: always fails", 1) if err.Error() != want { t.Fatalf("error = %q, want %q", err.Error(), want) } @@ -124,7 +127,7 @@ func TestOperation_DefaultRetries(t *testing.T) { if callCount2 != consts.DefaultRetries { t.Fatalf("expected %d calls (DefaultRetries), got %d", consts.DefaultRetries, callCount2) } - want := fmt.Sprintf("operation unsuccessful after %d attempts", consts.DefaultRetries) + want := fmt.Sprintf("operation unsuccessful after %d attempts: fail", consts.DefaultRetries) if err2.Error() != want { t.Fatalf("error = %q, want %q", err2.Error(), want) } @@ -139,6 +142,12 @@ func TestOperation_EnvVar_IgnoreErrors(t *testing.T) { t.Setenv(consts.HaulerIgnoreErrors, "true") + // Confirm the pure helper observes the env var without needing Operation to + // mutate anything first. + if !flags.ShouldIgnoreErrors(ro) { + t.Fatal("expected ShouldIgnoreErrors(ro)=true once env var is set") + } + callCount := 0 err := Operation(ctx, rso, ro, func() error { callCount++ @@ -151,10 +160,89 @@ func TestOperation_EnvVar_IgnoreErrors(t *testing.T) { if err == nil { t.Fatal("expected error after exhausting retries, got nil") } - if !ro.IgnoreErrors { - t.Fatal("expected ro.IgnoreErrors=true after env var override") + // Operation must NOT mutate ro — the env var override is read via the pure + // flags.ShouldIgnoreErrors helper on every call instead of being cached back + // onto the shared *CliRootOpts (that mutation was a data race once callers + // run concurrently). + if ro.IgnoreErrors { + t.Fatal("expected ro.IgnoreErrors to remain false: Operation must not mutate ro") } if callCount != 1 { t.Fatalf("expected 1 call, got %d", callCount) } } + +func TestOperation_ContextCancelledDuringSleep(t *testing.T) { + ctx, cancel := context.WithCancel(testContext()) + rso := &flags.StoreRootOpts{Retries: 3} + ro := &flags.CliRootOpts{} + + callCount := 0 + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + + start := time.Now() + err := Operation(ctx, rso, ro, func() error { + callCount++ + return fmt.Errorf("always fails") + }) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected error, got nil") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected error to wrap context.Canceled, got: %v", err) + } + if elapsed >= time.Second { + t.Fatalf("expected Operation to return promptly after cancellation, took %v", elapsed) + } + if callCount == 0 { + t.Fatal("expected at least one attempt before cancellation") + } +} + +func TestOperation_AlreadyCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(testContext()) + cancel() + + rso := &flags.StoreRootOpts{Retries: 3} + ro := &flags.CliRootOpts{} + + callCount := 0 + err := Operation(ctx, rso, ro, func() error { + callCount++ + return nil + }) + + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected error to wrap context.Canceled, got: %v", err) + } + if callCount != 0 { + t.Fatalf("expected operation to never be called, got %d calls", callCount) + } +} + +func TestOperation_WrapsLastError(t *testing.T) { + ctx := testContext() + rso := &flags.StoreRootOpts{Retries: 1} + ro := &flags.CliRootOpts{} + + sentinel := errors.New("sentinel: 404 not found") + + err := Operation(ctx, rso, ro, func() error { + return sentinel + }) + + if err == nil { + t.Fatal("expected error, got nil") + } + if !errors.Is(err, sentinel) { + t.Fatalf("expected error to wrap sentinel via errors.Is, got: %v", err) + } + if !strings.Contains(err.Error(), "sentinel: 404 not found") { + t.Fatalf("expected error message to contain sentinel text, got: %q", err.Error()) + } +} diff --git a/pkg/store/check_test.go b/pkg/store/check_test.go index 875853c..7734472 100644 --- a/pkg/store/check_test.go +++ b/pkg/store/check_test.go @@ -67,7 +67,7 @@ func pushAndAddExistingImage(t *testing.T, s *store.Layout, host, repo, tag stri if err := remote.Write(ref, img, opts...); err != nil { t.Fatalf("remote.Write: %v", err) } - if _, err := s.AddImage(context.Background(), ref.Name(), "", true, opts...); err != nil { + if _, err := s.AddImage(context.Background(), ref.Name(), "", true, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } return findManifestDescForRef(t, s, repo+":"+tag) diff --git a/pkg/store/stats.go b/pkg/store/stats.go new file mode 100644 index 0000000..05ddd6d --- /dev/null +++ b/pkg/store/stats.go @@ -0,0 +1,36 @@ +package store + +import ( + "context" + "sync/atomic" +) + +// ImageStats accumulates layer count and total blob bytes written while +// adding a single image, for cosmetic completion-line reporting in the CLI. +// +// Fields use sync/atomic rather than plain +=: today's call graph gives +// exactly one writer per pointer (writeImageBlobs computes size before its +// per-layer errgroup; writeIndexBlobs iterates children sequentially), but +// that single-writer property is an artifact of writeIndexBlobs's loop +// being sequential today, not a structural guarantee -- parallelizing it is +// a planned next step now that a global blob-write semaphore exists. +type ImageStats struct { + Layers atomic.Int64 + Bytes atomic.Int64 +} + +type imageStatsKey struct{} + +// WithImageStats attaches s to ctx so that writeImageBlobs (and anything +// else in this package's AddImage call graph) can record layer count/bytes +// into it. +func WithImageStats(ctx context.Context, s *ImageStats) context.Context { + return context.WithValue(ctx, imageStatsKey{}, s) +} + +// imageStatsFromContext returns the *ImageStats attached via WithImageStats, +// or nil if none was attached. +func imageStatsFromContext(ctx context.Context) *ImageStats { + s, _ := ctx.Value(imageStatsKey{}).(*ImageStats) + return s +} diff --git a/pkg/store/stats_test.go b/pkg/store/stats_test.go new file mode 100644 index 0000000..f22f005 --- /dev/null +++ b/pkg/store/stats_test.go @@ -0,0 +1,109 @@ +package store_test + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + + gname "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "hauler.dev/go/hauler/v2/pkg/store" +) + +// TestAddImage_ImageStatsAccumulation proves that attaching an *ImageStats +// to the context via WithImageStats before calling AddImage causes +// writeImageBlobs to record the image's layer count and total blob bytes, +// for cosmetic completion-line reporting in the CLI. +func TestAddImage_ImageStatsAccumulation(t *testing.T) { + srv := httptest.NewServer(registry.New()) + t.Cleanup(srv.Close) + host := strings.TrimPrefix(srv.URL, "http://") + + remoteOpts := []remote.Option{ + remote.WithTransport(srv.Client().Transport), + } + + const numLayers = 3 + img, err := random.Image(512, numLayers) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + + layers, err := img.Layers() + if err != nil { + t.Fatalf("img.Layers: %v", err) + } + var wantBytes int64 + for _, l := range layers { + size, err := l.Size() + if err != nil { + t.Fatalf("layer.Size: %v", err) + } + wantBytes += size + } + + tag, err := gname.NewTag(host+"/test/image:v1", gname.Insecure) + if err != nil { + t.Fatalf("new tag: %v", err) + } + if err := remote.Write(tag, img, remoteOpts...); err != nil { + t.Fatalf("push image: %v", err) + } + + s, err := store.NewLayout(t.TempDir()) + if err != nil { + t.Fatalf("new layout: %v", err) + } + + stats := &store.ImageStats{} + ctx := store.WithImageStats(context.Background(), stats) + + if _, err := s.AddImage(ctx, tag.Name(), "", false, "", remoteOpts...); err != nil { + t.Fatalf("AddImage: %v", err) + } + + if got := stats.Layers.Load(); got != int64(numLayers) { + t.Errorf("stats.Layers = %d, want %d", got, numLayers) + } + if got := stats.Bytes.Load(); got != wantBytes { + t.Errorf("stats.Bytes = %d, want %d", got, wantBytes) + } +} + +// TestAddImage_NoImageStatsInContext proves AddImage works fine (no panic) +// when the context has no ImageStats attached -- the common case for every +// call site that doesn't care about stats. +func TestAddImage_NoImageStatsInContext(t *testing.T) { + srv := httptest.NewServer(registry.New()) + t.Cleanup(srv.Close) + host := strings.TrimPrefix(srv.URL, "http://") + + remoteOpts := []remote.Option{ + remote.WithTransport(srv.Client().Transport), + } + + img, err := random.Image(512, 1) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + tag, err := gname.NewTag(host+"/test/image:v1", gname.Insecure) + if err != nil { + t.Fatalf("new tag: %v", err) + } + if err := remote.Write(tag, img, remoteOpts...); err != nil { + t.Fatalf("push image: %v", err) + } + + s, err := store.NewLayout(t.TempDir()) + if err != nil { + t.Fatalf("new layout: %v", err) + } + + if _, err := s.AddImage(context.Background(), tag.Name(), "", false, "", remoteOpts...); err != nil { + t.Fatalf("AddImage: %v", err) + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index 3e57531..f3231e9 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -37,6 +37,12 @@ type Layout struct { StoreID string haulerDir string cache layer.Cache + + // blobConcurrency overrides the OCI layout's default blob-write + // concurrency ceiling (content.OCI.blobSem) when > 0. Set via + // WithBlobConcurrency; must be known before content.NewOCI is called, so + // NewLayout applies opts before constructing the OCI store. + blobConcurrency int } type Options func(*Layout) @@ -55,19 +61,18 @@ func WithHaulerDir(dir string) Options { } } +// WithBlobConcurrency overrides the OCI layout's default blob-write +// concurrency ceiling. See content.WithBlobConcurrency for the mechanics and +// why the floor in flags.BlobConcurrencyFor matters. +func WithBlobConcurrency(n int) Options { + return func(l *Layout) { + l.blobConcurrency = n + } +} + func NewLayout(rootdir string, opts ...Options) (*Layout, error) { - ociStore, err := content.NewOCI(rootdir) - if err != nil { - return nil, err - } - - if err := ociStore.LoadIndex(); err != nil { - return nil, err - } - l := &Layout{ Root: rootdir, - OCI: ociStore, StoreID: loadOrCreateStoreID(rootdir), } @@ -75,6 +80,19 @@ func NewLayout(rootdir string, opts ...Options) (*Layout, error) { opt(l) } + var ociOpts []content.OCIOption + if l.blobConcurrency > 0 { + ociOpts = append(ociOpts, content.WithBlobConcurrency(l.blobConcurrency)) + } + ociStore, err := content.NewOCI(rootdir, ociOpts...) + if err != nil { + return nil, err + } + if err := ociStore.LoadIndex(); err != nil { + return nil, err + } + l.OCI = ociStore + if l.haulerDir != "" { updateStoreInventory(l.haulerDir, l.StoreID, rootdir) } @@ -139,7 +157,7 @@ func (l *Layout) AddArtifact(ctx context.Context, oci artifacts.OCI, ref string) if err != nil { return ocispec.Descriptor{}, err } - if err := l.writeBlobData(mdata); err != nil { + if err := l.writeBlobData(ctx, mdata); err != nil { return ocispec.Descriptor{}, err } @@ -149,7 +167,7 @@ func (l *Layout) AddArtifact(ctx context.Context, oci artifacts.OCI, ref string) return ocispec.Descriptor{}, err } - if err := l.writeBlobData(cdata); err != nil { + if err := l.writeBlobData(ctx, cdata); err != nil { return ocispec.Descriptor{}, err } @@ -159,11 +177,16 @@ func (l *Layout) AddArtifact(ctx context.Context, oci artifacts.OCI, ref string) return ocispec.Descriptor{}, err } - var g errgroup.Group + // errgroup.WithContext, not a zero-value Group: Wait() on a zero-value + // group never cancels siblings on failure, so other layers would keep + // downloading after one fails. gctx is cancelled the moment any + // writeLayer errors, which content.OCI.WriteBlob observes (via ctx.Err() + // and the wrapped reader) to abort in-flight writes promptly. + g, gctx := errgroup.WithContext(ctx) for _, lyr := range layers { lyr := lyr g.Go(func() error { - return l.writeLayer(lyr) + return l.writeLayer(gctx, lyr) }) } if err := g.Wait(); err != nil { @@ -209,7 +232,13 @@ func (l *Layout) AddArtifactCollection(ctx context.Context, collection artifacts // discovered via cosign's tag convention (.sig, .att, .sbom). // When platform is non-empty and the ref is a multi-arch index, only that platform is fetched. // When excludeExtras is true, cosign signatures, attestations, SBOMs, and OCI referrers are skipped. -func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, opts ...remote.Option) (string, error) { +// +// pinnedDigest, when non-empty, is the digest actually fetched -- ref supplies +// only the name recorded in the index annotations. Callers that verified a +// signature pass the digest they verified, so the bytes stored are provably +// the bytes checked even if the tag moves mid-run. An empty pinnedDigest +// resolves ref normally. +func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, pinnedDigest string, opts ...remote.Option) (string, error) { allOpts := append([]remote.Option{ remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithContext(ctx), @@ -220,11 +249,25 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl return "", fmt.Errorf("parsing reference %q: %w", ref, err) } - desc, err := remote.Get(parsedRef, allOpts...) + // fetchRef drives the network; parsedRef stays the annotation ref so the + // store keeps recording the tag a user asked for rather than a digest. + fetchRef := parsedRef + if pinnedDigest != "" { + fetchRef = parsedRef.Context().Digest(pinnedDigest) + } + + desc, err := remote.Get(fetchRef, allOpts...) if err != nil { return "", fmt.Errorf("fetching descriptor for %q: %w", ref, err) } + // go-containerregistry already validates content against a digest ref; + // this restates the invariant locally so a future refactor that switches + // fetchRef back to a tag fails loudly instead of silently unpinning. + if pinnedDigest != "" && desc.Digest.String() != pinnedDigest { + return "", fmt.Errorf("digest mismatch for %q: fetched %s, pinned %s", ref, desc.Digest, pinnedDigest) + } + var imageDigest v1.Hash if idx, idxErr := desc.ImageIndex(); idxErr == nil && platform == "" { @@ -233,11 +276,15 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl if err != nil { return "", fmt.Errorf("getting index digest for %q: %w", ref, err) } - if err := l.writeIndex(parsedRef, idx, consts.KindAnnotationIndex); err != nil { + if err := l.writeIndex(ctx, parsedRef, idx, consts.KindAnnotationIndex); err != nil { return "", err } } else { // Single-platform image, or the caller requested a specific platform. + // + // Under a platform filter the pinned digest is the index's while the stored + // digest is the selected child's. The child is content-addressed within the + // verified index, so the chain of trust holds. imgOpts := append([]remote.Option{}, allOpts...) if platform != "" { p, err := parsePlatform(platform) @@ -246,7 +293,7 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl } imgOpts = append(imgOpts, remote.WithPlatform(p)) } - img, err := remote.Image(parsedRef, imgOpts...) + img, err := remote.Image(fetchRef, imgOpts...) if err != nil { return "", fmt.Errorf("fetching image %q: %w", ref, err) } @@ -254,7 +301,7 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl if err != nil { return "", fmt.Errorf("getting image digest for %q: %w", ref, err) } - if err := l.writeImage(parsedRef, img, consts.KindAnnotationImage, ""); err != nil { + if err := l.writeImage(ctx, parsedRef, img, consts.KindAnnotationImage, ""); err != nil { return "", err } } @@ -291,7 +338,7 @@ func (l *Layout) AddLocalImage(ctx context.Context, ref string) (string, error) return "", fmt.Errorf("getting image digest for %q: %w", ref, err) } - if err := l.writeImage(parsedRef, img, consts.KindAnnotationImage, ""); err != nil { + if err := l.writeImage(ctx, parsedRef, img, consts.KindAnnotationImage, ""); err != nil { return "", err } return d.String(), nil @@ -325,15 +372,31 @@ func ensureDockerHost() error { // writeImageBlobs writes all blobs for a single image (layers, config, manifest) to the store's // blob directory. It does not add an entry to the OCI index. -func (l *Layout) writeImageBlobs(img v1.Image) error { +func (l *Layout) writeImageBlobs(ctx context.Context, img v1.Image) error { layers, err := img.Layers() if err != nil { return fmt.Errorf("getting layers: %w", err) } - var g errgroup.Group + + if stats := imageStatsFromContext(ctx); stats != nil { + var totalBytes int64 + for _, lyr := range layers { + size, err := lyr.Size() + if err != nil { + return fmt.Errorf("getting layer size: %w", err) + } + totalBytes += size + } + stats.Layers.Add(int64(len(layers))) + stats.Bytes.Add(totalBytes) + } + + // See AddArtifact's identical errgroup.WithContext conversion for why this + // can't stay a zero-value errgroup.Group. + g, gctx := errgroup.WithContext(ctx) for _, lyr := range layers { lyr := lyr - g.Go(func() error { return l.writeLayer(lyr) }) + g.Go(func() error { return l.writeLayer(gctx, lyr) }) } if err := g.Wait(); err != nil { return err @@ -343,7 +406,7 @@ func (l *Layout) writeImageBlobs(img v1.Image) error { if err != nil { return fmt.Errorf("getting config: %w", err) } - if err := l.writeBlobData(cfgData); err != nil { + if err := l.writeBlobData(ctx, cfgData); err != nil { return fmt.Errorf("writing config blob: %w", err) } @@ -351,14 +414,14 @@ func (l *Layout) writeImageBlobs(img v1.Image) error { if err != nil { return fmt.Errorf("getting manifest: %w", err) } - return l.writeBlobData(manifestData) + return l.writeBlobData(ctx, manifestData) } // writeImage writes all blobs for img and adds a descriptor entry to the OCI index with the // given annotationRef and kind. containerdName overrides the io.containerd.image.name annotation; // if empty it defaults to annotationRef.Name(). -func (l *Layout) writeImage(annotationRef gname.Reference, img v1.Image, kind string, containerdName string) error { - if err := l.writeImageBlobs(img); err != nil { +func (l *Layout) writeImage(ctx context.Context, annotationRef gname.Reference, img v1.Image, kind string, containerdName string) error { + if err := l.writeImageBlobs(ctx, img); err != nil { return err } @@ -397,7 +460,7 @@ func (l *Layout) writeImage(annotationRef gname.Reference, img v1.Image, kind st // writeIndexBlobs recursively writes all child image blobs for an image index to the store's blob // directory. It does not write the top-level index manifest or add index entries. -func (l *Layout) writeIndexBlobs(idx v1.ImageIndex) error { +func (l *Layout) writeIndexBlobs(ctx context.Context, idx v1.ImageIndex) error { manifest, err := idx.IndexManifest() if err != nil { return fmt.Errorf("getting index manifest: %w", err) @@ -406,14 +469,14 @@ func (l *Layout) writeIndexBlobs(idx v1.ImageIndex) error { for _, childDesc := range manifest.Manifests { // Try as a nested index first, then fall back to a regular image. if childIdx, err := idx.ImageIndex(childDesc.Digest); err == nil { - if err := l.writeIndexBlobs(childIdx); err != nil { + if err := l.writeIndexBlobs(ctx, childIdx); err != nil { return err } raw, err := childIdx.RawManifest() if err != nil { return fmt.Errorf("getting nested index manifest: %w", err) } - if err := l.writeBlobData(raw); err != nil { + if err := l.writeBlobData(ctx, raw); err != nil { return err } } else { @@ -421,7 +484,7 @@ func (l *Layout) writeIndexBlobs(idx v1.ImageIndex) error { if err != nil { return fmt.Errorf("getting child image %v: %w", childDesc.Digest, err) } - if err := l.writeImageBlobs(childImg); err != nil { + if err := l.writeImageBlobs(ctx, childImg); err != nil { return err } } @@ -431,8 +494,8 @@ func (l *Layout) writeIndexBlobs(idx v1.ImageIndex) error { // writeIndex writes all blobs for an image index (including all child platform images) and adds // a descriptor entry to the OCI index with the given annotationRef and kind. -func (l *Layout) writeIndex(annotationRef gname.Reference, idx v1.ImageIndex, kind string) error { - if err := l.writeIndexBlobs(idx); err != nil { +func (l *Layout) writeIndex(ctx context.Context, annotationRef gname.Reference, idx v1.ImageIndex, kind string) error { + if err := l.writeIndexBlobs(ctx, idx); err != nil { return err } @@ -440,7 +503,7 @@ func (l *Layout) writeIndex(annotationRef gname.Reference, idx v1.ImageIndex, ki if err != nil { return fmt.Errorf("getting index manifest: %w", err) } - if err := l.writeBlobData(raw); err != nil { + if err := l.writeBlobData(ctx, raw); err != nil { return fmt.Errorf("writing index manifest blob: %w", err) } @@ -470,11 +533,10 @@ func (l *Layout) writeIndex(annotationRef gname.Reference, idx v1.ImageIndex, ki return l.OCI.AddIndex(desc) } -// saveReferrers discovers and saves OCI 1.1 referrers for the image identified by ref/hash. -// This captures cosign v3 new-bundle-format signatures/attestations stored as OCI referrers -// (via the subject field) rather than the legacy sha256-.sig/.att/.sbom tag convention. -// go-containerregistry handles both the native referrers API and the tag-based fallback. -// Missing referrers and fetch errors are logged at debug level and silently skipped. +// saveReferrers discovers and saves OCI 1.1 referrers for the image identified by ref/hash -- +// cosign v3 new-bundle-format sigs/attestations stored via the subject field, as opposed to the +// legacy sha256-.sig/.att/.sbom tag convention. Missing referrers and fetch errors are +// logged at debug level and silently skipped. func (l *Layout) saveReferrers(ctx context.Context, ref gname.Reference, hash v1.Hash, alreadySaved map[string]bool, opts ...remote.Option) error { log := zerolog.Ctx(ctx) @@ -521,7 +583,7 @@ func (l *Layout) saveReferrers(ctx context.Context, ref gname.Reference, hash v1 // Embed the referrer manifest digest in the kind annotation so that multiple // referrers for the same base image each get a unique entry in the OCI index. kind := consts.KindAnnotationReferrers + "/" + referrerDesc.Digest.Hex - if err := l.writeImage(ref, img, kind, ""); err != nil { + if err := l.writeImage(ctx, ref, img, kind, ""); err != nil { return fmt.Errorf("saving OCI referrer %s for %s: %w", referrerDesc.Digest, ref.Name(), err) } log.Debug().Msgf("saved OCI referrer %s (%s) for %s", referrerDesc.Digest, string(referrerDesc.ArtifactType), ref.Name()) @@ -530,9 +592,8 @@ func (l *Layout) saveReferrers(ctx context.Context, ref gname.Reference, hash v1 } // saveRelatedArtifacts discovers and saves cosign-compatible signature, attestation, and SBOM -// artifacts for the image identified by ref/hash. Missing artifacts are silently skipped. -// Returns the set of manifest digest strings (e.g. "sha256:abc...") that were saved, so that -// saveReferrers can skip duplicates when a registry exposes the same manifest via both paths. +// artifacts for the image identified by ref/hash, skipping missing ones silently. Returns the +// set of saved manifest digest strings so saveReferrers can skip duplicates exposed via both paths. func (l *Layout) saveRelatedArtifacts(ctx context.Context, ref gname.Reference, hash v1.Hash, opts ...remote.Option) (map[string]bool, error) { saved := make(map[string]bool) @@ -558,7 +619,7 @@ func (l *Layout) saveRelatedArtifacts(ctx context.Context, ref gname.Reference, // Artifact doesn't exist at this registry; skip silently. continue } - if err := l.writeImage(ref, img, r.kind, ""); err != nil { + if err := l.writeImage(ctx, ref, img, r.kind, ""); err != nil { return saved, fmt.Errorf("saving %s for %s: %w", r.kind, ref.Name(), err) } if d, err := img.Digest(); err == nil { @@ -809,11 +870,9 @@ func (l *Layout) CopyAll(ctx context.Context, to content.Target, toMapper func(s toRef = tr } - // Append the digest to help the target pusher identify the root descriptor. - // AnnotationRefName for digest-only images already ends in "@sha256:...". - // Strip any existing digest before appending the authoritative descriptor - // digest so the destination pusher can match the root manifest. A double "@" - // yields a digest the pusher never matches, leaving the image unindexed (#642). + // Append the digest so the target pusher can identify the root descriptor. + // AnnotationRefName for digest-only images already ends in "@sha256:...", so + // strip any existing digest first -- a double "@" leaves the image unindexed (#642). if desc.Digest.Validate() == nil { if at := strings.Index(toRef, "@"); at != -1 { toRef = toRef[:at] @@ -855,51 +914,32 @@ func (l *Layout) Identify(ctx context.Context, desc ocispec.Descriptor) string { return m.Config.MediaType } -func (l *Layout) writeBlobData(data []byte) error { +func (l *Layout) writeBlobData(ctx context.Context, data []byte) error { blob := static.NewLayer(data, "") // NOTE: MediaType isn't actually used in the writing - return l.writeLayer(blob) + return l.writeLayer(ctx, blob) } -func (l *Layout) writeLayer(layer v1.Layer) error { +// writeLayer writes a single layer's content to the store's blob directory. +// Writes are atomic (temp file + rename), digest-verified, and deduplicated +// across concurrent callers writing the same digest -- see +// content.OCI.WriteBlob for the implementation. +func (l *Layout) writeLayer(ctx context.Context, layer v1.Layer) error { d, err := layer.Digest() if err != nil { return err } - - dir := filepath.Join(l.Root, ocispec.ImageBlobsDir, d.Algorithm) - if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { - return err - } - - blobPath := filepath.Join(dir, d.Hex) - // Skip entirely if something exists, assume layer is present already - if _, err := os.Stat(blobPath); err == nil { - return nil - } - - r, err := layer.Compressed() + expected, err := digest.Parse(d.String()) if err != nil { return err } - defer r.Close() - - w, err := os.Create(blobPath) + size, err := layer.Size() if err != nil { return err } - _, copyErr := io.Copy(w, r) - if closeErr := w.Close(); closeErr != nil && copyErr == nil { - copyErr = closeErr - } - - // Remove a partially-written or corrupt blob on any failure so retries - // can attempt a fresh download rather than skipping the file. - if copyErr != nil { - os.Remove(blobPath) - } - - return copyErr + return l.OCI.WriteBlob(ctx, expected, size, func() (io.ReadCloser, error) { + return layer.Compressed() + }) } // Remove artifact reference from the store diff --git a/pkg/store/store_blob_concurrency_test.go b/pkg/store/store_blob_concurrency_test.go new file mode 100644 index 0000000..86ced2b --- /dev/null +++ b/pkg/store/store_blob_concurrency_test.go @@ -0,0 +1,115 @@ +package store + +// store_blob_concurrency_test.go covers store.WithBlobConcurrency, the +// plumbing that lets `store sync --concurrency` override the OCI layout's +// default blob-write concurrency ceiling (content.OCI.blobSem) at +// construction time. blobSem itself is unexported on content.OCI, so this +// test proves the override reached it indirectly: by writing several +// distinct-digest layers concurrently through Layout.writeLayer (which is +// unexported here too -- this file is `package store`, matching +// store_concurrency_test.go's whitebox convention) and observing that the +// concurrent in-flight high-water mark never exceeds the configured limit. + +import ( + "bytes" + "context" + "fmt" + "io" + "sync" + "sync/atomic" + "testing" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/opencontainers/go-digest" +) + +// TestWithBlobConcurrency_BoundsConcurrentWrites constructs a Layout with a +// blob concurrency ceiling of 2 and writes 8 distinct-digest layers +// concurrently (distinct so none hit the fast path or dedupe through +// singleflight). The observed concurrent-in-flight high-water mark must +// never exceed 2, which is only true if WithBlobConcurrency's override +// actually reached the underlying OCI's blobSem. +func TestWithBlobConcurrency_BoundsConcurrentWrites(t *testing.T) { + dir := t.TempDir() + s, err := NewLayout(dir, WithBlobConcurrency(2)) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + + const n = 8 + var inFlight int32 + var maxInFlight int32 + + var wg sync.WaitGroup + errs := make([]error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + data := []byte(fmt.Sprintf("distinct blob content #%d", i)) + h, err := v1.NewHash(digest.FromBytes(data).String()) + if err != nil { + errs[i] = err + return + } + lyr := &fakeLayer{ + hash: h, + size: int64(len(data)), + compressed: func() (io.ReadCloser, error) { + cur := atomic.AddInt32(&inFlight, 1) + for { + old := atomic.LoadInt32(&maxInFlight) + if cur <= old { + break + } + if atomic.CompareAndSwapInt32(&maxInFlight, old, cur) { + break + } + } + time.Sleep(30 * time.Millisecond) + atomic.AddInt32(&inFlight, -1) + return io.NopCloser(bytes.NewReader(data)), nil + }, + } + errs[i] = s.writeLayer(context.Background(), lyr) + }() + } + wg.Wait() + + for i, e := range errs { + if e != nil { + t.Errorf("goroutine %d: writeLayer error: %v", i, e) + } + } + + if maxInFlight > 2 { + t.Errorf("max concurrent blob writes = %d, want <= 2 (WithBlobConcurrency(2) not applied)", maxInFlight) + } +} + +// TestNewLayout_WithoutBlobConcurrency_UsesDefault asserts that NewLayout +// without WithBlobConcurrency behaves exactly as before -- a store usable for +// normal operations, no error. +func TestNewLayout_WithoutBlobConcurrency_UsesDefault(t *testing.T) { + dir := t.TempDir() + if _, err := NewLayout(dir); err != nil { + t.Fatalf("NewLayout(dir) with no options: %v", err) + } +} + +// TestNewLayout_WithCache_StillWorksAlongsideBlobConcurrencyRestructure is a +// regression guard: NewLayout was restructured to run the opts loop before +// constructing the underlying OCI store, so WithCache (which only touches +// l.cache, independent of l.OCI) must still apply correctly alongside +// WithBlobConcurrency. +func TestNewLayout_WithCache_StillWorksAlongsideBlobConcurrencyRestructure(t *testing.T) { + dir := t.TempDir() + s, err := NewLayout(dir, WithCache(nil), WithBlobConcurrency(4)) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + if s.OCI == nil { + t.Fatal("NewLayout: s.OCI is nil") + } +} diff --git a/pkg/store/store_concurrency_test.go b/pkg/store/store_concurrency_test.go new file mode 100644 index 0000000..918b381 --- /dev/null +++ b/pkg/store/store_concurrency_test.go @@ -0,0 +1,406 @@ +package store + +// store_concurrency_test.go covers the atomic/verified/deduplicated blob +// write path (writeLayer -> content.OCI.WriteBlob). This file is +// intentionally `package store` (whitebox) rather than `package store_test` +// because writeLayer is unexported. + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "hauler.dev/go/hauler/v2/pkg/content" +) + +// fakeLayer is a hand-rolled v1.Layer that can lie about its digest -- useful +// for the digest-mismatch tests where static.NewLayer (which always computes +// its digest correctly from the given bytes) can't be used. +type fakeLayer struct { + hash v1.Hash + size int64 + compressed func() (io.ReadCloser, error) +} + +func (f *fakeLayer) Digest() (v1.Hash, error) { return f.hash, nil } +func (f *fakeLayer) DiffID() (v1.Hash, error) { return f.hash, nil } +func (f *fakeLayer) Size() (int64, error) { return f.size, nil } +func (f *fakeLayer) MediaType() (types.MediaType, error) { return types.OCILayer, nil } +func (f *fakeLayer) Compressed() (io.ReadCloser, error) { return f.compressed() } +func (f *fakeLayer) Uncompressed() (io.ReadCloser, error) { return f.compressed() } + +var _ v1.Layer = (*fakeLayer)(nil) + +// stallingReader delays its first Read call so that concurrent goroutines +// racing to write the same digest have a chance to actually overlap (join +// the same singleflight flight) before the winner finishes streaming. +type stallingReader struct { + r io.Reader + once sync.Once + delay time.Duration +} + +func (s *stallingReader) Read(p []byte) (int, error) { + s.once.Do(func() { time.Sleep(s.delay) }) + return s.r.Read(p) +} + +func (s *stallingReader) Close() error { return nil } + +func blobPathForTest(root string, alg, hex string) string { + return filepath.Join(root, ocispec.ImageBlobsDir, alg, hex) +} + +func countTmpFilesInStore(t *testing.T, root string) int { + t.Helper() + dir := filepath.Join(root, ocispec.ImageBlobsDir, "sha256") + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("ReadDir %s: %v", dir, err) + } + count := 0 + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp-") { + count++ + } + } + return count +} + +// TestWriteLayer_ConcurrentSameDigest writes the same digest through 16 +// concurrent goroutines using a stalling reader so the goroutines actually +// overlap. It asserts the final blob hashes correctly and that no leftover +// temp files remain. +func TestWriteLayer_ConcurrentSameDigest(t *testing.T) { + dir := t.TempDir() + s, err := NewLayout(dir) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + + data := []byte("layer content shared by many concurrent writers for digest dedup test") + h, err := v1.NewHash(digest.FromBytes(data).String()) + if err != nil { + t.Fatalf("NewHash: %v", err) + } + + var opens int32 + lyr := &fakeLayer{ + hash: h, + size: int64(len(data)), + compressed: func() (io.ReadCloser, error) { + atomic.AddInt32(&opens, 1) + return &stallingReader{r: bytes.NewReader(data), delay: 30 * time.Millisecond}, nil + }, + } + + const n = 16 + start := make(chan struct{}) + var wg sync.WaitGroup + errs := make([]error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + errs[i] = s.writeLayer(context.Background(), lyr) + }(i) + } + close(start) + wg.Wait() + + for i, e := range errs { + if e != nil { + t.Errorf("goroutine %d: writeLayer error: %v", i, e) + } + } + + blobPath := blobPathForTest(dir, h.Algorithm, h.Hex) + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading blob: %v", err) + } + if !bytes.Equal(got, data) { + t.Errorf("blob content = %q, want %q", got, data) + } + + if tmp := countTmpFilesInStore(t, dir); tmp != 0 { + t.Errorf("left %d temp files behind, want 0", tmp) + } +} + +// TestWriteLayer_DigestMismatch uses a layer whose Digest() lies about its +// content. writeLayer must return an error wrapping content.ErrDigestMismatch, +// must not create the final blob path, and must not leave temp files behind. +func TestWriteLayer_DigestMismatch(t *testing.T) { + dir := t.TempDir() + s, err := NewLayout(dir) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + + actual := []byte("this is the actual streamed content") + wrongDigestStr := digest.FromBytes([]byte("this is not the actual content")).String() + h, err := v1.NewHash(wrongDigestStr) + if err != nil { + t.Fatalf("NewHash: %v", err) + } + + lyr := &fakeLayer{ + hash: h, + size: int64(len(actual)), + compressed: func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(actual)), nil + }, + } + + err = s.writeLayer(context.Background(), lyr) + if err == nil { + t.Fatal("writeLayer: expected digest mismatch error, got nil") + } + if !errors.Is(err, content.ErrDigestMismatch) { + t.Errorf("writeLayer error %v does not wrap content.ErrDigestMismatch", err) + } + + blobPath := blobPathForTest(dir, h.Algorithm, h.Hex) + if _, statErr := os.Stat(blobPath); !os.IsNotExist(statErr) { + t.Errorf("final blob path exists after digest mismatch: stat err = %v", statErr) + } + + if tmp := countTmpFilesInStore(t, dir); tmp != 0 { + t.Errorf("left %d temp files behind after digest mismatch, want 0", tmp) + } +} + +// TestWriteLayer_DoesNotDeletePeerBlob directly targets the removed +// store.go:899 `os.Remove(blobPath)` bug: a peer writes digest X +// successfully and commits it to disk; a second, independent write for the +// SAME digest X is then deliberately made to fail (it declares a different +// size, so the fast path can't skip it, and streams content that doesn't +// hash to X). The already-committed blob at X must survive the second +// writer's failure untouched. +func TestWriteLayer_DoesNotDeletePeerBlob(t *testing.T) { + dir := t.TempDir() + s, err := NewLayout(dir) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + + good := []byte("the correct content that a peer successfully commits for digest X") + h, err := v1.NewHash(digest.FromBytes(good).String()) + if err != nil { + t.Fatalf("NewHash: %v", err) + } + + goodLayer := &fakeLayer{ + hash: h, + size: int64(len(good)), + compressed: func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(good)), nil + }, + } + if err := s.writeLayer(context.Background(), goodLayer); err != nil { + t.Fatalf("peer writeLayer (good): unexpected error: %v", err) + } + + // A second, independent layer claims the SAME digest X but streams + // unrelated content of a different length -- this forces the fast path + // (which only compares sizes) to miss and re-enter the write path, where + // the digest check must fail without touching the peer's committed blob. + badContent := []byte("totally different content, different length, will not hash to X") + failingLayer := &fakeLayer{ + hash: h, + size: int64(len(badContent)), + compressed: func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(badContent)), nil + }, + } + + err = s.writeLayer(context.Background(), failingLayer) + if err == nil { + t.Fatal("writeLayer (failing peer): expected error, got nil") + } + if !errors.Is(err, content.ErrDigestMismatch) { + t.Errorf("writeLayer (failing peer) error %v does not wrap content.ErrDigestMismatch", err) + } + + // The peer's originally-committed blob must be untouched. + blobPath := blobPathForTest(dir, h.Algorithm, h.Hex) + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading blob after failing peer write: %v", err) + } + if !bytes.Equal(got, good) { + t.Errorf("peer blob was corrupted/deleted: got %q, want %q", got, good) + } +} + +// TestWriteLayer_TruncatedBlobDetectedAndRewritten pre-places a short/corrupt +// file directly at a blob's final path (bypassing writeLayer entirely, as a +// crash mid-download would leave behind), then calls writeLayer for that same +// digest with the correct known size. The fast path must NOT trust the +// truncated file; it must detect the size mismatch and rewrite the blob +// correctly. +func TestWriteLayer_TruncatedBlobDetectedAndRewritten(t *testing.T) { + dir := t.TempDir() + s, err := NewLayout(dir) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + + data := []byte("this is the correct, full-length layer content for the truncation test") + lyr := static.NewLayer(data, types.OCILayer) + d, err := lyr.Digest() + if err != nil { + t.Fatalf("Digest: %v", err) + } + + blobPath := blobPathForTest(dir, d.Algorithm, d.Hex) + if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(blobPath, []byte("short"), 0644); err != nil { + t.Fatalf("pre-writing truncated blob: %v", err) + } + + if err := s.writeLayer(context.Background(), lyr); err != nil { + t.Fatalf("writeLayer: unexpected error: %v", err) + } + + got, err := os.ReadFile(blobPath) + if err != nil { + t.Fatalf("reading blob: %v", err) + } + if !bytes.Equal(got, data) { + t.Errorf("blob content = %q, want %q (truncated blob was not detected/rewritten)", got, data) + } +} + +// TestWriteImageBlobs_FailureCancelsSiblings is the acceptance test for the +// errgroup.Group -> errgroup.WithContext conversion in writeImageBlobs: on a +// zero-value errgroup.Group, Wait() returns the first error but never +// cancels the group's derived context, so every other layer's writeLayer +// call would run to completion regardless of the failure. It builds one +// image (via mutate.AppendLayers over empty.Image, which lets fakeLayer +// stand in as a v1.Layer without hand-rolling the rest of the v1.Image +// interface) with several slow "good" layers and one layer whose open() +// fails immediately, then asserts that not every good layer's blob actually +// landed on disk -- proof that the group's context was cancelled and +// propagated into content.OCI.WriteBlob before all of them finished. +func TestWriteImageBlobs_FailureCancelsSiblings(t *testing.T) { + dir := t.TempDir() + s, err := NewLayout(dir) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + + const nGood = 8 + var layers []v1.Layer + var goodHashes []v1.Hash + + // Each good layer streams its content in small delayed chunks so that + // the group's context -- cancelled almost immediately by the failing + // layer below -- has many chances to be observed mid-copy, the same + // reasoning as content's slowChunkedReader. + for i := 0; i < nGood; i++ { + data := bytes.Repeat([]byte(fmt.Sprintf("g%d", i)), 200*1024) // ~400KB+, distinct per layer + h, err := v1.NewHash(digest.FromBytes(data).String()) + if err != nil { + t.Fatalf("NewHash: %v", err) + } + goodHashes = append(goodHashes, h) + layers = append(layers, &fakeLayer{ + hash: h, + size: int64(len(data)), + compressed: func() (io.ReadCloser, error) { + return &slowChunkReader{data: data, chunk: 32 * 1024, delay: 5 * time.Millisecond}, nil + }, + }) + } + + // The failing layer's open() errors out immediately -- no delay -- so it + // wins the race to fail and cancel the group's context well before the + // slow good layers finish streaming. + badData := []byte("this layer's open always fails") + badHash, err := v1.NewHash(digest.FromBytes(badData).String()) + if err != nil { + t.Fatalf("NewHash: %v", err) + } + layers = append(layers, &fakeLayer{ + hash: badHash, + size: int64(len(badData)), + compressed: func() (io.ReadCloser, error) { + return nil, errors.New("simulated layer fetch failure") + }, + }) + + img, err := mutate.AppendLayers(empty.Image, layers...) + if err != nil { + t.Fatalf("mutate.AppendLayers: %v", err) + } + + err = s.writeImageBlobs(context.Background(), img) + if err == nil { + t.Fatal("writeImageBlobs: expected an error from the failing layer, got nil") + } + + completed := 0 + for _, h := range goodHashes { + blobPath := blobPathForTest(dir, h.Algorithm, h.Hex) + if _, statErr := os.Stat(blobPath); statErr == nil { + completed++ + } + } + if completed == nGood { + t.Errorf("all %d good layers completed despite a sibling failure -- errgroup did not cancel them (zero-value errgroup.Group regression)", nGood) + } +} + +// slowChunkReader hands out data in small fixed-size chunks with a delay +// before each one, giving a concurrently cancelled context many chances to +// be observed between chunks rather than requiring the whole blob to stream +// before cancellation is noticed. +type slowChunkReader struct { + data []byte + chunk int + delay time.Duration +} + +func (r *slowChunkReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, io.EOF + } + time.Sleep(r.delay) + n := r.chunk + if n > len(p) { + n = len(p) + } + if n > len(r.data) { + n = len(r.data) + } + copy(p, r.data[:n]) + r.data = r.data[n:] + return n, nil +} + +func (r *slowChunkReader) Close() error { return nil } diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 30638d8..2ea23fc 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -464,7 +464,7 @@ func TestCopyDescriptorGraph_Index(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := src.AddImage(ctx, idxTag.Name(), "", false, remoteOpts...); err != nil { + if _, err := src.AddImage(ctx, idxTag.Name(), "", false, "", remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } if err := src.OCI.SaveIndex(); err != nil { @@ -767,7 +767,7 @@ func TestAddImage_OCI11Referrers(t *testing.T) { if err != nil { t.Fatalf("new layout: %v", err) } - if _, err := s.AddImage(context.Background(), baseTag.Name(), "", false, remoteOpts...); err != nil { + if _, err := s.AddImage(context.Background(), baseTag.Name(), "", false, "", remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -787,3 +787,97 @@ func TestAddImage_OCI11Referrers(t *testing.T) { } t.Logf("captured %d OCI referrer(s) for %s", referrerCount, baseTag.Name()) } + +// newTestRegistry starts an in-process registry and returns its host and the +// remote.Option needed to talk to it over plain HTTP. +func newTestRegistry(t *testing.T) (string, []remote.Option) { + t.Helper() + srv := httptest.NewServer(registry.New()) + t.Cleanup(srv.Close) + host := strings.TrimPrefix(srv.URL, "http://") + return host, []remote.Option{remote.WithTransport(srv.Client().Transport)} +} + +// newTestStore creates a fresh OCI layout store rooted in a temp directory. +func newTestStore(t *testing.T) *store.Layout { + t.Helper() + s, err := store.NewLayout(t.TempDir()) + if err != nil { + t.Fatalf("new layout: %v", err) + } + return s +} + +// seedImage pushes a random image to host/repo:tag and returns it, so a test +// can later assert on the exact bytes/digest that were pushed. +func seedImage(t *testing.T, host, repo, tag string, opts ...remote.Option) v1.Image { + t.Helper() + img, err := random.Image(1024, 3) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + ref, err := gname.NewTag(host+"/"+repo+":"+tag, gname.Insecure) + if err != nil { + t.Fatalf("new tag: %v", err) + } + if err := remote.Write(ref, img, opts...); err != nil { + t.Fatalf("remote.Write: %v", err) + } + return img +} + +// TestAddImagePinnedDigestIgnoresMovedTag proves the TOCTOU fix: once a caller +// pins the digest it verified, a tag that moves to different content between +// verification and the fetch cannot substitute its bytes into the store. +func TestAddImagePinnedDigestIgnoresMovedTag(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + original := seedImage(t, host, "test/pinned", "v1", remoteOpts...) + originalDigest, err := original.Digest() + if err != nil { + t.Fatalf("original digest: %v", err) + } + + // Move the tag to different content, exactly as a mutable tag could be + // re-pushed between verification and the pull. + replacement, err := random.Image(1024, 3) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + ref, err := gname.ParseReference(host + "/test/pinned:v1") + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := remote.Write(ref, replacement, remoteOpts...); err != nil { + t.Fatalf("remote.Write replacement: %v", err) + } + + s := newTestStore(t) + got, err := s.AddImage(context.Background(), host+"/test/pinned:v1", "", true, + originalDigest.String(), remoteOpts...) + if err != nil { + t.Fatalf("AddImage: %v", err) + } + if got != originalDigest.String() { + t.Fatalf("stored digest = %s, want the pinned %s (the moved tag won)", got, originalDigest.String()) + } +} + +// TestAddImageEmptyPinResolvesTag confirms the unpinned path is untouched: +// an empty pinnedDigest still resolves the tag normally. +func TestAddImageEmptyPinResolvesTag(t *testing.T) { + host, remoteOpts := newTestRegistry(t) + img := seedImage(t, host, "test/unpinned", "v1", remoteOpts...) + want, err := img.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + + s := newTestStore(t) + got, err := s.AddImage(context.Background(), host+"/test/unpinned:v1", "", true, "", remoteOpts...) + if err != nil { + t.Fatalf("AddImage: %v", err) + } + if got != want.String() { + t.Fatalf("stored digest = %s, want %s", got, want.String()) + } +}