diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5f08c48..b69f4f6 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -251,13 +251,13 @@ jobs: hauler store save --filename store.tar.zst # verify via save with filename and platform (amd64) hauler store save --filename store-amd64.tar.zst --platform linux/amd64 - # verify via save with chunk-size (splits into haul-chunked_0.tar.zst, haul-chunked_1.tar.zst, ...) + # verify via save with chunk-size (splits into haul-chunked.tar.zst.001, haul-chunked.tar.zst.002, ...) hauler store save --filename haul-chunked.tar.zst --chunk-size 50M # verify chunk files exist and original is removed - ls haul-chunked_*.tar.zst + ls haul-chunked.tar.zst.* ! test -f haul-chunked.tar.zst # verify at least two chunks were produced - [ $(ls haul-chunked_*.tar.zst | wc -l) -ge 2 ] + [ $(ls haul-chunked.tar.zst.* | wc -l) -ge 2 ] - name: Remove Hauler Store Contents run: | @@ -279,7 +279,7 @@ jobs: hauler store load --filename store-amd64.tar.zst # verify via load from chunks using explicit first chunk rm -rf store - hauler store load --filename haul-chunked_0.tar.zst + hauler store load --filename haul-chunked.tar.zst.001 hauler store info # verify via load from chunks using base filename (auto-detect) rm -rf store @@ -307,7 +307,7 @@ jobs: - name: Remove Hauler Store Contents run: | - rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst haul-chunked_*.tar.zst + rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst haul-chunked.tar.zst.* hauler store info - name: Verify - hauler store sync diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index 9a912ad..13ba594 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -3,10 +3,13 @@ package store import ( "context" "encoding/json" + "fmt" "io" "net/url" "os" "path/filepath" + "regexp" + "strconv" "strings" "hauler.dev/go/hauler/v2/internal/flags" @@ -20,6 +23,10 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) +// matches the old _NNN. chunk naming of haul_0.tar.zst +// used only to print a hint when a load fails on a file shaped like a stale chunk +var legacyChunkRe = regexp.MustCompile(`_\d+\.`) + // extracts the contents of an archived oci layout to an existing oci layout func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) @@ -36,12 +43,26 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r } defer os.RemoveAll(tempDir) + // kept separate from tempDir since that gets wiped after every haul + // below — remote chunks need to stick around until the whole set lands + stageDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName) + if err != nil { + return err + } + defer os.RemoveAll(stageDir) + + fileNames, remoteOrigin, err := stageRemoteChunks(ctx, o.FileName, stageDir) + if err != nil { + return err + } + l.Debugf("using temporary directory at [%s]", tempDir) - for _, fileName := range o.FileName { + for _, fileName := range fileNames { resolved := resolveHaulPath(fileName) + wasRemote := strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") || remoteOrigin[fileName] l.Infof("loading haul [%s] to [%s]", resolved, o.StoreDir) - err := unarchiveLayoutTo(ctx, resolved, o.StoreDir, tempDir) + err := unarchiveLayoutTo(ctx, resolved, o.StoreDir, tempDir, ro, wasRemote) if err != nil { return err } @@ -51,42 +72,104 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r return nil } +// stageRemoteChunks pre-downloads any remote URL that looks chunk-shaped +// into stageDir before the main load loop starts. That's what makes +// `store load -f url1 -f url2 ...` work for a remote chunk set — they all +// land on disk together, so JoinChunks can just find them once it runs. +// Non-chunk URLs are left alone and go through unarchiveLayoutTo's normal +// single-file download instead. +// +// Only one path per chunk set gets added to the returned list, so the main +// loop doesn't process the same haul twice. remoteOrigin tracks which of +// those returned paths actually came from a download, since the caller +// needs that later to pick the right wording for a failure hint. +func stageRemoteChunks(ctx context.Context, fileNames []string, stageDir string) ([]string, map[string]bool, error) { + added := map[string]bool{} + remoteOrigin := map[string]bool{} + var result []string + + for _, fn := range fileNames { + if !strings.HasPrefix(fn, "http://") && !strings.HasPrefix(fn, "https://") { + result = append(result, fn) + continue + } + + parsedURL, err := url.Parse(fn) + if err != nil { + return nil, nil, err + } + if _, ok := archives.ChunkGroupKey(filepath.Base(parsedURL.Path)); !ok { + result = append(result, fn) + continue + } + + local, err := downloadHaul(ctx, fn, stageDir) + if err != nil { + return nil, nil, err + } + remoteOrigin[local] = true + + key, _ := archives.ChunkGroupKey(filepath.Base(local)) + if !added[key] { + result = append(result, local) + added[key] = true + } + } + + return result, remoteOrigin, nil +} + +// downloadHaul fetches urlStr into destDir, using the server-provided +// filename when available, and returns the local path it was saved to. +func downloadHaul(ctx context.Context, urlStr, destDir string) (string, error) { + h := getter.NewHttp() + parsedURL, err := url.Parse(urlStr) + if err != nil { + return "", err + } + rc, err := h.Open(ctx, parsedURL) + if err != nil { + return "", err + } + defer rc.Close() + + fileName := h.Name(parsedURL) + if fileName == "" { + fileName = filepath.Base(parsedURL.Path) + } + localPath := filepath.Join(destDir, fileName) + + out, err := os.Create(localPath) + if err != nil { + return "", err + } + defer out.Close() + + if _, err = io.Copy(out, rc); err != nil { + return "", err + } + return localPath, nil +} + // accepts an archived OCI layout, extracts the contents to an existing OCI layout, and preserves the index -func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDir string) error { +func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDir string, ro *flags.CliRootOpts, wasRemote bool) error { l := log.FromContext(ctx) if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") { l.Debugf("detected remote archive... starting download... [%s]", haulPath) - - h := getter.NewHttp() - parsedURL, err := url.Parse(haulPath) + local, err := downloadHaul(ctx, haulPath, tempDir) if err != nil { return err } - rc, err := h.Open(ctx, parsedURL) - if err != nil { - return err - } - defer rc.Close() - - fileName := h.Name(parsedURL) - if fileName == "" { - fileName = filepath.Base(parsedURL.Path) - } - haulPath = filepath.Join(tempDir, fileName) - - out, err := os.Create(haulPath) - if err != nil { - return err - } - defer out.Close() - - if _, err = io.Copy(out, rc); err != nil { - return err - } + haulPath = local } - // reassemble chunk files if haulPath matches the chunk naming pattern + // reassemble chunk files if haulPath matches the chunk naming pattern. + // hang onto the pre-join name for the hint below — once joined, even a + // lone unjoinable fragment looks like a normal file (it just gets + // copied to itself), so the hint needs the name we were actually asked + // to load, not whatever JoinChunks renamed it to. + preChunkPath := haulPath joined, err := archives.JoinChunks(ctx, haulPath, tempDir) if err != nil { return err @@ -94,6 +177,19 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi haulPath = joined if err := archives.Unarchive(ctx, haulPath, tempDir); err != nil { + if line1, line2, ok := chunkHint(preChunkPath, wasRemote); ok { + ignoreErrors := ro.IgnoreErrors + if !ignoreErrors && os.Getenv(consts.HaulerIgnoreErrors) == "true" { + ignoreErrors = true + } + if ignoreErrors { + l.Warnf("%s", line1) + l.Warnf("%s", line2) + return nil + } + l.Errorf("%s", line1) + l.Errorf("%s", line2) + } return err } @@ -150,9 +246,41 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi return err } -// resolveHaulPath returns path as-is if it exists or is a URL. If the file is -// not found, it globs for chunk files matching _* in the same -// directory and returns the first match so JoinChunks can reassemble them. +// matches the .NNN chunk suffix that is used to filter resolveHaulPath +// glob matches down to real chunks, so an unrelated sibling file (.sig, .bak, +// etc...) is never picked up instead of the actual first chunk +var chunkSuffixRe = regexp.MustCompile(`\.(\d{3,})$`) + +// chunkHint builds a two-line hint for a load failure on a chunk-shaped +// filename, if one applies. Wording depends on where it came from: a URL +// can't just be renamed, so it points at downloading or listing every chunk +// instead; a local file gets told to rename (old naming) or go find its +// missing siblings (new naming). +func chunkHint(haulPath string, wasRemote bool) (line1, line2 string, ok bool) { + base := filepath.Base(haulPath) + isLegacyShaped := legacyChunkRe.MatchString(base) + isNewShaped := chunkSuffixRe.MatchString(base) + if !isLegacyShaped && !isNewShaped { + return "", "", false + } + + if wasRemote { + return fmt.Sprintf("possibly detected an unjoined remote chunk for haul: [%s]", haulPath), + "specify each chunk with its own --filename flag and try loading it again...", + true + } + if isLegacyShaped { + return fmt.Sprintf("possibly detected an old chunk format for haul: [%s]", haulPath), + "attempt to rename to '..NNN' and try loading it again...", + true + } + return fmt.Sprintf("possibly detected a missing chunk for haul: [%s]", haulPath), + "ensure every chunked haul is present in the same directory and try loading it again...", + true +} + +// resolveHaulPath returns path as-is if it exists or is a URL, otherwise +// globs for chunk files matching .NNN so JoinChunks can reassemble them. func resolveHaulPath(path string) string { if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { return path @@ -160,17 +288,20 @@ func resolveHaulPath(path string) string { if _, err := os.Stat(path); err == nil { return path } - base := path - ext := "" - for filepath.Ext(base) != "" { - ext = filepath.Ext(base) + ext - base = strings.TrimSuffix(base, filepath.Ext(base)) - } - matches, err := filepath.Glob(base + "_*" + ext) - if err != nil || len(matches) == 0 { + matches, err := filepath.Glob(path + ".*") + if err != nil { return path } - return matches[0] + for _, m := range matches { + sub := chunkSuffixRe.FindStringSubmatch(m) + if sub == nil { + continue + } + if idx, err := strconv.Atoi(sub[1]); err == nil && idx != 0 { + return m + } + } + return path } func clearDir(path string) error { diff --git a/cmd/hauler/cli/store/load_test.go b/cmd/hauler/cli/store/load_test.go index 0073743..0cf4df7 100644 --- a/cmd/hauler/cli/store/load_test.go +++ b/cmd/hauler/cli/store/load_test.go @@ -71,7 +71,7 @@ func TestUnarchiveLayoutTo(t *testing.T) { destDir := t.TempDir() tempDir := t.TempDir() - if err := unarchiveLayoutTo(ctx, testHaulArchive, destDir, tempDir); err != nil { + if err := unarchiveLayoutTo(ctx, testHaulArchive, destDir, tempDir, defaultCliOpts(), false); err != nil { t.Fatalf("unarchiveLayoutTo: %v", err) } @@ -262,7 +262,7 @@ func TestUnarchiveLayoutTo_AnnotationBackfill(t *testing.T) { // Step 4: Load the stripped archive. destDir := t.TempDir() tempDir := t.TempDir() - if err := unarchiveLayoutTo(ctx, strippedArchive, destDir, tempDir); err != nil { + if err := unarchiveLayoutTo(ctx, strippedArchive, destDir, tempDir, defaultCliOpts(), false); err != nil { t.Fatalf("unarchiveLayoutTo stripped: %v", err) } @@ -354,7 +354,7 @@ func TestUnarchiveLayoutTo_LegacyKindMigration(t *testing.T) { // Step 4: Load the legacy archive. destDir := t.TempDir() tempDir := t.TempDir() - if err := unarchiveLayoutTo(ctx, legacyArchive, destDir, tempDir); err != nil { + if err := unarchiveLayoutTo(ctx, legacyArchive, destDir, tempDir, defaultCliOpts(), false); err != nil { t.Fatalf("unarchiveLayoutTo legacy: %v", err) } diff --git a/cmd/hauler/cli/store/save.go b/cmd/hauler/cli/store/save.go index 03f0652..57984f9 100644 --- a/cmd/hauler/cli/store/save.go +++ b/cmd/hauler/cli/store/save.go @@ -113,7 +113,7 @@ func parseChunkSize(s string) (int64, error) { if strings.HasSuffix(s, suffix) { n, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64) if err != nil { - return 0, fmt.Errorf("invalid chunk size %q", s) + return 0, fmt.Errorf("invalid chunk size %q: %w", s, err) } result = n * mult matched = true diff --git a/cmd/hauler/cli/store/save_test.go b/cmd/hauler/cli/store/save_test.go index f73809a..37988dc 100644 --- a/cmd/hauler/cli/store/save_test.go +++ b/cmd/hauler/cli/store/save_test.go @@ -325,7 +325,7 @@ func TestSaveCmd_ChunkSize(t *testing.T) { } // at least one chunk must exist - matches, err := filepath.Glob(filepath.Join(archiveDir, "haul-chunked_*.tar.zst")) + matches, err := filepath.Glob(filepath.Join(archiveDir, "haul-chunked.tar.zst.*")) if err != nil { t.Fatalf("glob chunks: %v", err) } diff --git a/pkg/archives/archiver.go b/pkg/archives/archiver.go index 18b57d1..2f5e08e 100644 --- a/pkg/archives/archiver.go +++ b/pkg/archives/archiver.go @@ -6,7 +6,6 @@ import ( "io" "os" "path/filepath" - "strings" "github.com/mholt/archives" "hauler.dev/go/hauler/v2/pkg/log" @@ -105,19 +104,13 @@ func Archive(ctx context.Context, dir, outfile string, compression archives.Comp return nil } -// SplitArchive splits an existing archive into chunks of at most maxBytes each. -// Chunks are named _0, _1, ... where base is the archive -// path with all extensions stripped, and ext is the compound extension (e.g. .tar.zst). -// The original archive is removed after successful splitting. +// splits an existing archive into chunks of at most maxBytes each, named +// .001, .002, ... and removes the original archive afterward. func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]string, error) { l := log.FromContext(ctx) - // derive base path and compound extension by stripping all extensions - base := archivePath - ext := "" - for filepath.Ext(base) != "" { - ext = filepath.Ext(base) + ext - base = strings.TrimSuffix(base, filepath.Ext(base)) + if maxBytes <= 0 { + return nil, fmt.Errorf("maxBytes must be greater than zero, received %d", maxBytes) } f, err := os.Open(archivePath) @@ -127,24 +120,11 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st var chunks []string buf := make([]byte, 32*1024) - chunkIdx := 0 + chunkIdx := 1 var written int64 var outf *os.File for { - if outf == nil { - chunkPath := fmt.Sprintf("%s_%d%s", base, chunkIdx, ext) - outf, err = os.Create(chunkPath) - if err != nil { - f.Close() - return nil, fmt.Errorf("failed to create chunk %d: %w", chunkIdx, err) - } - chunks = append(chunks, chunkPath) - l.Debugf("creating chunk [%s]", chunkPath) - written = 0 - chunkIdx++ - } - remaining := maxBytes - written readSize := int64(len(buf)) if readSize > remaining { @@ -153,6 +133,20 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st n, readErr := f.Read(buf[:readSize]) if n > 0 { + // chunk files are only created once there's real data to write, + // so an archive size that's an exact multiple of maxBytes never + // leaves a trailing empty chunk behind. + if outf == nil { + chunkPath := fmt.Sprintf("%s.%03d", archivePath, chunkIdx) + outf, err = os.Create(chunkPath) + if err != nil { + f.Close() + return nil, fmt.Errorf("failed to create chunk %d: %w", chunkIdx, err) + } + chunks = append(chunks, chunkPath) + l.Debugf("creating chunk [%s]", chunkPath) + chunkIdx++ + } if _, writeErr := outf.Write(buf[:n]); writeErr != nil { outf.Close() f.Close() @@ -162,12 +156,15 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st } if readErr == io.EOF { - outf.Close() - outf = nil + if outf != nil { + outf.Close() + } break } if readErr != nil { - outf.Close() + if outf != nil { + outf.Close() + } f.Close() return nil, fmt.Errorf("failed to read archive: %w", readErr) } @@ -175,6 +172,7 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st if written >= maxBytes { outf.Close() outf = nil + written = 0 } } diff --git a/pkg/archives/archives_test.go b/pkg/archives/archives_test.go index 00c7ce2..a2c7940 100644 --- a/pkg/archives/archives_test.go +++ b/pkg/archives/archives_test.go @@ -174,34 +174,37 @@ func TestChunkInfo(t *testing.T) { name string path string wantBase string - wantExt string wantIndex int wantOk bool }{ { name: "compound extension", - path: "/tmp/haul_3.tar.zst", - wantBase: "/tmp/haul", - wantExt: ".tar.zst", + path: "/tmp/haul.tar.zst.003", + wantBase: "/tmp/haul.tar.zst", wantIndex: 3, wantOk: true, }, { name: "single extension", - path: "/tmp/archive_0.zst", - wantBase: "/tmp/archive", - wantExt: ".zst", - wantIndex: 0, + path: "/tmp/archive.zst.001", + wantBase: "/tmp/archive.zst", + wantIndex: 1, wantOk: true, }, { name: "large index", - path: "/tmp/haul_42.tar.zst", - wantBase: "/tmp/haul", - wantExt: ".tar.zst", + path: "/tmp/haul.tar.zst.042", + wantBase: "/tmp/haul.tar.zst", wantIndex: 42, wantOk: true, }, + { + name: "beyond 3-digit padding", + path: "/tmp/haul.tar.zst.1000", + wantBase: "/tmp/haul.tar.zst", + wantIndex: 1000, + wantOk: true, + }, { name: "no numeric suffix", path: "/tmp/haul.tar.zst", @@ -209,13 +212,18 @@ func TestChunkInfo(t *testing.T) { }, { name: "alphabetic suffix", - path: "/tmp/haul_abc.tar.zst", + path: "/tmp/haul.tar.zst.abc", + wantOk: false, + }, + { + name: "short numeric suffix rejected", + path: "/tmp/report.v1.2", wantOk: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - base, ext, index, ok := chunkInfo(tt.path) + base, index, ok := chunkInfo(tt.path) if ok != tt.wantOk { t.Fatalf("chunkInfo() ok = %v, want %v", ok, tt.wantOk) } @@ -225,9 +233,6 @@ func TestChunkInfo(t *testing.T) { if base != tt.wantBase { t.Errorf("chunkInfo() base = %q, want %q", base, tt.wantBase) } - if ext != tt.wantExt { - t.Errorf("chunkInfo() ext = %q, want %q", ext, tt.wantExt) - } if index != tt.wantIndex { t.Errorf("chunkInfo() index = %d, want %d", index, tt.wantIndex) } @@ -275,9 +280,9 @@ func TestSplitArchive(t *testing.T) { t.Error("original archive should be removed after splitting") } - // chunks must follow _N naming + // chunks must follow .NNN naming (3-digit, 1-based) for i, chunk := range chunks { - expected := filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i)) + expected := filepath.Join(dir, fmt.Sprintf("haul.tar.zst.%03d", i+1)) if chunk != expected { t.Errorf("chunk[%d] = %s, want %s", i, chunk, expected) } @@ -319,12 +324,12 @@ func TestJoinChunks(t *testing.T) { dir := t.TempDir() tempDir := t.TempDir() for i, content := range []string{"chunk0-data", "chunk1-data", "chunk2-data"} { - if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i)), []byte(content), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("haul.tar.zst.%03d", i+1)), []byte(content), 0o644); err != nil { t.Fatal(err) } } - got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir) + got, err := JoinChunks(ctx, filepath.Join(dir, "haul.tar.zst.001"), tempDir) if err != nil { t.Fatalf("JoinChunks() error = %v", err) } @@ -341,13 +346,13 @@ func TestJoinChunks(t *testing.T) { dir := t.TempDir() tempDir := t.TempDir() for i, content := range []string{"aaa", "bbb"} { - if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("data_%d.tar.zst", i)), []byte(content), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("data.tar.zst.%03d", i+1)), []byte(content), 0o644); err != nil { t.Fatal(err) } } - // pass chunk_1, not chunk_0 — should still assemble from chunk_0 - got, err := JoinChunks(ctx, filepath.Join(dir, "data_1.tar.zst"), tempDir) + // pass chunk .002, not .001... should still assemble from .001 + got, err := JoinChunks(ctx, filepath.Join(dir, "data.tar.zst.002"), tempDir) if err != nil { t.Fatalf("JoinChunks() error = %v", err) } @@ -378,15 +383,15 @@ func TestJoinChunks(t *testing.T) { t.Run("non-numeric suffix files excluded", func(t *testing.T) { dir := t.TempDir() tempDir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "haul_0.tar.zst"), []byte("valid"), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(dir, "haul.tar.zst.001"), []byte("valid"), 0o644); err != nil { t.Fatal(err) } // glob matches this but chunkInfo rejects it - if err := os.WriteFile(filepath.Join(dir, "haul_foo.tar.zst"), []byte("invalid"), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(dir, "haul.tar.zst.foo"), []byte("invalid"), 0o644); err != nil { t.Fatal(err) } - got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir) + got, err := JoinChunks(ctx, filepath.Join(dir, "haul.tar.zst.001"), tempDir) if err != nil { t.Fatalf("JoinChunks() error = %v", err) } diff --git a/pkg/archives/unarchiver.go b/pkg/archives/unarchiver.go index 180a71c..d3e0c2f 100644 --- a/pkg/archives/unarchiver.go +++ b/pkg/archives/unarchiver.go @@ -160,66 +160,130 @@ func Unarchive(ctx context.Context, tarball, dst string) error { return nil } -var chunkSuffixRe = regexp.MustCompile(`^(.+)_(\d+)$`) +var chunkSuffixRe = regexp.MustCompile(`^(.+)\.(\d{3,})$`) -// chunkInfo checks whether archivePath matches the chunk naming pattern (_N). -// Returns the base path (without index), compound extension, numeric index, and whether it matched. -func chunkInfo(archivePath string) (base, ext string, index int, ok bool) { +// checks whether archivePath matches the .NNN chunk naming pattern +func chunkInfo(archivePath string) (base string, index int, ok bool) { dir := filepath.Dir(archivePath) name := filepath.Base(archivePath) - // strip compound extension (e.g. .tar.zst) - nameBase := name - nameExt := "" - for filepath.Ext(nameBase) != "" { - nameExt = filepath.Ext(nameBase) + nameExt - nameBase = strings.TrimSuffix(nameBase, filepath.Ext(nameBase)) + m := chunkSuffixRe.FindStringSubmatch(name) + if m == nil { + return "", 0, false } - m := chunkSuffixRe.FindStringSubmatch(nameBase) + idx, err := strconv.Atoi(m[2]) + if err != nil || idx == 0 { + return "", 0, false + } + return filepath.Join(dir, m[1]), idx, true +} + +// the safe subset of the old _N naming (e.g. haul_0.tar.zst): +// dot-free base, 0-based index with no leading zero, simple extension. A +// dotted base like a version string ("airgapped-docs_0.1.8...") is exactly +// what caused the original false-positive bug, so it's never matched here — +// only names where the split can't be ambiguous get legacy support back. +var legacyChunkSuffixRe = regexp.MustCompile(`^([^./]+)_(0|[1-9]\d*)\.([A-Za-z0-9]+(?:\.[A-Za-z0-9]+)?)$`) + +// checks whether archivePath matches the unambiguous legacy chunk pattern. +// See legacyChunkSuffixRe for exactly what is and isn't recognized. +func legacyChunkInfo(archivePath string) (base, ext string, index int, ok bool) { + dir := filepath.Dir(archivePath) + name := filepath.Base(archivePath) + + m := legacyChunkSuffixRe.FindStringSubmatch(name) if m == nil { return "", "", 0, false } - idx, _ := strconv.Atoi(m[2]) - return filepath.Join(dir, m[1]), nameExt, idx, true + idx, err := strconv.Atoi(m[2]) + if err != nil { + return "", "", 0, false + } + return filepath.Join(dir, m[1]), "." + m[3], idx, true } -// JoinChunks detects whether archivePath is a chunk file and, if so, finds all -// sibling chunks, concatenates them in numeric order into a single file in tempDir, -// and returns the path to the joined file. If archivePath is not a chunk, it is -// returned unchanged. +// ChunkGroupKey checks name against both chunk schemes (current .NNN and the +// safe legacy subset) and, if it matches either, returns a key that's the +// same for every chunk in that set. Same detection JoinChunks uses under the +// hood, so callers grouping filenames (e.g. remote URLs) never disagree with +// what JoinChunks will actually recognize later. +func ChunkGroupKey(name string) (key string, ok bool) { + if base, _, ok := chunkInfo(name); ok { + return base, true + } + if base, ext, _, ok := legacyChunkInfo(name); ok { + return base + ext, true + } + return "", false +} + +// JoinChunks detects whether archivePath is a chunk file and, if so, finds all sibling +// chunks, concatenates them in numeric order into a single file in tempDir, and +// returns the path to the joined file. If archivePath is not a chunk, it is unchanged. +// +// Both the current .NNN scheme and the unambiguous subset of the old +// _N scheme are recognized (see legacyChunkInfo), so valid +// pre-v2.1 chunk sets that were never at risk of misdetection don't need to +// be manually renamed. func JoinChunks(ctx context.Context, archivePath, tempDir string) (string, error) { - l := log.FromContext(ctx) - - base, ext, _, ok := chunkInfo(archivePath) - if !ok { - return archivePath, nil - } - - all, err := filepath.Glob(base + "_*" + ext) - if err != nil { - return archivePath, nil - } - var matches []string - for _, m := range all { - if _, _, _, ok := chunkInfo(m); ok { - matches = append(matches, m) + if base, _, ok := chunkInfo(archivePath); ok { + all, err := filepath.Glob(base + ".*") + if err != nil { + return archivePath, nil } - } - if len(matches) == 0 { - return archivePath, nil + var matches []string + for _, m := range all { + // the glob is a string-prefix match, so it can also catch siblings + // like .old.001 whose own base differs from ours + if mBase, _, ok := chunkInfo(m); ok && mBase == base { + matches = append(matches, m) + } + } + if len(matches) == 0 { + return archivePath, nil + } + sort.Slice(matches, func(i, j int) bool { + _, idxI, _ := chunkInfo(matches[i]) + _, idxJ, _ := chunkInfo(matches[j]) + return idxI < idxJ + }) + return joinFiles(ctx, matches, tempDir, filepath.Base(base)) } - sort.Slice(matches, func(i, j int) bool { - _, _, idxI, _ := chunkInfo(matches[i]) - _, _, idxJ, _ := chunkInfo(matches[j]) - return idxI < idxJ - }) + if base, ext, _, ok := legacyChunkInfo(archivePath); ok { + all, err := filepath.Glob(base + "_*" + ext) + if err != nil { + return archivePath, nil + } + var matches []string + for _, m := range all { + // same prefix-collision guard as above, applied to the legacy shape + if mBase, mExt, _, ok := legacyChunkInfo(m); ok && mBase == base && mExt == ext { + matches = append(matches, m) + } + } + if len(matches) == 0 { + return archivePath, nil + } + sort.Slice(matches, func(i, j int) bool { + _, _, idxI, _ := legacyChunkInfo(matches[i]) + _, _, idxJ, _ := legacyChunkInfo(matches[j]) + return idxI < idxJ + }) + return joinFiles(ctx, matches, tempDir, filepath.Base(base)+ext) + } - l.Debugf("joining %d chunk(s) for [%s]", len(matches), base) + return archivePath, nil +} - joinedPath := filepath.Join(tempDir, filepath.Base(base)+ext) +// concatenates matches in order into a new file named joinedName in tempDir. +func joinFiles(ctx context.Context, matches []string, tempDir, joinedName string) (string, error) { + l := log.FromContext(ctx) + l.Debugf("joining %d chunk(s) into [%s]", len(matches), joinedName) + + joinedPath := filepath.Join(tempDir, joinedName) outf, err := os.Create(joinedPath) if err != nil { return "", fmt.Errorf("failed to create joined archive: %w", err) @@ -239,6 +303,6 @@ func JoinChunks(ctx context.Context, archivePath, tempDir string) (string, error cf.Close() } - l.Infof("joined %d chunk(s) into [%s]", len(matches), filepath.Base(joinedPath)) + l.Infof("joined %d chunk(s) into [%s]", len(matches), joinedName) return joinedPath, nil }