fixed bugs with chunking and added support for remote chunks (#702)

.github/workflows/tests.yaml

@@ -251,13 +251,13 @@ jobs:
           hauler store save --filename store.tar.zst
           # verify via save with filename and platform (amd64)
           hauler store save --filename store-amd64.tar.zst --platform linux/amd64
-          # verify via save with chunk-size (splits into haul-chunked_0.tar.zst, haul-chunked_1.tar.zst, ...)
+          # verify via save with chunk-size (splits into haul-chunked.tar.zst.001, haul-chunked.tar.zst.002, ...)
           hauler store save --filename haul-chunked.tar.zst --chunk-size 50M
           # verify chunk files exist and original is removed
-          ls haul-chunked_*.tar.zst
+          ls haul-chunked.tar.zst.*
           ! test -f haul-chunked.tar.zst
           # verify at least two chunks were produced
-          [ $(ls haul-chunked_*.tar.zst | wc -l) -ge 2 ]
+          [ $(ls haul-chunked.tar.zst.* | wc -l) -ge 2 ]
 
       - name: Remove Hauler Store Contents
         run: |
@@ -279,7 +279,7 @@ jobs:
           hauler store load --filename store-amd64.tar.zst
           # verify via load from chunks using explicit first chunk
           rm -rf store
-          hauler store load --filename haul-chunked_0.tar.zst
+          hauler store load --filename haul-chunked.tar.zst.001
           hauler store info
           # verify via load from chunks using base filename (auto-detect)
           rm -rf store
@@ -307,7 +307,7 @@ jobs:
 
       - name: Remove Hauler Store Contents
         run: |
-          rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst haul-chunked_*.tar.zst
+          rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst haul-chunked.tar.zst.*
           hauler store info
 
       - name: Verify - hauler store sync

cmd/hauler/cli/store/load.go

@@ -3,10 +3,13 @@ package store
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strconv"
 	"strings"
 
 	"hauler.dev/go/hauler/v2/internal/flags"
@@ -20,6 +23,10 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+// matches the old <base>_NNN.<ext> chunk naming of haul_0.tar.zst
+// used only to print a hint when a load fails on a file shaped like a stale chunk
+var legacyChunkRe = regexp.MustCompile(`_\d+\.`)
+
 // extracts the contents of an archived oci layout to an existing oci layout
 func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
 	l := log.FromContext(ctx)
@@ -36,12 +43,26 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r
 	}
 	defer os.RemoveAll(tempDir)
 
+	// kept separate from tempDir since that gets wiped after every haul
+	// below — remote chunks need to stick around until the whole set lands
+	stageDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(stageDir)
+
+	fileNames, remoteOrigin, err := stageRemoteChunks(ctx, o.FileName, stageDir)
+	if err != nil {
+		return err
+	}
+
 	l.Debugf("using temporary directory at [%s]", tempDir)
 
-	for _, fileName := range o.FileName {
+	for _, fileName := range fileNames {
 		resolved := resolveHaulPath(fileName)
+		wasRemote := strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") || remoteOrigin[fileName]
 		l.Infof("loading haul [%s] to [%s]", resolved, o.StoreDir)
-		err := unarchiveLayoutTo(ctx, resolved, o.StoreDir, tempDir)
+		err := unarchiveLayoutTo(ctx, resolved, o.StoreDir, tempDir, ro, wasRemote)
 		if err != nil {
 			return err
 		}
@@ -51,42 +72,104 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r
 	return nil
 }
 
+// stageRemoteChunks pre-downloads any remote URL that looks chunk-shaped
+// into stageDir before the main load loop starts. That's what makes
+// `store load -f url1 -f url2 ...` work for a remote chunk set — they all
+// land on disk together, so JoinChunks can just find them once it runs.
+// Non-chunk URLs are left alone and go through unarchiveLayoutTo's normal
+// single-file download instead.
+//
+// Only one path per chunk set gets added to the returned list, so the main
+// loop doesn't process the same haul twice. remoteOrigin tracks which of
+// those returned paths actually came from a download, since the caller
+// needs that later to pick the right wording for a failure hint.
+func stageRemoteChunks(ctx context.Context, fileNames []string, stageDir string) ([]string, map[string]bool, error) {
+	added := map[string]bool{}
+	remoteOrigin := map[string]bool{}
+	var result []string
+
+	for _, fn := range fileNames {
+		if !strings.HasPrefix(fn, "http://") && !strings.HasPrefix(fn, "https://") {
+			result = append(result, fn)
+			continue
+		}
+
+		parsedURL, err := url.Parse(fn)
+		if err != nil {
+			return nil, nil, err
+		}
+		if _, ok := archives.ChunkGroupKey(filepath.Base(parsedURL.Path)); !ok {
+			result = append(result, fn)
+			continue
+		}
+
+		local, err := downloadHaul(ctx, fn, stageDir)
+		if err != nil {
+			return nil, nil, err
+		}
+		remoteOrigin[local] = true
+
+		key, _ := archives.ChunkGroupKey(filepath.Base(local))
+		if !added[key] {
+			result = append(result, local)
+			added[key] = true
+		}
+	}
+
+	return result, remoteOrigin, nil
+}
+
+// downloadHaul fetches urlStr into destDir, using the server-provided
+// filename when available, and returns the local path it was saved to.
+func downloadHaul(ctx context.Context, urlStr, destDir string) (string, error) {
+	h := getter.NewHttp()
+	parsedURL, err := url.Parse(urlStr)
+	if err != nil {
+		return "", err
+	}
+	rc, err := h.Open(ctx, parsedURL)
+	if err != nil {
+		return "", err
+	}
+	defer rc.Close()
+
+	fileName := h.Name(parsedURL)
+	if fileName == "" {
+		fileName = filepath.Base(parsedURL.Path)
+	}
+	localPath := filepath.Join(destDir, fileName)
+
+	out, err := os.Create(localPath)
+	if err != nil {
+		return "", err
+	}
+	defer out.Close()
+
+	if _, err = io.Copy(out, rc); err != nil {
+		return "", err
+	}
+	return localPath, nil
+}
+
 // accepts an archived OCI layout, extracts the contents to an existing OCI layout, and preserves the index
-func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDir string) error {
+func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDir string, ro *flags.CliRootOpts, wasRemote bool) error {
 	l := log.FromContext(ctx)
 
 	if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
 		l.Debugf("detected remote archive... starting download... [%s]", haulPath)
-
-		h := getter.NewHttp()
-		parsedURL, err := url.Parse(haulPath)
+		local, err := downloadHaul(ctx, haulPath, tempDir)
 		if err != nil {
 			return err
 		}
-		rc, err := h.Open(ctx, parsedURL)
-		if err != nil {
-			return err
-		}
-		defer rc.Close()
-
-		fileName := h.Name(parsedURL)
-		if fileName == "" {
-			fileName = filepath.Base(parsedURL.Path)
-		}
-		haulPath = filepath.Join(tempDir, fileName)
-
-		out, err := os.Create(haulPath)
-		if err != nil {
-			return err
-		}
-		defer out.Close()
-
-		if _, err = io.Copy(out, rc); err != nil {
-			return err
-		}
+		haulPath = local
 	}
 
-	// reassemble chunk files if haulPath matches the chunk naming pattern
+	// reassemble chunk files if haulPath matches the chunk naming pattern.
+	// hang onto the pre-join name for the hint below — once joined, even a
+	// lone unjoinable fragment looks like a normal file (it just gets
+	// copied to itself), so the hint needs the name we were actually asked
+	// to load, not whatever JoinChunks renamed it to.
+	preChunkPath := haulPath
 	joined, err := archives.JoinChunks(ctx, haulPath, tempDir)
 	if err != nil {
 		return err
@@ -94,6 +177,19 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
 	haulPath = joined
 
 	if err := archives.Unarchive(ctx, haulPath, tempDir); err != nil {
+		if line1, line2, ok := chunkHint(preChunkPath, wasRemote); ok {
+			ignoreErrors := ro.IgnoreErrors
+			if !ignoreErrors && os.Getenv(consts.HaulerIgnoreErrors) == "true" {
+				ignoreErrors = true
+			}
+			if ignoreErrors {
+				l.Warnf("%s", line1)
+				l.Warnf("%s", line2)
+				return nil
+			}
+			l.Errorf("%s", line1)
+			l.Errorf("%s", line2)
+		}
 		return err
 	}
 
@@ -150,9 +246,41 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
 	return err
 }
 
-// resolveHaulPath returns path as-is if it exists or is a URL. If the file is
-// not found, it globs for chunk files matching <base>_*<ext> in the same
-// directory and returns the first match so JoinChunks can reassemble them.
+// matches the <path>.NNN chunk suffix that is used to filter resolveHaulPath
+// glob matches down to real chunks, so an unrelated sibling file (.sig, .bak,
+// etc...) is never picked up instead of the actual first chunk
+var chunkSuffixRe = regexp.MustCompile(`\.(\d{3,})$`)
+
+// chunkHint builds a two-line hint for a load failure on a chunk-shaped
+// filename, if one applies. Wording depends on where it came from: a URL
+// can't just be renamed, so it points at downloading or listing every chunk
+// instead; a local file gets told to rename (old naming) or go find its
+// missing siblings (new naming).
+func chunkHint(haulPath string, wasRemote bool) (line1, line2 string, ok bool) {
+	base := filepath.Base(haulPath)
+	isLegacyShaped := legacyChunkRe.MatchString(base)
+	isNewShaped := chunkSuffixRe.MatchString(base)
+	if !isLegacyShaped && !isNewShaped {
+		return "", "", false
+	}
+
+	if wasRemote {
+		return fmt.Sprintf("possibly detected an unjoined remote chunk for haul: [%s]", haulPath),
+			"specify each chunk with its own --filename flag and try loading it again...",
+			true
+	}
+	if isLegacyShaped {
+		return fmt.Sprintf("possibly detected an old chunk format for haul: [%s]", haulPath),
+			"attempt to rename to '<base>.<ext>.NNN' and try loading it again...",
+			true
+	}
+	return fmt.Sprintf("possibly detected a missing chunk for haul: [%s]", haulPath),
+		"ensure every chunked haul is present in the same directory and try loading it again...",
+		true
+}
+
+// resolveHaulPath returns path as-is if it exists or is a URL, otherwise
+// globs for chunk files matching <path>.NNN so JoinChunks can reassemble them.
 func resolveHaulPath(path string) string {
 	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
 		return path
@@ -160,17 +288,20 @@ func resolveHaulPath(path string) string {
 	if _, err := os.Stat(path); err == nil {
 		return path
 	}
-	base := path
-	ext := ""
-	for filepath.Ext(base) != "" {
-		ext = filepath.Ext(base) + ext
-		base = strings.TrimSuffix(base, filepath.Ext(base))
-	}
-	matches, err := filepath.Glob(base + "_*" + ext)
-	if err != nil || len(matches) == 0 {
+	matches, err := filepath.Glob(path + ".*")
+	if err != nil {
 		return path
 	}
-	return matches[0]
+	for _, m := range matches {
+		sub := chunkSuffixRe.FindStringSubmatch(m)
+		if sub == nil {
+			continue
+		}
+		if idx, err := strconv.Atoi(sub[1]); err == nil && idx != 0 {
+			return m
+		}
+	}
+	return path
 }
 
 func clearDir(path string) error {

cmd/hauler/cli/store/load_test.go

@@ -71,7 +71,7 @@ func TestUnarchiveLayoutTo(t *testing.T) {
 	destDir := t.TempDir()
 	tempDir := t.TempDir()
 
-	if err := unarchiveLayoutTo(ctx, testHaulArchive, destDir, tempDir); err != nil {
+	if err := unarchiveLayoutTo(ctx, testHaulArchive, destDir, tempDir, defaultCliOpts(), false); err != nil {
 		t.Fatalf("unarchiveLayoutTo: %v", err)
 	}
 
@@ -262,7 +262,7 @@ func TestUnarchiveLayoutTo_AnnotationBackfill(t *testing.T) {
 	// Step 4: Load the stripped archive.
 	destDir := t.TempDir()
 	tempDir := t.TempDir()
-	if err := unarchiveLayoutTo(ctx, strippedArchive, destDir, tempDir); err != nil {
+	if err := unarchiveLayoutTo(ctx, strippedArchive, destDir, tempDir, defaultCliOpts(), false); err != nil {
 		t.Fatalf("unarchiveLayoutTo stripped: %v", err)
 	}
 
@@ -354,7 +354,7 @@ func TestUnarchiveLayoutTo_LegacyKindMigration(t *testing.T) {
 	// Step 4: Load the legacy archive.
 	destDir := t.TempDir()
 	tempDir := t.TempDir()
-	if err := unarchiveLayoutTo(ctx, legacyArchive, destDir, tempDir); err != nil {
+	if err := unarchiveLayoutTo(ctx, legacyArchive, destDir, tempDir, defaultCliOpts(), false); err != nil {
 		t.Fatalf("unarchiveLayoutTo legacy: %v", err)
 	}
 

cmd/hauler/cli/store/save.go

@@ -113,7 +113,7 @@ func parseChunkSize(s string) (int64, error) {
 		if strings.HasSuffix(s, suffix) {
 			n, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64)
 			if err != nil {
-				return 0, fmt.Errorf("invalid chunk size %q", s)
+				return 0, fmt.Errorf("invalid chunk size %q: %w", s, err)
 			}
 			result = n * mult
 			matched = true

cmd/hauler/cli/store/save_test.go

@@ -325,7 +325,7 @@ func TestSaveCmd_ChunkSize(t *testing.T) {
 	}
 
 	// at least one chunk must exist
-	matches, err := filepath.Glob(filepath.Join(archiveDir, "haul-chunked_*.tar.zst"))
+	matches, err := filepath.Glob(filepath.Join(archiveDir, "haul-chunked.tar.zst.*"))
 	if err != nil {
 		t.Fatalf("glob chunks: %v", err)
 	}

pkg/archives/archiver.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/mholt/archives"
 	"hauler.dev/go/hauler/v2/pkg/log"
@@ -105,19 +104,13 @@ func Archive(ctx context.Context, dir, outfile string, compression archives.Comp
 	return nil
 }
 
-// SplitArchive splits an existing archive into chunks of at most maxBytes each.
-// Chunks are named <base>_0<ext>, <base>_1<ext>, ... where base is the archive
-// path with all extensions stripped, and ext is the compound extension (e.g. .tar.zst).
-// The original archive is removed after successful splitting.
+// splits an existing archive into chunks of at most maxBytes each, named
+// <archivePath>.001, .002, ... and removes the original archive afterward.
 func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]string, error) {
 	l := log.FromContext(ctx)
 
-	// derive base path and compound extension by stripping all extensions
-	base := archivePath
-	ext := ""
-	for filepath.Ext(base) != "" {
-		ext = filepath.Ext(base) + ext
-		base = strings.TrimSuffix(base, filepath.Ext(base))
+	if maxBytes <= 0 {
+		return nil, fmt.Errorf("maxBytes must be greater than zero, received %d", maxBytes)
 	}
 
 	f, err := os.Open(archivePath)
@@ -127,24 +120,11 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st
 
 	var chunks []string
 	buf := make([]byte, 32*1024)
-	chunkIdx := 0
+	chunkIdx := 1
 	var written int64
 	var outf *os.File
 
 	for {
-		if outf == nil {
-			chunkPath := fmt.Sprintf("%s_%d%s", base, chunkIdx, ext)
-			outf, err = os.Create(chunkPath)
-			if err != nil {
-				f.Close()
-				return nil, fmt.Errorf("failed to create chunk %d: %w", chunkIdx, err)
-			}
-			chunks = append(chunks, chunkPath)
-			l.Debugf("creating chunk [%s]", chunkPath)
-			written = 0
-			chunkIdx++
-		}
-
 		remaining := maxBytes - written
 		readSize := int64(len(buf))
 		if readSize > remaining {
@@ -153,6 +133,20 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st
 
 		n, readErr := f.Read(buf[:readSize])
 		if n > 0 {
+			// chunk files are only created once there's real data to write,
+			// so an archive size that's an exact multiple of maxBytes never
+			// leaves a trailing empty chunk behind.
+			if outf == nil {
+				chunkPath := fmt.Sprintf("%s.%03d", archivePath, chunkIdx)
+				outf, err = os.Create(chunkPath)
+				if err != nil {
+					f.Close()
+					return nil, fmt.Errorf("failed to create chunk %d: %w", chunkIdx, err)
+				}
+				chunks = append(chunks, chunkPath)
+				l.Debugf("creating chunk [%s]", chunkPath)
+				chunkIdx++
+			}
 			if _, writeErr := outf.Write(buf[:n]); writeErr != nil {
 				outf.Close()
 				f.Close()
@@ -162,12 +156,15 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st
 		}
 
 		if readErr == io.EOF {
-			outf.Close()
-			outf = nil
+			if outf != nil {
+				outf.Close()
+			}
 			break
 		}
 		if readErr != nil {
-			outf.Close()
+			if outf != nil {
+				outf.Close()
+			}
 			f.Close()
 			return nil, fmt.Errorf("failed to read archive: %w", readErr)
 		}
@@ -175,6 +172,7 @@ func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]st
 		if written >= maxBytes {
 			outf.Close()
 			outf = nil
+			written = 0
 		}
 	}
 

pkg/archives/archives_test.go

@@ -174,34 +174,37 @@ func TestChunkInfo(t *testing.T) {
 		name      string
 		path      string
 		wantBase  string
-		wantExt   string
 		wantIndex int
 		wantOk    bool
 	}{
 		{
 			name:      "compound extension",
-			path:      "/tmp/haul_3.tar.zst",
-			wantBase:  "/tmp/haul",
-			wantExt:   ".tar.zst",
+			path:      "/tmp/haul.tar.zst.003",
+			wantBase:  "/tmp/haul.tar.zst",
 			wantIndex: 3,
 			wantOk:    true,
 		},
 		{
 			name:      "single extension",
-			path:      "/tmp/archive_0.zst",
-			wantBase:  "/tmp/archive",
-			wantExt:   ".zst",
-			wantIndex: 0,
+			path:      "/tmp/archive.zst.001",
+			wantBase:  "/tmp/archive.zst",
+			wantIndex: 1,
 			wantOk:    true,
 		},
 		{
 			name:      "large index",
-			path:      "/tmp/haul_42.tar.zst",
-			wantBase:  "/tmp/haul",
-			wantExt:   ".tar.zst",
+			path:      "/tmp/haul.tar.zst.042",
+			wantBase:  "/tmp/haul.tar.zst",
 			wantIndex: 42,
 			wantOk:    true,
 		},
+		{
+			name:      "beyond 3-digit padding",
+			path:      "/tmp/haul.tar.zst.1000",
+			wantBase:  "/tmp/haul.tar.zst",
+			wantIndex: 1000,
+			wantOk:    true,
+		},
 		{
 			name:   "no numeric suffix",
 			path:   "/tmp/haul.tar.zst",
@@ -209,13 +212,18 @@ func TestChunkInfo(t *testing.T) {
 		},
 		{
 			name:   "alphabetic suffix",
-			path:   "/tmp/haul_abc.tar.zst",
+			path:   "/tmp/haul.tar.zst.abc",
+			wantOk: false,
+		},
+		{
+			name:   "short numeric suffix rejected",
+			path:   "/tmp/report.v1.2",
 			wantOk: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			base, ext, index, ok := chunkInfo(tt.path)
+			base, index, ok := chunkInfo(tt.path)
 			if ok != tt.wantOk {
 				t.Fatalf("chunkInfo() ok = %v, want %v", ok, tt.wantOk)
 			}
@@ -225,9 +233,6 @@ func TestChunkInfo(t *testing.T) {
 			if base != tt.wantBase {
 				t.Errorf("chunkInfo() base = %q, want %q", base, tt.wantBase)
 			}
-			if ext != tt.wantExt {
-				t.Errorf("chunkInfo() ext = %q, want %q", ext, tt.wantExt)
-			}
 			if index != tt.wantIndex {
 				t.Errorf("chunkInfo() index = %d, want %d", index, tt.wantIndex)
 			}
@@ -275,9 +280,9 @@ func TestSplitArchive(t *testing.T) {
 				t.Error("original archive should be removed after splitting")
 			}
 
-			// chunks must follow <base>_N<ext> naming
+			// chunks must follow <archivePath>.NNN naming (3-digit, 1-based)
 			for i, chunk := range chunks {
-				expected := filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i))
+				expected := filepath.Join(dir, fmt.Sprintf("haul.tar.zst.%03d", i+1))
 				if chunk != expected {
 					t.Errorf("chunk[%d] = %s, want %s", i, chunk, expected)
 				}
@@ -319,12 +324,12 @@ func TestJoinChunks(t *testing.T) {
 		dir := t.TempDir()
 		tempDir := t.TempDir()
 		for i, content := range []string{"chunk0-data", "chunk1-data", "chunk2-data"} {
-			if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i)), []byte(content), 0o644); err != nil {
+			if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("haul.tar.zst.%03d", i+1)), []byte(content), 0o644); err != nil {
 				t.Fatal(err)
 			}
 		}
 
-		got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir)
+		got, err := JoinChunks(ctx, filepath.Join(dir, "haul.tar.zst.001"), tempDir)
 		if err != nil {
 			t.Fatalf("JoinChunks() error = %v", err)
 		}
@@ -341,13 +346,13 @@ func TestJoinChunks(t *testing.T) {
 		dir := t.TempDir()
 		tempDir := t.TempDir()
 		for i, content := range []string{"aaa", "bbb"} {
-			if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("data_%d.tar.zst", i)), []byte(content), 0o644); err != nil {
+			if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("data.tar.zst.%03d", i+1)), []byte(content), 0o644); err != nil {
 				t.Fatal(err)
 			}
 		}
 
-		// pass chunk_1, not chunk_0 — should still assemble from chunk_0
-		got, err := JoinChunks(ctx, filepath.Join(dir, "data_1.tar.zst"), tempDir)
+		// pass chunk .002, not .001... should still assemble from .001
+		got, err := JoinChunks(ctx, filepath.Join(dir, "data.tar.zst.002"), tempDir)
 		if err != nil {
 			t.Fatalf("JoinChunks() error = %v", err)
 		}
@@ -378,15 +383,15 @@ func TestJoinChunks(t *testing.T) {
 	t.Run("non-numeric suffix files excluded", func(t *testing.T) {
 		dir := t.TempDir()
 		tempDir := t.TempDir()
-		if err := os.WriteFile(filepath.Join(dir, "haul_0.tar.zst"), []byte("valid"), 0o644); err != nil {
+		if err := os.WriteFile(filepath.Join(dir, "haul.tar.zst.001"), []byte("valid"), 0o644); err != nil {
 			t.Fatal(err)
 		}
 		// glob matches this but chunkInfo rejects it
-		if err := os.WriteFile(filepath.Join(dir, "haul_foo.tar.zst"), []byte("invalid"), 0o644); err != nil {
+		if err := os.WriteFile(filepath.Join(dir, "haul.tar.zst.foo"), []byte("invalid"), 0o644); err != nil {
 			t.Fatal(err)
 		}
 
-		got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir)
+		got, err := JoinChunks(ctx, filepath.Join(dir, "haul.tar.zst.001"), tempDir)
 		if err != nil {
 			t.Fatalf("JoinChunks() error = %v", err)
 		}

pkg/archives/unarchiver.go

@@ -160,66 +160,130 @@ func Unarchive(ctx context.Context, tarball, dst string) error {
 	return nil
 }
 
-var chunkSuffixRe = regexp.MustCompile(`^(.+)_(\d+)$`)
+var chunkSuffixRe = regexp.MustCompile(`^(.+)\.(\d{3,})$`)
 
-// chunkInfo checks whether archivePath matches the chunk naming pattern (<base>_N<ext>).
-// Returns the base path (without index), compound extension, numeric index, and whether it matched.
-func chunkInfo(archivePath string) (base, ext string, index int, ok bool) {
+// checks whether archivePath matches the <name>.NNN chunk naming pattern
+func chunkInfo(archivePath string) (base string, index int, ok bool) {
 	dir := filepath.Dir(archivePath)
 	name := filepath.Base(archivePath)
 
-	// strip compound extension (e.g. .tar.zst)
-	nameBase := name
-	nameExt := ""
-	for filepath.Ext(nameBase) != "" {
-		nameExt = filepath.Ext(nameBase) + nameExt
-		nameBase = strings.TrimSuffix(nameBase, filepath.Ext(nameBase))
+	m := chunkSuffixRe.FindStringSubmatch(name)
+	if m == nil {
+		return "", 0, false
 	}
 
-	m := chunkSuffixRe.FindStringSubmatch(nameBase)
+	idx, err := strconv.Atoi(m[2])
+	if err != nil || idx == 0 {
+		return "", 0, false
+	}
+	return filepath.Join(dir, m[1]), idx, true
+}
+
+// the safe subset of the old <base>_N<ext> naming (e.g. haul_0.tar.zst):
+// dot-free base, 0-based index with no leading zero, simple extension. A
+// dotted base like a version string ("airgapped-docs_0.1.8...") is exactly
+// what caused the original false-positive bug, so it's never matched here —
+// only names where the split can't be ambiguous get legacy support back.
+var legacyChunkSuffixRe = regexp.MustCompile(`^([^./]+)_(0|[1-9]\d*)\.([A-Za-z0-9]+(?:\.[A-Za-z0-9]+)?)$`)
+
+// checks whether archivePath matches the unambiguous legacy chunk pattern.
+// See legacyChunkSuffixRe for exactly what is and isn't recognized.
+func legacyChunkInfo(archivePath string) (base, ext string, index int, ok bool) {
+	dir := filepath.Dir(archivePath)
+	name := filepath.Base(archivePath)
+
+	m := legacyChunkSuffixRe.FindStringSubmatch(name)
 	if m == nil {
 		return "", "", 0, false
 	}
 
-	idx, _ := strconv.Atoi(m[2])
-	return filepath.Join(dir, m[1]), nameExt, idx, true
+	idx, err := strconv.Atoi(m[2])
+	if err != nil {
+		return "", "", 0, false
+	}
+	return filepath.Join(dir, m[1]), "." + m[3], idx, true
 }
 
-// JoinChunks detects whether archivePath is a chunk file and, if so, finds all
-// sibling chunks, concatenates them in numeric order into a single file in tempDir,
-// and returns the path to the joined file. If archivePath is not a chunk, it is
-// returned unchanged.
+// ChunkGroupKey checks name against both chunk schemes (current .NNN and the
+// safe legacy subset) and, if it matches either, returns a key that's the
+// same for every chunk in that set. Same detection JoinChunks uses under the
+// hood, so callers grouping filenames (e.g. remote URLs) never disagree with
+// what JoinChunks will actually recognize later.
+func ChunkGroupKey(name string) (key string, ok bool) {
+	if base, _, ok := chunkInfo(name); ok {
+		return base, true
+	}
+	if base, ext, _, ok := legacyChunkInfo(name); ok {
+		return base + ext, true
+	}
+	return "", false
+}
+
+// JoinChunks detects whether archivePath is a chunk file and, if so, finds all sibling
+// chunks, concatenates them in numeric order into a single file in tempDir, and
+// returns the path to the joined file. If archivePath is not a chunk, it is unchanged.
+//
+// Both the current <name>.NNN scheme and the unambiguous subset of the old
+// <base>_N<ext> scheme are recognized (see legacyChunkInfo), so valid
+// pre-v2.1 chunk sets that were never at risk of misdetection don't need to
+// be manually renamed.
 func JoinChunks(ctx context.Context, archivePath, tempDir string) (string, error) {
-	l := log.FromContext(ctx)
-
-	base, ext, _, ok := chunkInfo(archivePath)
-	if !ok {
-		return archivePath, nil
-	}
-
-	all, err := filepath.Glob(base + "_*" + ext)
-	if err != nil {
-		return archivePath, nil
-	}
-	var matches []string
-	for _, m := range all {
-		if _, _, _, ok := chunkInfo(m); ok {
-			matches = append(matches, m)
+	if base, _, ok := chunkInfo(archivePath); ok {
+		all, err := filepath.Glob(base + ".*")
+		if err != nil {
+			return archivePath, nil
 		}
-	}
-	if len(matches) == 0 {
-		return archivePath, nil
+		var matches []string
+		for _, m := range all {
+			// the glob is a string-prefix match, so it can also catch siblings
+			// like <base>.old.001 whose own base differs from ours
+			if mBase, _, ok := chunkInfo(m); ok && mBase == base {
+				matches = append(matches, m)
+			}
+		}
+		if len(matches) == 0 {
+			return archivePath, nil
+		}
+		sort.Slice(matches, func(i, j int) bool {
+			_, idxI, _ := chunkInfo(matches[i])
+			_, idxJ, _ := chunkInfo(matches[j])
+			return idxI < idxJ
+		})
+		return joinFiles(ctx, matches, tempDir, filepath.Base(base))
 	}
 
-	sort.Slice(matches, func(i, j int) bool {
-		_, _, idxI, _ := chunkInfo(matches[i])
-		_, _, idxJ, _ := chunkInfo(matches[j])
-		return idxI < idxJ
-	})
+	if base, ext, _, ok := legacyChunkInfo(archivePath); ok {
+		all, err := filepath.Glob(base + "_*" + ext)
+		if err != nil {
+			return archivePath, nil
+		}
+		var matches []string
+		for _, m := range all {
+			// same prefix-collision guard as above, applied to the legacy shape
+			if mBase, mExt, _, ok := legacyChunkInfo(m); ok && mBase == base && mExt == ext {
+				matches = append(matches, m)
+			}
+		}
+		if len(matches) == 0 {
+			return archivePath, nil
+		}
+		sort.Slice(matches, func(i, j int) bool {
+			_, _, idxI, _ := legacyChunkInfo(matches[i])
+			_, _, idxJ, _ := legacyChunkInfo(matches[j])
+			return idxI < idxJ
+		})
+		return joinFiles(ctx, matches, tempDir, filepath.Base(base)+ext)
+	}
 
-	l.Debugf("joining %d chunk(s) for [%s]", len(matches), base)
+	return archivePath, nil
+}
 
-	joinedPath := filepath.Join(tempDir, filepath.Base(base)+ext)
+// concatenates matches in order into a new file named joinedName in tempDir.
+func joinFiles(ctx context.Context, matches []string, tempDir, joinedName string) (string, error) {
+	l := log.FromContext(ctx)
+	l.Debugf("joining %d chunk(s) into [%s]", len(matches), joinedName)
+
+	joinedPath := filepath.Join(tempDir, joinedName)
 	outf, err := os.Create(joinedPath)
 	if err != nil {
 		return "", fmt.Errorf("failed to create joined archive: %w", err)
@@ -239,6 +303,6 @@ func JoinChunks(ctx context.Context, archivePath, tempDir string) (string, error
 		cf.Close()
 	}
 
-	l.Infof("joined %d chunk(s) into [%s]", len(matches), filepath.Base(joinedPath))
+	l.Infof("joined %d chunk(s) into [%s]", len(matches), joinedName)
 	return joinedPath, nil
 }