Compare commits

...

9 Commits

Author SHA1 Message Date
dependabot[bot]
93938d1acb bump github.com/go-jose/go-jose/v4 (#542)
bumps the go_modules group with 1 update in the / directory: [github.com/go-jose/go-jose/v4](https://github.com/go-jose/go-jose).


updates `github.com/go-jose/go-jose/v4` from 4.1.3 to 4.1.4

- [Release notes](https://github.com/go-jose/go-jose/releases)
- [Commits](https://github.com/go-jose/go-jose/compare/v4.1.3...v4.1.4)

---

updated-dependencies:
- dependency-name: github.com/go-jose/go-jose/v4
  dependency-version: 4.1.4
  dependency-type: indirect
  dependency-group: go_modules

...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-04-03 08:41:54 -04:00
Camryn Carter
641b9db8fd chunk the haul (#519)
* chunk the haul
* validate numeric suffix on join
* enforce valid chunk size
* containerd warning
* updated test.go files
2026-03-25 10:59:51 -04:00
Camryn Carter
c68e72df1c option to sync images.txt files natively (#538)
* sync images.txt files
* test worklflow sync w image list
* images.txt
2026-03-25 10:53:54 -04:00
Camryn Carter
e8046a1e30 fix keep registry logic (#537)
* fixed keep registry logic
* trim library/
* updated test
* test updates
2026-03-25 10:45:48 -04:00
Camryn Carter
7fd03ea52e images.txt testdata file (#539) 2026-03-24 13:27:05 -04:00
Camryn Carter
aa9b883d4c add cherry-pick workflow for release branches (#533)
this workflow automates cherry-picking changes from merged pull requests to specified release branches based on comments... it handles permission checks, version parsing, and conflict resolution during the cherry-pick process.

Signed-off-by: Camryn Carter <camryn.carter@ranchergovernment.com>
2026-03-18 23:26:13 -04:00
dependabot[bot]
565b27d54b bump google.golang.org/grpc in the go_modules group across 1 directory (#536)
bumps the go_modules group with 1 update in the / directory: [google.golang.org/grpc](https://github.com/grpc/grpc-go).

updates `google.golang.org/grpc` from 1.78.0 to 1.79.3
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.78.0...v1.79.3)

---

updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-version: 1.79.3
  dependency-type: indirect
  dependency-group: go_modules

...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-03-18 23:25:22 -04:00
Adam Martin
3adb9257b7 adjust hauler's kind annotation to not reflect cosign (#535)
Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com>
2026-03-18 23:24:47 -04:00
Adam Martin
268485f6d6 fix dockerhub default host bug (#534)
Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com>
2026-03-18 23:24:05 -04:00
25 changed files with 1812 additions and 58 deletions

298
.github/workflows/cherrypick.yml vendored Normal file
View File

@@ -0,0 +1,298 @@
name: Cherry-pick to release branch
on:
issue_comment:
types: [created]
pull_request:
types: [closed]
permissions:
contents: write
pull-requests: write
issues: write
jobs:
# ──────────────────────────────────────────────────────────────
# Trigger 1: /cherrypick-X.Y comment on a PR
# - If already merged → run cherry-pick immediately
# - If not yet merged → add label, cherry-pick will run on merge
# ──────────────────────────────────────────────────────────────
handle-comment:
if: >
github.event_name == 'issue_comment' &&
github.event.issue.pull_request &&
startsWith(github.event.comment.body, '/cherrypick-')
runs-on: ubuntu-latest
steps:
- name: Check commenter permissions
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMENTER: ${{ github.event.comment.user.login }}
run: |
PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/${COMMENTER}/permission \
--jq '.permission')
echo "Permission level for $COMMENTER: $PERMISSION"
if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" && "$PERMISSION" != "write" ]]; then
echo "::warning::User $COMMENTER does not have write access, ignoring cherry-pick request"
exit 1
fi
- name: Parse version from comment
id: parse
env:
COMMENT_BODY: ${{ github.event.comment.body }}
run: |
VERSION=$(echo "$COMMENT_BODY" | head -1 | grep -oP '(?<=/cherrypick-)\d+\.\d+')
if [ -z "$VERSION" ]; then
echo "::error::Could not parse version from comment"
exit 1
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "target_branch=release/$VERSION" >> "$GITHUB_OUTPUT"
echo "label=cherrypick/$VERSION" >> "$GITHUB_OUTPUT"
- name: React to comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
-f content='+1'
- name: Check if PR is merged
id: check
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }})
MERGED=$(echo "$PR_JSON" | jq -r '.merged')
echo "merged=$MERGED" >> "$GITHUB_OUTPUT"
echo "pr_title=$(echo "$PR_JSON" | jq -r '.title')" >> "$GITHUB_OUTPUT"
echo "base_sha=$(echo "$PR_JSON" | jq -r '.base.sha')" >> "$GITHUB_OUTPUT"
echo "head_sha=$(echo "$PR_JSON" | jq -r '.head.sha')" >> "$GITHUB_OUTPUT"
- name: Add cherry-pick label
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LABEL: ${{ steps.parse.outputs.label }}
PR_NUMBER: ${{ github.event.issue.number }}
run: |
gh api repos/${{ github.repository }}/labels \
-f name="$LABEL" -f color="fbca04" -f description="Queued for cherry-pick" 2>/dev/null || true
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/labels \
-f "labels[]=$LABEL"
- name: Notify if queued (not yet merged)
if: steps.check.outputs.merged != 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LABEL: ${{ steps.parse.outputs.label }}
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
PR_NUMBER: ${{ github.event.issue.number }}
run: |
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
-f body="🏷️ Labeled \`$LABEL\` — backport to \`$TARGET_BRANCH\` will be created automatically when this PR is merged."
- name: Checkout repository
if: steps.check.outputs.merged == 'true'
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Verify target branch exists
if: steps.check.outputs.merged == 'true'
env:
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.issue.number }}
run: |
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null 2>&1; then
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
-f body="❌ Cannot cherry-pick: branch \`$TARGET_BRANCH\` does not exist."
exit 1
fi
- name: Apply PR diff and push
if: steps.check.outputs.merged == 'true'
id: apply
env:
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
PR_NUMBER: ${{ github.event.issue.number }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
BACKPORT_BRANCH="backport/${PR_NUMBER}-to-${TARGET_BRANCH//\//-}"
echo "backport_branch=$BACKPORT_BRANCH" >> "$GITHUB_OUTPUT"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git checkout "$TARGET_BRANCH"
git checkout -b "$BACKPORT_BRANCH"
# Download the PR's patch from GitHub (pure diff of the PR's changes)
gh api repos/${{ github.repository }}/pulls/${PR_NUMBER} \
-H "Accept: application/vnd.github.v3.patch" > /tmp/pr.patch
# Apply the patch
HAS_CONFLICTS="false"
CONFLICTED_FILES=""
if git apply --check /tmp/pr.patch 2>/dev/null; then
# Clean apply
git apply /tmp/pr.patch
git add -A
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}"
elif git apply --3way /tmp/pr.patch; then
# Applied with 3-way merge (auto-resolved)
git add -A
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}" || true
else
# Has real conflicts — apply what we can
HAS_CONFLICTS="true"
CONFLICTED_FILES=$(git diff --name-only --diff-filter=U | tr '\n' ',' | sed 's/,$//')
# Take the incoming version for conflicted files
git diff --name-only --diff-filter=U | while read -r file; do
git checkout --theirs -- "$file"
done
git add -A
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH} (conflicts)" || true
fi
echo "has_conflicts=$HAS_CONFLICTS" >> "$GITHUB_OUTPUT"
echo "conflicted_files=$CONFLICTED_FILES" >> "$GITHUB_OUTPUT"
git push origin "$BACKPORT_BRANCH"
- name: Create backport PR
if: steps.check.outputs.merged == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
VERSION: ${{ steps.parse.outputs.version }}
PR_TITLE: ${{ steps.check.outputs.pr_title }}
PR_NUMBER: ${{ github.event.issue.number }}
BACKPORT_BRANCH: ${{ steps.apply.outputs.backport_branch }}
run: |
TITLE="[${VERSION}] ${PR_TITLE}"
BODY="Backport of #${PR_NUMBER} to \`${TARGET_BRANCH}\`."
PR_URL=$(gh pr create \
--base "$TARGET_BRANCH" \
--head "$BACKPORT_BRANCH" \
--title "$TITLE" \
--body "$BODY")
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
-f body="✅ Backport PR created: ${PR_URL}"
# ──────────────────────────────────────────────────────────────
# Trigger 2: PR merged → process any queued cherrypick/* labels
# ──────────────────────────────────────────────────────────────
handle-merge:
if: >
github.event_name == 'pull_request' &&
github.event.pull_request.merged == true
runs-on: ubuntu-latest
steps:
- name: Collect cherry-pick labels
id: labels
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
LABELS=$(gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/labels \
--jq '[.[] | select(.name | startswith("cherrypick/")) | .name] | join(",")')
if [ -z "$LABELS" ]; then
echo "No cherrypick labels found, nothing to do."
echo "has_labels=false" >> "$GITHUB_OUTPUT"
else
echo "Found labels: $LABELS"
echo "has_labels=true" >> "$GITHUB_OUTPUT"
echo "labels=$LABELS" >> "$GITHUB_OUTPUT"
fi
- name: Checkout repository
if: steps.labels.outputs.has_labels == 'true'
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download PR patch
if: steps.labels.outputs.has_labels == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
gh api repos/${{ github.repository }}/pulls/${PR_NUMBER} \
-H "Accept: application/vnd.github.v3.patch" > /tmp/pr.patch
- name: Process each cherry-pick label
if: steps.labels.outputs.has_labels == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LABELS: ${{ steps.labels.outputs.labels }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_TITLE: ${{ github.event.pull_request.title }}
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
IFS=',' read -ra LABEL_ARRAY <<< "$LABELS"
for LABEL in "${LABEL_ARRAY[@]}"; do
VERSION="${LABEL#cherrypick/}"
TARGET_BRANCH="release/$VERSION"
echo "=== Processing backport to $TARGET_BRANCH ==="
# Verify target branch exists
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null 2>&1; then
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
-f body="❌ Cannot cherry-pick to \`$TARGET_BRANCH\`: branch does not exist."
continue
fi
BACKPORT_BRANCH="backport/${PR_NUMBER}-to-${TARGET_BRANCH//\//-}"
git checkout "$TARGET_BRANCH"
git checkout -b "$BACKPORT_BRANCH"
# Apply the patch
HAS_CONFLICTS="false"
CONFLICTED_FILES=""
if git apply --check /tmp/pr.patch 2>/dev/null; then
git apply /tmp/pr.patch
git add -A
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}"
elif git apply --3way /tmp/pr.patch; then
git add -A
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}" || true
else
HAS_CONFLICTS="true"
CONFLICTED_FILES=$(git diff --name-only --diff-filter=U | tr '\n' ',' | sed 's/,$//')
git diff --name-only --diff-filter=U | while read -r file; do
git checkout --theirs -- "$file"
done
git add -A
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH} (conflicts)" || true
fi
git push origin "$BACKPORT_BRANCH"
# Build PR title and body
TITLE="[${VERSION}] ${PR_TITLE}"
BODY="Backport of #${PR_NUMBER} to \`${TARGET_BRANCH}\`."
PR_URL=$(gh pr create \
--base "$TARGET_BRANCH" \
--head "$BACKPORT_BRANCH" \
--title "$TITLE" \
--body "$BODY")
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
-f body="✅ Backport PR to \`$TARGET_BRANCH\` created: ${PR_URL}"
# Clean up for next iteration
git checkout "$TARGET_BRANCH"
git branch -D "$BACKPORT_BRANCH"
done

View File

@@ -250,6 +250,13 @@ jobs:
hauler store save --filename store.tar.zst
# verify via save with filename and platform (amd64)
hauler store save --filename store-amd64.tar.zst --platform linux/amd64
# verify via save with chunk-size (splits into haul-chunked_0.tar.zst, haul-chunked_1.tar.zst, ...)
hauler store save --filename haul-chunked.tar.zst --chunk-size 50M
# verify chunk files exist and original is removed
ls haul-chunked_*.tar.zst
! test -f haul-chunked.tar.zst
# verify at least two chunks were produced
[ $(ls haul-chunked_*.tar.zst | wc -l) -ge 2 ]
- name: Remove Hauler Store Contents
run: |
@@ -269,6 +276,14 @@ jobs:
hauler store load --filename store.tar.zst --tempdir /opt
# verify via load with filename and platform (amd64)
hauler store load --filename store-amd64.tar.zst
# verify via load from chunks using explicit first chunk
rm -rf store
hauler store load --filename haul-chunked_0.tar.zst
hauler store info
# verify via load from chunks using base filename (auto-detect)
rm -rf store
hauler store load --filename haul-chunked.tar.zst
hauler store info
- name: Verify Hauler Store Contents
run: |
@@ -291,7 +306,7 @@ jobs:
- name: Remove Hauler Store Contents
run: |
rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst
rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst haul-chunked_*.tar.zst
hauler store info
- name: Verify - hauler store sync
@@ -303,6 +318,17 @@ jobs:
hauler store sync --filename testdata/hauler-manifest-pipeline.yaml --filename testdata/hauler-manifest.yaml
# need more tests here
- name: Verify - hauler store sync (image list)
run: |
# verify via local image list file
hauler store sync --image-txt testdata/images.txt
# verify via multiple image list files
hauler store sync --image-txt testdata/images.txt --image-txt testdata/images.txt
# verify via remote image list file
hauler store sync --image-txt https://raw.githubusercontent.com/hauler-dev/hauler/main/testdata/images.txt
# confirm images are present in the store
hauler store info | grep 'busybox'
- name: Verify - hauler store serve
run: |
hauler store serve --help

View File

@@ -133,6 +133,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
}
if rewrite != "" {
rawRewrite := rewrite
rewrite = strings.TrimPrefix(rewrite, "/")
if !strings.Contains(rewrite, ":") {
if tag, ok := r.(name.Tag); ok {
@@ -146,7 +147,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
if err != nil {
return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err)
}
if err := rewriteReference(ctx, s, r, newRef); err != nil {
if err := rewriteReference(ctx, s, r, newRef, rawRewrite); err != nil {
return err
}
}
@@ -155,7 +156,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
return nil
}
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference) error {
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference, rawRewrite string) error {
l := log.FromContext(ctx)
if err := s.OCI.LoadIndex(); err != nil {
@@ -184,8 +185,9 @@ func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Referenc
newRegistry := newRefContext.RegistryStr()
// If user omitted a registry in the rewrite string, go-containerregistry defaults to
// index.docker.io. Preserve the original registry when the source is non-docker.
if newRegistry == "index.docker.io" && oldRegistry != "index.docker.io" {
if newRegistry == "index.docker.io" && !strings.HasPrefix(rawRewrite, "docker.io") && !strings.HasPrefix(rawRewrite, "index.docker.io") {
newRegistry = oldRegistry
newRepo = strings.TrimPrefix(newRepo, "library/") //if rewrite has library/ prefix in path it is stripped off unless registry specified in rewrite
}
oldTotal := oldRepo + ":" + oldTag
newTotal := newRepo + ":" + newTag

View File

@@ -12,6 +12,7 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/registry"
"github.com/google/go-containerregistry/pkg/v1/remote"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
helmchart "helm.sh/helm/v3/pkg/chart"
"hauler.dev/go/hauler/internal/flags"
@@ -248,7 +249,9 @@ func TestRewriteReference(t *testing.T) {
t.Fatalf("parse newRef: %v", err)
}
if err := rewriteReference(ctx, s, oldRef, newRef); err != nil {
rawRewrite := newRef.String()
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
@@ -259,8 +262,9 @@ func TestRewriteReference(t *testing.T) {
s := newTestStore(t)
oldRef, _ := name.NewTag("docker.io/missing/repo:v1")
newRef, _ := name.NewTag("docker.io/new/repo:v2")
rawRewrite := newRef.String()
err := rewriteReference(ctx, s, oldRef, newRef)
err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite)
if err == nil {
t.Fatal("expected error, got nil")
}
@@ -268,6 +272,86 @@ func TestRewriteReference(t *testing.T) {
t.Errorf("expected 'could not find' in error, got: %v", err)
}
})
// Tests for the registry-preservation / library/-stripping logic (lines 188-191).
// go-containerregistry normalises bare single-name Docker Hub refs (e.g. "nginx:latest")
// to "index.docker.io/library/nginx:latest". When the rewrite string omits a registry,
// rewriteReference must (a) preserve the source registry and (b) strip the injected
// "library/" prefix so that the stored ref looks like "nginx:v2", not "library/nginx:v2".
t.Run("path-only rewrite strips library/ prefix from docker hub official image", func(t *testing.T) {
s := newTestStore(t)
seedStoreDescriptor(t, s, map[string]string{
ocispec.AnnotationRefName: "library/nginx:latest",
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
})
oldRef, _ := name.NewTag("nginx:latest") // → index.docker.io/library/nginx:latest
newRef, _ := name.NewTag("nginx:v2") // → index.docker.io/library/nginx:v2
rawRewrite := "nginx:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// library/ must be stripped; registry stays index.docker.io
assertAnnotationsInStore(t, s, "nginx:v2", "index.docker.io/nginx:v2")
})
t.Run("explicit docker.io rewrite preserves library/ prefix", func(t *testing.T) {
s := newTestStore(t)
seedStoreDescriptor(t, s, map[string]string{
ocispec.AnnotationRefName: "library/nginx:latest",
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
})
oldRef, _ := name.NewTag("nginx:latest")
newRef, _ := name.NewTag("docker.io/nginx:v2") // → index.docker.io/library/nginx:v2
rawRewrite := "docker.io/nginx:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// rawRewrite starts with "docker.io" → condition must NOT fire → library/ preserved
assertAnnotationsInStore(t, s, "library/nginx:v2", "index.docker.io/library/nginx:v2")
})
t.Run("explicit index.docker.io rewrite preserves library/ prefix", func(t *testing.T) {
s := newTestStore(t)
seedStoreDescriptor(t, s, map[string]string{
ocispec.AnnotationRefName: "library/nginx:latest",
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
})
oldRef, _ := name.NewTag("nginx:latest")
newRef, _ := name.NewTag("index.docker.io/nginx:v2") // → index.docker.io/library/nginx:v2
rawRewrite := "index.docker.io/nginx:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// rawRewrite starts with "index.docker.io" → condition must NOT fire → library/ preserved
assertAnnotationsInStore(t, s, "library/nginx:v2", "index.docker.io/library/nginx:v2")
})
t.Run("non-docker source with path-only rewrite preserves original registry", func(t *testing.T) {
host, rOpts := newTestRegistry(t)
seedImage(t, host, "src/repo", "v1", rOpts...)
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/src/repo:v1", "", rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
oldRef, _ := name.NewTag(host+"/src/repo:v1", name.Insecure)
newRef, _ := name.NewTag("newrepo/img:v2") // defaults to index.docker.io
rawRewrite := "newrepo/img:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// condition fires → registry reverts to host, no library/ to strip
assertAnnotationsInStore(t, s, "newrepo/img:v2", host+"/newrepo/img:v2")
})
}
// --------------------------------------------------------------------------

View File

@@ -39,8 +39,9 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r
l.Debugf("using temporary directory at [%s]", tempDir)
for _, fileName := range o.FileName {
l.Infof("loading haul [%s] to [%s]", fileName, o.StoreDir)
err := unarchiveLayoutTo(ctx, fileName, o.StoreDir, tempDir)
resolved := resolveHaulPath(fileName)
l.Infof("loading haul [%s] to [%s]", resolved, o.StoreDir)
err := unarchiveLayoutTo(ctx, resolved, o.StoreDir, tempDir)
if err != nil {
return err
}
@@ -85,6 +86,13 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
}
}
// reassemble chunk files if haulPath matches the chunk naming pattern
joined, err := archives.JoinChunks(ctx, haulPath, tempDir)
if err != nil {
return err
}
haulPath = joined
if err := archives.Unarchive(ctx, haulPath, tempDir); err != nil {
return err
}
@@ -107,6 +115,9 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
if _, exists := idx.Manifests[i].Annotations[consts.KindAnnotationName]; !exists {
idx.Manifests[i].Annotations[consts.KindAnnotationName] = consts.KindAnnotationImage
}
// Translate legacy dev.cosignproject.cosign values to dev.hauler equivalents.
kind := idx.Manifests[i].Annotations[consts.KindAnnotationName]
idx.Manifests[i].Annotations[consts.KindAnnotationName] = consts.NormalizeLegacyKind(kind)
if ref, ok := idx.Manifests[i].Annotations[consts.ContainerdImageNameKey]; ok {
if slash := strings.Index(ref, "/"); slash != -1 {
ref = ref[slash+1:]
@@ -139,6 +150,29 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
return err
}
// resolveHaulPath returns path as-is if it exists or is a URL. If the file is
// not found, it globs for chunk files matching <base>_*<ext> in the same
// directory and returns the first match so JoinChunks can reassemble them.
func resolveHaulPath(path string) string {
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
return path
}
if _, err := os.Stat(path); err == nil {
return path
}
base := path
ext := ""
for filepath.Ext(base) != "" {
ext = filepath.Ext(base) + ext
base = strings.TrimSuffix(base, filepath.Ext(base))
}
matches, err := filepath.Glob(base + "_*" + ext)
if err != nil || len(matches) == 0 {
return path
}
return matches[0]
}
func clearDir(path string) error {
entries, err := os.ReadDir(path)
if err != nil {

View File

@@ -287,6 +287,99 @@ func TestUnarchiveLayoutTo_AnnotationBackfill(t *testing.T) {
}
}
// --------------------------------------------------------------------------
// TestUnarchiveLayoutTo_LegacyKindMigration
// --------------------------------------------------------------------------
// TestUnarchiveLayoutTo_LegacyKindMigration crafts a haul archive whose
// index.json contains old dev.cosignproject.cosign kind values, then verifies
// that unarchiveLayoutTo translates them to dev.hauler equivalents.
func TestUnarchiveLayoutTo_LegacyKindMigration(t *testing.T) {
ctx := newTestContext(t)
// Step 1: Extract the real test archive to obtain a valid OCI layout on disk.
extractDir := t.TempDir()
if err := archives.Unarchive(ctx, testHaulArchive, extractDir); err != nil {
t.Fatalf("Unarchive: %v", err)
}
// Step 2: Read index.json and inject old dev.cosignproject.cosign kind values.
indexPath := filepath.Join(extractDir, "index.json")
data, err := os.ReadFile(indexPath)
if err != nil {
t.Fatalf("read index.json: %v", err)
}
var idx ocispec.Index
if err := json.Unmarshal(data, &idx); err != nil {
t.Fatalf("unmarshal index.json: %v", err)
}
if len(idx.Manifests) == 0 {
t.Skip("testdata/haul.tar.zst has no top-level manifests — cannot test legacy kind migration")
}
// Replace all kind annotations with old-prefix equivalents so we can verify
// that unarchiveLayoutTo normalizes them to the new dev.hauler prefix.
const legacyPrefix = "dev.cosignproject.cosign"
const newPrefix = "dev.hauler"
for i := range idx.Manifests {
if idx.Manifests[i].Annotations == nil {
idx.Manifests[i].Annotations = make(map[string]string)
}
kind := idx.Manifests[i].Annotations[consts.KindAnnotationName]
if kind == "" {
kind = consts.KindAnnotationImage
}
// Rewrite dev.hauler/* → dev.cosignproject.cosign/* to simulate legacy archive.
if strings.HasPrefix(kind, newPrefix) {
kind = legacyPrefix + kind[len(newPrefix):]
}
idx.Manifests[i].Annotations[consts.KindAnnotationName] = kind
}
out, err := json.MarshalIndent(idx, "", " ")
if err != nil {
t.Fatalf("marshal legacy index.json: %v", err)
}
if err := os.WriteFile(indexPath, out, 0644); err != nil {
t.Fatalf("write legacy index.json: %v", err)
}
// Step 3: Re-archive with files at the archive root (no subdir prefix).
legacyArchive := filepath.Join(t.TempDir(), "legacy.tar.zst")
if err := createRootLevelArchive(extractDir, legacyArchive); err != nil {
t.Fatalf("createRootLevelArchive: %v", err)
}
// Step 4: Load the legacy archive.
destDir := t.TempDir()
tempDir := t.TempDir()
if err := unarchiveLayoutTo(ctx, legacyArchive, destDir, tempDir); err != nil {
t.Fatalf("unarchiveLayoutTo legacy: %v", err)
}
// Step 5: Every descriptor in the dest store must now have a dev.hauler kind.
s, err := store.NewLayout(destDir)
if err != nil {
t.Fatalf("store.NewLayout(destDir): %v", err)
}
if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error {
kind := desc.Annotations[consts.KindAnnotationName]
if strings.HasPrefix(kind, legacyPrefix) {
t.Errorf("descriptor %s still has legacy kind %q; expected dev.hauler prefix",
desc.Digest, kind)
}
if !strings.HasPrefix(kind, newPrefix) {
t.Errorf("descriptor %s has unexpected kind %q; expected dev.hauler prefix",
desc.Digest, kind)
}
return nil
}); err != nil {
t.Fatalf("Walk: %v", err)
}
}
// --------------------------------------------------------------------------
// TestClearDir
// --------------------------------------------------------------------------

View File

@@ -4,10 +4,13 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"path"
"path/filepath"
"slices"
"strconv"
"strings"
referencev3 "github.com/distribution/distribution/v3/reference"
"github.com/google/go-containerregistry/pkg/name"
@@ -72,10 +75,64 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, rso *flags.StoreRootOpts, r
return err
}
l.Infof("saving store [%s] to archive [%s]", o.StoreDir, o.FileName)
if o.ChunkSize != "" {
if o.ContainerdCompatibility == true {
l.Warnf("compatibility warning... stores split by chunk size must be imported using `hauler store load` to rejoin before import to containerd")
}
maxBytes, err := parseChunkSize(o.ChunkSize)
if err != nil {
return err
}
chunks, err := archives.SplitArchive(ctx, absOutputfile, maxBytes)
if err != nil {
return err
}
for _, c := range chunks {
l.Infof("saving store [%s] to chunk [%s]", o.StoreDir, filepath.Base(c))
}
} else {
l.Infof("saving store [%s] to archive [%s]", o.StoreDir, o.FileName)
}
return nil
}
// parseChunkSize parses a human-readable byte size string (e.g. "1G", "500M", "2GB")
// into a byte count. Suffixes are treated as binary units (1K = 1024).
func parseChunkSize(s string) (int64, error) {
units := map[string]int64{
"K": 1 << 10, "KB": 1 << 10,
"M": 1 << 20, "MB": 1 << 20,
"G": 1 << 30, "GB": 1 << 30,
"T": 1 << 40, "TB": 1 << 40,
}
s = strings.ToUpper(strings.TrimSpace(s))
var result int64
matched := false
for suffix, mult := range units {
if strings.HasSuffix(s, suffix) {
n, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid chunk size %q", s)
}
result = n * mult
matched = true
break
}
}
if !matched {
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid chunk size %q: %w", s, err)
}
result = n
}
if result <= 0 {
return 0, fmt.Errorf("chunk size must be greater than zero, received %q", s)
}
return result, nil
}
type exports struct {
digests []string
records map[string]tarball.Descriptor

View File

@@ -221,3 +221,99 @@ func TestSaveCmd_EmptyStore(t *testing.T) {
t.Fatalf("archive not created for empty store: %v", err)
}
}
// --------------------------------------------------------------------------
// parseChunkSize unit tests
// --------------------------------------------------------------------------
func TestParseChunkSize(t *testing.T) {
tests := []struct {
name string
input string
want int64
wantErr bool
}{
{name: "kilobytes", input: "1K", want: 1 << 10},
{name: "kilobytes long", input: "1KB", want: 1 << 10},
{name: "megabytes", input: "500M", want: 500 << 20},
{name: "megabytes long", input: "500MB", want: 500 << 20},
{name: "gigabytes", input: "2G", want: 2 << 30},
{name: "gigabytes long", input: "2GB", want: 2 << 30},
{name: "terabytes", input: "1T", want: 1 << 40},
{name: "terabytes long", input: "1TB", want: 1 << 40},
{name: "plain bytes", input: "1024", want: 1024},
{name: "lowercase", input: "1g", want: 1 << 30},
{name: "whitespace trimmed", input: " 1G ", want: 1 << 30},
{name: "zero is invalid", input: "0", wantErr: true},
{name: "zero with suffix", input: "0M", wantErr: true},
{name: "negative bytes", input: "-1", wantErr: true},
{name: "negative with suffix", input: "-1G", wantErr: true},
{name: "empty string", input: "", wantErr: true},
{name: "invalid suffix", input: "1X", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseChunkSize(tt.input)
if (err != nil) != tt.wantErr {
t.Fatalf("parseChunkSize(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
}
if !tt.wantErr && got != tt.want {
t.Errorf("parseChunkSize(%q) = %d, want %d", tt.input, got, tt.want)
}
})
}
}
// --------------------------------------------------------------------------
// SaveCmd chunk-size integration tests
// Do NOT use t.Parallel() — SaveCmd calls os.Chdir.
// --------------------------------------------------------------------------
func TestSaveCmd_ChunkSize(t *testing.T) {
ctx := newTestContext(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "test/chunksave", "v1")
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/test/chunksave:v1", ""); err != nil {
t.Fatalf("AddImage: %v", err)
}
archiveDir := t.TempDir()
archivePath := filepath.Join(archiveDir, "haul-chunked.tar.zst")
o := newSaveOpts(s.Root, archivePath)
o.ChunkSize = "1K"
if err := SaveCmd(ctx, o, defaultRootOpts(s.Root), defaultCliOpts()); err != nil {
t.Fatalf("SaveCmd with chunk-size: %v", err)
}
// original archive must be replaced by chunk files
if _, err := os.Stat(archivePath); !os.IsNotExist(err) {
t.Error("original archive should be removed after chunking")
}
// at least one chunk must exist
matches, err := filepath.Glob(filepath.Join(archiveDir, "haul-chunked_*.tar.zst"))
if err != nil {
t.Fatalf("glob chunks: %v", err)
}
if len(matches) == 0 {
t.Fatal("expected at least one chunk file, found none")
}
}
func TestSaveCmd_ChunkSize_Invalid(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
if err := s.SaveIndex(); err != nil {
t.Fatalf("SaveIndex: %v", err)
}
o := newSaveOpts(s.Root, filepath.Join(t.TempDir(), "haul.tar.zst"))
o.ChunkSize = "0"
if err := SaveCmd(ctx, o, defaultRootOpts(s.Root), defaultCliOpts()); err == nil {
t.Fatal("SaveCmd: expected error for chunk-size=0, got nil")
}
}

View File

@@ -81,54 +81,108 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
l.Infof("processing completed successfully")
}
// If passed a local manifest, process it
for _, fileName := range o.FileName {
l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
// If passed a hauler manifest, process it
if len(o.FileName) != 0 {
for _, fileName := range o.FileName {
l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
haulPath := fileName
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
haulPath := fileName
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
h := getter.NewHttp()
parsedURL, err := url.Parse(haulPath)
h := getter.NewHttp()
parsedURL, err := url.Parse(haulPath)
if err != nil {
return err
}
rc, err := h.Open(ctx, parsedURL)
if err != nil {
return err
}
defer rc.Close()
fileName := h.Name(parsedURL)
if fileName == "" {
fileName = filepath.Base(parsedURL.Path)
}
haulPath = filepath.Join(tempDir, fileName)
out, err := os.Create(haulPath)
if err != nil {
return err
}
defer out.Close()
if _, err = io.Copy(out, rc); err != nil {
return err
}
}
fi, err := os.Open(haulPath)
if err != nil {
return err
}
rc, err := h.Open(ctx, parsedURL)
defer fi.Close()
err = processContent(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
defer rc.Close()
fileName := h.Name(parsedURL)
if fileName == "" {
fileName = filepath.Base(parsedURL.Path)
l.Infof("processing completed successfully")
}
}
// If passed an image.txt file, process it
if len(o.ImageTxt) != 0 {
for _, imageTxt := range o.ImageTxt {
l.Infof("processing image.txt [%s] to store [%s]", imageTxt, o.StoreDir)
haulPath := imageTxt
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
l.Debugf("detected remote image.txt... starting download... [%s]", haulPath)
h := getter.NewHttp()
parsedURL, err := url.Parse(haulPath)
if err != nil {
return err
}
rc, err := h.Open(ctx, parsedURL)
if err != nil {
return err
}
defer rc.Close()
fileName := h.Name(parsedURL)
if fileName == "" {
fileName = filepath.Base(parsedURL.Path)
}
haulPath = filepath.Join(tempDir, fileName)
out, err := os.Create(haulPath)
if err != nil {
return err
}
defer out.Close()
if _, err = io.Copy(out, rc); err != nil {
return err
}
}
haulPath = filepath.Join(tempDir, fileName)
out, err := os.Create(haulPath)
fi, err := os.Open(haulPath)
if err != nil {
return err
}
defer out.Close()
defer fi.Close()
if _, err = io.Copy(out, rc); err != nil {
err = processImageTxt(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
}
fi, err := os.Open(haulPath)
if err != nil {
return err
l.Infof("processing completed successfully")
}
defer fi.Close()
err = processContent(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
l.Infof("processing completed successfully")
}
return nil
@@ -363,3 +417,21 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
}
return nil
}
func processImageTxt(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
l.Infof("syncing images from [%s] to store", filepath.Base(fi.Name()))
scanner := bufio.NewScanner(fi)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
img := v1.Image{Name: line}
l.Infof("adding image [%s] to the store [%s]", line, o.StoreDir)
if err := storeImage(ctx, s, img, o.Platform, rso, ro, ""); err != nil {
return err
}
}
return scanner.Err()
}

View File

@@ -256,6 +256,157 @@ spec:
assertArtifactInStore(t, s, "synced-local.sh")
}
// --------------------------------------------------------------------------
// processImageTxt tests
// --------------------------------------------------------------------------
// writeImageTxtFile writes lines to a temp file and returns it seeked to the
// start, ready for processImageTxt to consume.
func writeImageTxtFile(t *testing.T, lines string) *os.File {
t.Helper()
fi, err := os.CreateTemp(t.TempDir(), "images-*.txt")
if err != nil {
t.Fatalf("writeImageTxtFile CreateTemp: %v", err)
}
t.Cleanup(func() { fi.Close() })
if _, err := fi.WriteString(lines); err != nil {
t.Fatalf("writeImageTxtFile WriteString: %v", err)
}
if _, err := fi.Seek(0, io.SeekStart); err != nil {
t.Fatalf("writeImageTxtFile Seek: %v", err)
}
return fi
}
func TestProcessImageTxt_SingleImage(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/txtimage", "v1")
fi := writeImageTxtFile(t, fmt.Sprintf("%s/myorg/txtimage:v1\n", host))
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt single image: %v", err)
}
assertArtifactInStore(t, s, "myorg/txtimage")
}
func TestProcessImageTxt_MultipleImages(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/alpha", "v1")
seedImage(t, host, "myorg/beta", "v2")
content := fmt.Sprintf("%s/myorg/alpha:v1\n%s/myorg/beta:v2\n", host, host)
fi := writeImageTxtFile(t, content)
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt multiple images: %v", err)
}
assertArtifactInStore(t, s, "myorg/alpha")
assertArtifactInStore(t, s, "myorg/beta")
}
func TestProcessImageTxt_SkipsBlankLinesAndComments(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/commenttest", "v1")
content := fmt.Sprintf("# this is a comment\n\n%s/myorg/commenttest:v1\n\n# another comment\n", host)
fi := writeImageTxtFile(t, content)
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt skip blanks/comments: %v", err)
}
assertArtifactInStore(t, s, "myorg/commenttest")
if n := countArtifactsInStore(t, s); n != 1 {
t.Errorf("expected 1 artifact, got %d", n)
}
}
func TestProcessImageTxt_EmptyFile(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
fi := writeImageTxtFile(t, "")
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt empty file: %v", err)
}
if n := countArtifactsInStore(t, s); n != 0 {
t.Errorf("expected 0 artifacts for empty file, got %d", n)
}
}
// --------------------------------------------------------------------------
// SyncCmd --image-txt integration tests
// --------------------------------------------------------------------------
func TestSyncCmd_ImageTxt_LocalFile(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/syncedtxt", "v1")
txtFile, err := os.CreateTemp(t.TempDir(), "images-*.txt")
if err != nil {
t.Fatalf("CreateTemp: %v", err)
}
txtPath := txtFile.Name()
fmt.Fprintf(txtFile, "%s/myorg/syncedtxt:v1\n", host)
txtFile.Close()
o := newSyncOpts(s.Root)
o.ImageTxt = []string{txtPath}
rso := defaultRootOpts(s.Root)
ro := defaultCliOpts()
if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
t.Fatalf("SyncCmd ImageTxt LocalFile: %v", err)
}
assertArtifactInStore(t, s, "myorg/syncedtxt")
}
func TestSyncCmd_ImageTxt_RemoteFile(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/remotetxt", "v1")
imageListContent := fmt.Sprintf("%s/myorg/remotetxt:v1\n", host)
imageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
io.WriteString(w, imageListContent) //nolint:errcheck
}))
t.Cleanup(imageSrv.Close)
o := newSyncOpts(s.Root)
o.ImageTxt = []string{imageSrv.URL + "/images.txt"}
rso := defaultRootOpts(s.Root)
ro := defaultCliOpts()
if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
t.Fatalf("SyncCmd ImageTxt RemoteFile: %v", err)
}
assertArtifactInStore(t, s, "myorg/remotetxt")
}
func TestSyncCmd_RemoteManifest(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)

View File

@@ -22,6 +22,7 @@ import (
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/static"
gvtypes "github.com/google/go-containerregistry/pkg/v1/types"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
"helm.sh/helm/v3/pkg/action"
@@ -282,6 +283,41 @@ func seedOCI11Referrer(t *testing.T, host, repo string, baseImg gcrv1.Image, opt
}
}
// seedStoreDescriptor injects a descriptor with the given annotations directly
// into the store index without requiring a real registry or blob. This is used
// to pre-populate the store for rewriteReference unit tests.
func seedStoreDescriptor(t *testing.T, s *store.Layout, annotations map[string]string) {
t.Helper()
desc := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Digest: digest.Digest("sha256:" + strings.Repeat("a", 64)),
Size: 1,
Annotations: annotations,
}
if err := s.OCI.AddIndex(desc); err != nil {
t.Fatalf("seedStoreDescriptor: %v", err)
}
}
// assertAnnotationsInStore walks the store and fails if no descriptor has both
// AnnotationRefName == refName AND ContainerdImageNameKey == containerdName.
func assertAnnotationsInStore(t *testing.T, s *store.Layout, refName, containerdName string) {
t.Helper()
found := false
if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error {
if desc.Annotations[ocispec.AnnotationRefName] == refName &&
desc.Annotations[consts.ContainerdImageNameKey] == containerdName {
found = true
}
return nil
}); err != nil {
t.Fatalf("assertAnnotationsInStore walk: %v", err)
}
if !found {
t.Errorf("no artifact with AnnotationRefName=%q and ContainerdImageNameKey=%q found in store", refName, containerdName)
}
}
// assertReferrerInStore walks the store and fails if no descriptor has a kind
// annotation with the KindAnnotationReferrers prefix and a ref containing refSubstring.
func assertReferrerInStore(t *testing.T, s *store.Layout, refSubstring string) {

4
go.mod
View File

@@ -143,7 +143,7 @@ require (
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/analysis v0.24.1 // indirect
@@ -324,7 +324,7 @@ require (
google.golang.org/api v0.267.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
google.golang.org/grpc v1.78.0 // indirect
google.golang.org/grpc v1.79.3 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect

8
go.sum
View File

@@ -378,8 +378,8 @@ github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs
github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@@ -1294,8 +1294,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

View File

@@ -10,6 +10,7 @@ type SaveOpts struct {
FileName string
Platform string
ContainerdCompatibility bool
ChunkSize string
}
func (o *SaveOpts) AddFlags(cmd *cobra.Command) {
@@ -18,5 +19,6 @@ func (o *SaveOpts) AddFlags(cmd *cobra.Command) {
f.StringVarP(&o.FileName, "filename", "f", consts.DefaultHaulerArchiveName, "(Optional) Specify the name of outputted haul")
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform for runtime imports... i.e. linux/amd64 (unspecified implies all)")
f.BoolVar(&o.ContainerdCompatibility, "containerd", false, "(Optional) Enable import compatibility with containerd... removes oci-layout from the haul")
f.StringVar(&o.ChunkSize, "chunk-size", "", "(Optional) Split the output archive into chunks of the specified size (e.g. 1G, 500M, 2048M)")
}

View File

@@ -2,12 +2,12 @@ package flags
import (
"github.com/spf13/cobra"
"hauler.dev/go/hauler/pkg/consts"
)
type SyncOpts struct {
*StoreRootOpts
FileName []string
ImageTxt []string
Key string
CertOidcIssuer string
CertOidcIssuerRegexp string
@@ -25,7 +25,8 @@ type SyncOpts struct {
func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringSliceVarP(&o.FileName, "filename", "f", []string{consts.DefaultHaulerManifestName}, "Specify the name of manifest(s) to sync")
f.StringSliceVarP(&o.FileName, "filename", "f", []string{}, "Specify the name of manifest(s) to sync")
f.StringSliceVarP(&o.ImageTxt, "image-txt", "i", []string{}, "Specify local or remote image.txt file(s) to sync images")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification")
f.StringVar(&o.CertIdentity, "certificate-identity", "", "(Optional) Cosign certificate-identity (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")
f.StringVar(&o.CertIdentityRegexp, "certificate-identity-regexp", "", "(Optional) Cosign certificate-identity-regexp (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")

View File

@@ -3,8 +3,10 @@ package archives
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/mholt/archives"
"hauler.dev/go/hauler/pkg/log"
@@ -102,3 +104,85 @@ func Archive(ctx context.Context, dir, outfile string, compression archives.Comp
l.Debugf("archive created successfully [%s]", outfile)
return nil
}
// SplitArchive splits an existing archive into chunks of at most maxBytes each.
// Chunks are named <base>_0<ext>, <base>_1<ext>, ... where base is the archive
// path with all extensions stripped, and ext is the compound extension (e.g. .tar.zst).
// The original archive is removed after successful splitting.
func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]string, error) {
l := log.FromContext(ctx)
// derive base path and compound extension by stripping all extensions
base := archivePath
ext := ""
for filepath.Ext(base) != "" {
ext = filepath.Ext(base) + ext
base = strings.TrimSuffix(base, filepath.Ext(base))
}
f, err := os.Open(archivePath)
if err != nil {
return nil, fmt.Errorf("failed to open archive for splitting: %w", err)
}
var chunks []string
buf := make([]byte, 32*1024)
chunkIdx := 0
var written int64
var outf *os.File
for {
if outf == nil {
chunkPath := fmt.Sprintf("%s_%d%s", base, chunkIdx, ext)
outf, err = os.Create(chunkPath)
if err != nil {
f.Close()
return nil, fmt.Errorf("failed to create chunk %d: %w", chunkIdx, err)
}
chunks = append(chunks, chunkPath)
l.Debugf("creating chunk [%s]", chunkPath)
written = 0
chunkIdx++
}
remaining := maxBytes - written
readSize := int64(len(buf))
if readSize > remaining {
readSize = remaining
}
n, readErr := f.Read(buf[:readSize])
if n > 0 {
if _, writeErr := outf.Write(buf[:n]); writeErr != nil {
outf.Close()
f.Close()
return nil, fmt.Errorf("failed to write to chunk: %w", writeErr)
}
written += int64(n)
}
if readErr == io.EOF {
outf.Close()
outf = nil
break
}
if readErr != nil {
outf.Close()
f.Close()
return nil, fmt.Errorf("failed to read archive: %w", readErr)
}
if written >= maxBytes {
outf.Close()
outf = nil
}
}
f.Close()
if err := os.Remove(archivePath); err != nil {
return nil, fmt.Errorf("failed to remove original archive after splitting: %w", err)
}
l.Infof("split archive [%s] into %d chunk(s)", filepath.Base(archivePath), len(chunks))
return chunks, nil
}

View File

@@ -1,8 +1,10 @@
package archives
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
@@ -162,3 +164,278 @@ func TestSecurePath(t *testing.T) {
})
}
}
// --------------------------------------------------------------------------
// chunkInfo
// --------------------------------------------------------------------------
func TestChunkInfo(t *testing.T) {
tests := []struct {
name string
path string
wantBase string
wantExt string
wantIndex int
wantOk bool
}{
{
name: "compound extension",
path: "/tmp/haul_3.tar.zst",
wantBase: "/tmp/haul",
wantExt: ".tar.zst",
wantIndex: 3,
wantOk: true,
},
{
name: "single extension",
path: "/tmp/archive_0.zst",
wantBase: "/tmp/archive",
wantExt: ".zst",
wantIndex: 0,
wantOk: true,
},
{
name: "large index",
path: "/tmp/haul_42.tar.zst",
wantBase: "/tmp/haul",
wantExt: ".tar.zst",
wantIndex: 42,
wantOk: true,
},
{
name: "no numeric suffix",
path: "/tmp/haul.tar.zst",
wantOk: false,
},
{
name: "alphabetic suffix",
path: "/tmp/haul_abc.tar.zst",
wantOk: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
base, ext, index, ok := chunkInfo(tt.path)
if ok != tt.wantOk {
t.Fatalf("chunkInfo() ok = %v, want %v", ok, tt.wantOk)
}
if !ok {
return
}
if base != tt.wantBase {
t.Errorf("chunkInfo() base = %q, want %q", base, tt.wantBase)
}
if ext != tt.wantExt {
t.Errorf("chunkInfo() ext = %q, want %q", ext, tt.wantExt)
}
if index != tt.wantIndex {
t.Errorf("chunkInfo() index = %d, want %d", index, tt.wantIndex)
}
})
}
}
// --------------------------------------------------------------------------
// SplitArchive
// --------------------------------------------------------------------------
func TestSplitArchive(t *testing.T) {
ctx := testContext(t)
tests := []struct {
name string
dataSize int
maxBytes int64
}{
{name: "splits into multiple chunks", dataSize: 100, maxBytes: 30},
{name: "single chunk when data fits", dataSize: 50, maxBytes: 100},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
archivePath := filepath.Join(dir, "haul.tar.zst")
data := make([]byte, tt.dataSize)
for i := range data {
data[i] = byte(i % 256)
}
if err := os.WriteFile(archivePath, data, 0o644); err != nil {
t.Fatal(err)
}
chunks, err := SplitArchive(ctx, archivePath, tt.maxBytes)
if err != nil {
t.Fatalf("SplitArchive() error = %v", err)
}
if len(chunks) == 0 {
t.Fatal("SplitArchive() returned no chunks")
}
// original archive must be removed
if _, err := os.Stat(archivePath); !os.IsNotExist(err) {
t.Error("original archive should be removed after splitting")
}
// chunks must follow <base>_N<ext> naming
for i, chunk := range chunks {
expected := filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i))
if chunk != expected {
t.Errorf("chunk[%d] = %s, want %s", i, chunk, expected)
}
}
// concatenating chunks must reproduce the original data
var combined []byte
for _, chunk := range chunks {
b, err := os.ReadFile(chunk)
if err != nil {
t.Fatal(err)
}
combined = append(combined, b...)
}
if !bytes.Equal(combined, data) {
t.Error("combined chunks do not match original data")
}
})
}
}
func TestSplitArchive_MissingFile(t *testing.T) {
ctx := testContext(t)
dir := t.TempDir()
_, err := SplitArchive(ctx, filepath.Join(dir, "nonexistent.tar.zst"), 1<<30)
if err == nil {
t.Fatal("SplitArchive() expected error for missing file, got nil")
}
}
// --------------------------------------------------------------------------
// JoinChunks
// --------------------------------------------------------------------------
func TestJoinChunks(t *testing.T) {
ctx := testContext(t)
t.Run("joins multiple chunks in order", func(t *testing.T) {
dir := t.TempDir()
tempDir := t.TempDir()
for i, content := range []string{"chunk0-data", "chunk1-data", "chunk2-data"} {
if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i)), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir)
if err != nil {
t.Fatalf("JoinChunks() error = %v", err)
}
data, err := os.ReadFile(got)
if err != nil {
t.Fatal(err)
}
if want := []byte("chunk0-datachunk1-datachunk2-data"); !bytes.Equal(data, want) {
t.Errorf("JoinChunks() content = %q, want %q", data, want)
}
})
t.Run("any chunk triggers full assembly", func(t *testing.T) {
dir := t.TempDir()
tempDir := t.TempDir()
for i, content := range []string{"aaa", "bbb"} {
if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("data_%d.tar.zst", i)), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
// pass chunk_1, not chunk_0 — should still assemble from chunk_0
got, err := JoinChunks(ctx, filepath.Join(dir, "data_1.tar.zst"), tempDir)
if err != nil {
t.Fatalf("JoinChunks() error = %v", err)
}
data, err := os.ReadFile(got)
if err != nil {
t.Fatal(err)
}
if want := []byte("aaabbb"); !bytes.Equal(data, want) {
t.Errorf("JoinChunks() content = %q, want %q", data, want)
}
})
t.Run("non-chunk file returned unchanged", func(t *testing.T) {
dir := t.TempDir()
nonChunk := filepath.Join(dir, "haul.tar.zst")
if err := os.WriteFile(nonChunk, []byte("not-a-chunk"), 0o644); err != nil {
t.Fatal(err)
}
got, err := JoinChunks(ctx, nonChunk, t.TempDir())
if err != nil {
t.Fatalf("JoinChunks() error = %v", err)
}
if got != nonChunk {
t.Errorf("JoinChunks() = %s, want %s (unchanged)", got, nonChunk)
}
})
t.Run("non-numeric suffix files excluded", func(t *testing.T) {
dir := t.TempDir()
tempDir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "haul_0.tar.zst"), []byte("valid"), 0o644); err != nil {
t.Fatal(err)
}
// glob matches this but chunkInfo rejects it
if err := os.WriteFile(filepath.Join(dir, "haul_foo.tar.zst"), []byte("invalid"), 0o644); err != nil {
t.Fatal(err)
}
got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir)
if err != nil {
t.Fatalf("JoinChunks() error = %v", err)
}
data, err := os.ReadFile(got)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, []byte("valid")) {
t.Errorf("JoinChunks() included non-numeric suffix file; content = %q", data)
}
})
}
// --------------------------------------------------------------------------
// SplitArchive + JoinChunks round-trip
// --------------------------------------------------------------------------
func TestSplitJoinChunks_RoundTrip(t *testing.T) {
ctx := testContext(t)
original := make([]byte, 1000)
for i := range original {
original[i] = byte(i % 256)
}
dir := t.TempDir()
archivePath := filepath.Join(dir, "haul.tar.zst")
if err := os.WriteFile(archivePath, original, 0o644); err != nil {
t.Fatal(err)
}
chunks, err := SplitArchive(ctx, archivePath, 100)
if err != nil {
t.Fatalf("SplitArchive() error = %v", err)
}
if len(chunks) == 0 {
t.Fatal("SplitArchive() returned no chunks")
}
joined, err := JoinChunks(ctx, chunks[0], t.TempDir())
if err != nil {
t.Fatalf("JoinChunks() error = %v", err)
}
got, err := os.ReadFile(joined)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, original) {
t.Error("round-trip: joined data does not match original")
}
}

View File

@@ -6,6 +6,9 @@ import (
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"github.com/mholt/archives"
@@ -156,3 +159,86 @@ func Unarchive(ctx context.Context, tarball, dst string) error {
l.Infof("unarchiving completed successfully")
return nil
}
var chunkSuffixRe = regexp.MustCompile(`^(.+)_(\d+)$`)
// chunkInfo checks whether archivePath matches the chunk naming pattern (<base>_N<ext>).
// Returns the base path (without index), compound extension, numeric index, and whether it matched.
func chunkInfo(archivePath string) (base, ext string, index int, ok bool) {
dir := filepath.Dir(archivePath)
name := filepath.Base(archivePath)
// strip compound extension (e.g. .tar.zst)
nameBase := name
nameExt := ""
for filepath.Ext(nameBase) != "" {
nameExt = filepath.Ext(nameBase) + nameExt
nameBase = strings.TrimSuffix(nameBase, filepath.Ext(nameBase))
}
m := chunkSuffixRe.FindStringSubmatch(nameBase)
if m == nil {
return "", "", 0, false
}
idx, _ := strconv.Atoi(m[2])
return filepath.Join(dir, m[1]), nameExt, idx, true
}
// JoinChunks detects whether archivePath is a chunk file and, if so, finds all
// sibling chunks, concatenates them in numeric order into a single file in tempDir,
// and returns the path to the joined file. If archivePath is not a chunk, it is
// returned unchanged.
func JoinChunks(ctx context.Context, archivePath, tempDir string) (string, error) {
l := log.FromContext(ctx)
base, ext, _, ok := chunkInfo(archivePath)
if !ok {
return archivePath, nil
}
all, err := filepath.Glob(base + "_*" + ext)
if err != nil {
return archivePath, nil
}
var matches []string
for _, m := range all {
if _, _, _, ok := chunkInfo(m); ok {
matches = append(matches, m)
}
}
if len(matches) == 0 {
return archivePath, nil
}
sort.Slice(matches, func(i, j int) bool {
_, _, idxI, _ := chunkInfo(matches[i])
_, _, idxJ, _ := chunkInfo(matches[j])
return idxI < idxJ
})
l.Debugf("joining %d chunk(s) for [%s]", len(matches), base)
joinedPath := filepath.Join(tempDir, filepath.Base(base)+ext)
outf, err := os.Create(joinedPath)
if err != nil {
return "", fmt.Errorf("failed to create joined archive: %w", err)
}
defer outf.Close()
for _, chunk := range matches {
l.Debugf("joining chunk [%s]", chunk)
cf, err := os.Open(chunk)
if err != nil {
return "", fmt.Errorf("failed to open chunk [%s]: %w", chunk, err)
}
if _, err := io.Copy(outf, cf); err != nil {
cf.Close()
return "", fmt.Errorf("failed to copy chunk [%s]: %w", chunk, err)
}
cf.Close()
}
l.Infof("joined %d chunk(s) into [%s]", len(matches), filepath.Base(joinedPath))
return joinedPath, nil
}

View File

@@ -44,16 +44,16 @@ const (
// annotation keys
ContainerdImageNameKey = "io.containerd.image.name"
KindAnnotationName = "kind"
KindAnnotationImage = "dev.cosignproject.cosign/image"
KindAnnotationIndex = "dev.cosignproject.cosign/imageIndex"
KindAnnotationSigs = "dev.cosignproject.cosign/sigs"
KindAnnotationAtts = "dev.cosignproject.cosign/atts"
KindAnnotationSboms = "dev.cosignproject.cosign/sboms"
KindAnnotationImage = "dev.hauler/image"
KindAnnotationIndex = "dev.hauler/imageIndex"
KindAnnotationSigs = "dev.hauler/sigs"
KindAnnotationAtts = "dev.hauler/atts"
KindAnnotationSboms = "dev.hauler/sboms"
// KindAnnotationReferrers is the kind prefix for OCI 1.1 referrer manifests (cosign v3
// new-bundle-format). Each referrer gets a unique kind with the referrer manifest digest
// appended (e.g. "dev.cosignproject.cosign/referrers/sha256hex") so multiple referrers
// for the same base image coexist in the OCI index.
KindAnnotationReferrers = "dev.cosignproject.cosign/referrers"
// appended (e.g. "dev.hauler/referrers/sha256hex") so multiple referrers for the same
// base image coexist in the OCI index.
KindAnnotationReferrers = "dev.hauler/referrers"
// Sigstore / OCI 1.1 artifact media types used by cosign v3 new-bundle-format.
SigstoreBundleMediaType = "application/vnd.dev.sigstore.bundle.v0.3+json"

19
pkg/consts/migrate.go Normal file
View File

@@ -0,0 +1,19 @@
package consts
import "strings"
// NormalizeLegacyKind translates old dev.cosignproject.cosign kind annotation
// values to their dev.hauler equivalents. Returns the input unchanged if it is
// already a current value or empty.
//
// This handles all cases including the dynamic referrer suffix:
//
// dev.cosignproject.cosign/referrers/<sha256hex> → dev.hauler/referrers/<sha256hex>
func NormalizeLegacyKind(kind string) string {
const oldPrefix = "dev.cosignproject.cosign"
const newPrefix = "dev.hauler"
if strings.HasPrefix(kind, oldPrefix) {
return newPrefix + kind[len(oldPrefix):]
}
return kind
}

View File

@@ -0,0 +1,30 @@
package consts
import "testing"
func TestNormalizeLegacyKind(t *testing.T) {
tests := []struct {
input string
want string
}{
// Old dev.cosignproject.cosign values → new dev.hauler equivalents
{"dev.cosignproject.cosign/image", "dev.hauler/image"},
{"dev.cosignproject.cosign/imageIndex", "dev.hauler/imageIndex"},
{"dev.cosignproject.cosign/sigs", "dev.hauler/sigs"},
{"dev.cosignproject.cosign/atts", "dev.hauler/atts"},
{"dev.cosignproject.cosign/sboms", "dev.hauler/sboms"},
{"dev.cosignproject.cosign/referrers/abc123def456", "dev.hauler/referrers/abc123def456"},
// Already-new values pass through unchanged
{"dev.hauler/image", "dev.hauler/image"},
{"dev.hauler/imageIndex", "dev.hauler/imageIndex"},
{"dev.hauler/referrers/abc123", "dev.hauler/referrers/abc123"},
// Empty string passes through unchanged
{"", ""},
}
for _, tt := range tests {
got := NormalizeLegacyKind(tt.input)
if got != tt.want {
t.Errorf("NormalizeLegacyKind(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"maps"
"os"
"path/filepath"
"sort"
@@ -91,12 +92,21 @@ func (o *OCI) LoadIndex() error {
continue
}
// Set default kind if missing
// Set default kind if missing; normalize legacy dev.cosignproject.cosign values.
kind := desc.Annotations[consts.KindAnnotationName]
kind = consts.NormalizeLegacyKind(kind)
if kind == "" {
kind = consts.KindAnnotationImage
}
// Write the normalized kind back into a copy of the annotations map so
// that Walk() callers receive descriptors with dev.hauler/... values.
// We copy the map to avoid mutating the slice element's shared map.
normalized := make(map[string]string, len(desc.Annotations)+1)
maps.Copy(normalized, desc.Annotations)
normalized[consts.KindAnnotationName] = kind
desc.Annotations = normalized
if strings.TrimSpace(key.String()) != "--" {
switch key.(type) {
case name.Digest:
@@ -130,7 +140,7 @@ func (o *OCI) SaveIndex() error {
kindI := descs[i].Annotations["kind"]
kindJ := descs[j].Annotations["kind"]
// Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top.
// Objects with the prefix of KindAnnotationImage should be at the top.
if strings.HasPrefix(kindI, consts.KindAnnotationImage) && !strings.HasPrefix(kindJ, consts.KindAnnotationImage) {
return true
} else if !strings.HasPrefix(kindI, consts.KindAnnotationImage) && strings.HasPrefix(kindJ, consts.KindAnnotationImage) {
@@ -299,11 +309,18 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr
if err := p.oci.LoadIndex(); err != nil {
return nil, err
}
// Use compound key format: "reference-kind"
// Use compound key format: "reference-kind"; normalize legacy values.
kind := d.Annotations[consts.KindAnnotationName]
kind = consts.NormalizeLegacyKind(kind)
if kind == "" {
kind = consts.KindAnnotationImage
}
// Copy annotations map to avoid mutating the caller's descriptor,
// then write the normalized kind so Walk() callers see dev.hauler/... values.
normalizedAnnotations := make(map[string]string, len(d.Annotations)+1)
maps.Copy(normalizedAnnotations, d.Annotations)
normalizedAnnotations[consts.KindAnnotationName] = kind
d.Annotations = normalizedAnnotations
key := fmt.Sprintf("%s-%s", p.ref, kind)
p.oci.nameMap.Store(key, d)
if err := p.oci.SaveIndex(); err != nil {

281
pkg/content/oci_test.go Normal file
View File

@@ -0,0 +1,281 @@
package content
// oci_test.go covers the annotation-normalization correctness of LoadIndex()
// and ociPusher.Push(). Specifically, it verifies that descriptors returned
// by Walk() carry the normalized dev.hauler/... kind annotation value, not the
// legacy dev.cosignproject.cosign/... value that may be present on disk.
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"hauler.dev/go/hauler/pkg/consts"
)
// buildMinimalOCILayout writes the smallest valid OCI layout (oci-layout marker
// + index.json with the supplied descriptors) into dir. No blobs are written;
// this is sufficient for testing LoadIndex/Walk without a full store.
func buildMinimalOCILayout(t *testing.T, dir string, manifests []ocispec.Descriptor) {
t.Helper()
// oci-layout marker
layoutMarker := map[string]string{"imageLayoutVersion": "1.0.0"}
markerData, err := json.Marshal(layoutMarker)
if err != nil {
t.Fatalf("marshal oci-layout: %v", err)
}
if err := os.WriteFile(filepath.Join(dir, ocispec.ImageLayoutFile), markerData, 0644); err != nil {
t.Fatalf("write oci-layout: %v", err)
}
// index.json
idx := ocispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ocispec.MediaTypeImageIndex,
Manifests: manifests,
}
data, err := json.MarshalIndent(idx, "", " ")
if err != nil {
t.Fatalf("marshal index.json: %v", err)
}
if err := os.WriteFile(filepath.Join(dir, ocispec.ImageIndexFile), data, 0644); err != nil {
t.Fatalf("write index.json: %v", err)
}
}
// fakeDigest returns a syntactically valid digest string that can be used in
// test descriptors without any real blob.
func fakeDigest(hex string) digest.Digest {
// pad hex to 64 chars
for len(hex) < 64 {
hex += "0"
}
return digest.Digest("sha256:" + hex)
}
// --------------------------------------------------------------------------
// TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations
// --------------------------------------------------------------------------
// TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations verifies that
// after LoadIndex() (called implicitly by Walk()), every descriptor returned
// by Walk carries a normalized dev.hauler/... kind annotation, not the legacy
// dev.cosignproject.cosign/... value stored on disk.
func TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations(t *testing.T) {
dir := t.TempDir()
legacyKinds := []string{
"dev.cosignproject.cosign/image",
"dev.cosignproject.cosign/imageIndex",
"dev.cosignproject.cosign/sigs",
"dev.cosignproject.cosign/atts",
"dev.cosignproject.cosign/sboms",
}
var manifests []ocispec.Descriptor
for i, legacyKind := range legacyKinds {
d := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Digest: fakeDigest(strings.Repeat(string(rune('a'+i)), 1)),
Size: 100,
Annotations: map[string]string{
ocispec.AnnotationRefName: "example.com/repo:tag" + strings.Repeat(string(rune('a'+i)), 1),
consts.KindAnnotationName: legacyKind,
},
}
manifests = append(manifests, d)
}
buildMinimalOCILayout(t, dir, manifests)
o, err := NewOCI(dir)
if err != nil {
t.Fatalf("NewOCI: %v", err)
}
var walked []ocispec.Descriptor
if err := o.Walk(func(_ string, desc ocispec.Descriptor) error {
walked = append(walked, desc)
return nil
}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(walked) == 0 {
t.Fatal("Walk returned no descriptors")
}
const legacyPrefix = "dev.cosignproject.cosign"
const newPrefix = "dev.hauler"
for _, desc := range walked {
kind := desc.Annotations[consts.KindAnnotationName]
if strings.HasPrefix(kind, legacyPrefix) {
t.Errorf("descriptor %s: Walk returned legacy kind %q; want normalized dev.hauler/... value",
desc.Digest, kind)
}
if !strings.HasPrefix(kind, newPrefix) {
t.Errorf("descriptor %s: Walk returned unexpected kind %q; want dev.hauler/... prefix",
desc.Digest, kind)
}
}
}
// --------------------------------------------------------------------------
// TestLoadIndex_DoesNotMutateOnDiskAnnotations
// --------------------------------------------------------------------------
// TestLoadIndex_DoesNotMutateOnDiskAnnotations verifies that the normalization
// performed by LoadIndex() is in-memory only: the index.json on disk must
// still carry the original (legacy) annotation values after a Walk() call.
func TestLoadIndex_DoesNotMutateOnDiskAnnotations(t *testing.T) {
dir := t.TempDir()
legacyKind := "dev.cosignproject.cosign/image"
manifests := []ocispec.Descriptor{
{
MediaType: ocispec.MediaTypeImageManifest,
Digest: fakeDigest("b"),
Size: 100,
Annotations: map[string]string{
ocispec.AnnotationRefName: "example.com/repo:tagb",
consts.KindAnnotationName: legacyKind,
},
},
}
buildMinimalOCILayout(t, dir, manifests)
o, err := NewOCI(dir)
if err != nil {
t.Fatalf("NewOCI: %v", err)
}
// Trigger LoadIndex via Walk.
if err := o.Walk(func(_ string, _ ocispec.Descriptor) error { return nil }); err != nil {
t.Fatalf("Walk: %v", err)
}
// Re-read index.json from disk and verify the annotation is unchanged.
data, err := os.ReadFile(filepath.Join(dir, ocispec.ImageIndexFile))
if err != nil {
t.Fatalf("read index.json: %v", err)
}
var idx ocispec.Index
if err := json.Unmarshal(data, &idx); err != nil {
t.Fatalf("unmarshal index.json: %v", err)
}
for _, desc := range idx.Manifests {
got := desc.Annotations[consts.KindAnnotationName]
if got != legacyKind {
t.Errorf("on-disk kind was mutated: got %q, want %q", got, legacyKind)
}
}
}
// --------------------------------------------------------------------------
// TestPush_NormalizesLegacyKindInStoredDescriptor
// --------------------------------------------------------------------------
// TestPush_NormalizesLegacyKindInStoredDescriptor verifies that after a Push()
// that matches the root digest, the descriptor stored in nameMap (and therefore
// returned by subsequent Walk() calls) carries the normalized dev.hauler/...
// kind annotation rather than the legacy value.
func TestPush_NormalizesLegacyKindInStoredDescriptor(t *testing.T) {
dir := t.TempDir()
buildMinimalOCILayout(t, dir, nil) // start with empty index
o, err := NewOCI(dir)
if err != nil {
t.Fatalf("NewOCI: %v", err)
}
// Build a minimal manifest blob so Push() can write it to disk.
manifest := ocispec.Manifest{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ocispec.MediaTypeImageManifest,
Config: ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageConfig,
Digest: fakeDigest("config0"),
Size: 2,
},
}
manifestData, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("marshal manifest: %v", err)
}
manifestDigest := digest.FromBytes(manifestData)
// Ensure the blobs directory exists so Push can write.
blobsDir := filepath.Join(dir, ocispec.ImageBlobsDir, "sha256")
if err := os.MkdirAll(blobsDir, 0755); err != nil {
t.Fatalf("mkdir blobs: %v", err)
}
legacyKind := "dev.cosignproject.cosign/sigs"
baseRef := "example.com/repo:tagsig"
pusher, err := o.Pusher(context.Background(), baseRef+"@"+manifestDigest.String())
if err != nil {
t.Fatalf("Pusher: %v", err)
}
desc := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Digest: manifestDigest,
Size: int64(len(manifestData)),
Annotations: map[string]string{
ocispec.AnnotationRefName: baseRef,
consts.KindAnnotationName: legacyKind,
},
}
w, err := pusher.Push(context.Background(), desc)
if err != nil {
t.Fatalf("Push: %v", err)
}
if _, err := w.Write(manifestData); err != nil {
t.Fatalf("Write manifest: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("Close writer: %v", err)
}
// Now Walk and verify the descriptor in nameMap has the normalized kind.
// We need a fresh OCI instance so Walk calls LoadIndex (which reads SaveIndex output).
o2, err := NewOCI(dir)
if err != nil {
t.Fatalf("NewOCI second: %v", err)
}
const legacyPrefix = "dev.cosignproject.cosign"
const newPrefix = "dev.hauler"
var found bool
if err := o2.Walk(func(_ string, d ocispec.Descriptor) error {
found = true
kind := d.Annotations[consts.KindAnnotationName]
if strings.HasPrefix(kind, legacyPrefix) {
t.Errorf("Push stored descriptor with legacy kind %q; want normalized dev.hauler/... value", kind)
}
if !strings.HasPrefix(kind, newPrefix) {
t.Errorf("Push stored descriptor with unexpected kind %q; want dev.hauler/... prefix", kind)
}
return nil
}); err != nil {
t.Fatalf("Walk: %v", err)
}
if !found {
t.Fatal("Walk returned no descriptors after Push")
}
// Also verify the caller's original descriptor map was NOT mutated.
if desc.Annotations[consts.KindAnnotationName] != legacyKind {
t.Errorf("Push mutated caller's descriptor annotations: got %q, want %q",
desc.Annotations[consts.KindAnnotationName], legacyKind)
}
}

View File

@@ -47,6 +47,10 @@ func NewRegistryTarget(host string, opts RegistryOptions) *RegistryTarget {
)
hosts := func(h string) ([]cdocker.RegistryHost, error) {
host, err := cdocker.DefaultHost(h)
if err != nil {
return nil, err
}
scheme := "https"
if opts.PlainHTTP || opts.Insecure {
scheme = "http"
@@ -55,7 +59,7 @@ func NewRegistryTarget(host string, opts RegistryOptions) *RegistryTarget {
Client: http.DefaultClient,
Authorizer: authorizer,
Scheme: scheme,
Host: h,
Host: host,
Path: "/v2",
Capabilities: cdocker.HostCapabilityPull | cdocker.HostCapabilityResolve | cdocker.HostCapabilityPush,
}}, nil

4
testdata/images.txt vendored Normal file
View File

@@ -0,0 +1,4 @@
# hauler image list
# one image reference per line; blank lines and comments are ignored
ghcr.io/hauler-dev/library/busybox
ghcr.io/hauler-dev/library/busybox:stable