mirror of
https://github.com/hauler-dev/hauler.git
synced 2026-04-06 18:57:11 +00:00
Compare commits
9 Commits
v2.0.0-dev
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
93938d1acb | ||
|
|
641b9db8fd | ||
|
|
c68e72df1c | ||
|
|
e8046a1e30 | ||
|
|
7fd03ea52e | ||
|
|
aa9b883d4c | ||
|
|
565b27d54b | ||
|
|
3adb9257b7 | ||
|
|
268485f6d6 |
298
.github/workflows/cherrypick.yml
vendored
Normal file
298
.github/workflows/cherrypick.yml
vendored
Normal file
@@ -0,0 +1,298 @@
|
||||
name: Cherry-pick to release branch
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request:
|
||||
types: [closed]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# Trigger 1: /cherrypick-X.Y comment on a PR
|
||||
# - If already merged → run cherry-pick immediately
|
||||
# - If not yet merged → add label, cherry-pick will run on merge
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
handle-comment:
|
||||
if: >
|
||||
github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request &&
|
||||
startsWith(github.event.comment.body, '/cherrypick-')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check commenter permissions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COMMENTER: ${{ github.event.comment.user.login }}
|
||||
run: |
|
||||
PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/${COMMENTER}/permission \
|
||||
--jq '.permission')
|
||||
echo "Permission level for $COMMENTER: $PERMISSION"
|
||||
if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" && "$PERMISSION" != "write" ]]; then
|
||||
echo "::warning::User $COMMENTER does not have write access, ignoring cherry-pick request"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Parse version from comment
|
||||
id: parse
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: |
|
||||
VERSION=$(echo "$COMMENT_BODY" | head -1 | grep -oP '(?<=/cherrypick-)\d+\.\d+')
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::Could not parse version from comment"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "target_branch=release/$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "label=cherrypick/$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: React to comment
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
|
||||
-f content='+1'
|
||||
|
||||
- name: Check if PR is merged
|
||||
id: check
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }})
|
||||
MERGED=$(echo "$PR_JSON" | jq -r '.merged')
|
||||
echo "merged=$MERGED" >> "$GITHUB_OUTPUT"
|
||||
echo "pr_title=$(echo "$PR_JSON" | jq -r '.title')" >> "$GITHUB_OUTPUT"
|
||||
echo "base_sha=$(echo "$PR_JSON" | jq -r '.base.sha')" >> "$GITHUB_OUTPUT"
|
||||
echo "head_sha=$(echo "$PR_JSON" | jq -r '.head.sha')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Add cherry-pick label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
LABEL: ${{ steps.parse.outputs.label }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/labels \
|
||||
-f name="$LABEL" -f color="fbca04" -f description="Queued for cherry-pick" 2>/dev/null || true
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/labels \
|
||||
-f "labels[]=$LABEL"
|
||||
|
||||
- name: Notify if queued (not yet merged)
|
||||
if: steps.check.outputs.merged != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
LABEL: ${{ steps.parse.outputs.label }}
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="🏷️ Labeled \`$LABEL\` — backport to \`$TARGET_BRANCH\` will be created automatically when this PR is merged."
|
||||
|
||||
- name: Checkout repository
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Verify target branch exists
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
env:
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null 2>&1; then
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="❌ Cannot cherry-pick: branch \`$TARGET_BRANCH\` does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Apply PR diff and push
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
id: apply
|
||||
env:
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BACKPORT_BRANCH="backport/${PR_NUMBER}-to-${TARGET_BRANCH//\//-}"
|
||||
echo "backport_branch=$BACKPORT_BRANCH" >> "$GITHUB_OUTPUT"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
git checkout "$TARGET_BRANCH"
|
||||
git checkout -b "$BACKPORT_BRANCH"
|
||||
|
||||
# Download the PR's patch from GitHub (pure diff of the PR's changes)
|
||||
gh api repos/${{ github.repository }}/pulls/${PR_NUMBER} \
|
||||
-H "Accept: application/vnd.github.v3.patch" > /tmp/pr.patch
|
||||
|
||||
# Apply the patch
|
||||
HAS_CONFLICTS="false"
|
||||
CONFLICTED_FILES=""
|
||||
|
||||
if git apply --check /tmp/pr.patch 2>/dev/null; then
|
||||
# Clean apply
|
||||
git apply /tmp/pr.patch
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}"
|
||||
elif git apply --3way /tmp/pr.patch; then
|
||||
# Applied with 3-way merge (auto-resolved)
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}" || true
|
||||
else
|
||||
# Has real conflicts — apply what we can
|
||||
HAS_CONFLICTS="true"
|
||||
CONFLICTED_FILES=$(git diff --name-only --diff-filter=U | tr '\n' ',' | sed 's/,$//')
|
||||
# Take the incoming version for conflicted files
|
||||
git diff --name-only --diff-filter=U | while read -r file; do
|
||||
git checkout --theirs -- "$file"
|
||||
done
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH} (conflicts)" || true
|
||||
fi
|
||||
|
||||
echo "has_conflicts=$HAS_CONFLICTS" >> "$GITHUB_OUTPUT"
|
||||
echo "conflicted_files=$CONFLICTED_FILES" >> "$GITHUB_OUTPUT"
|
||||
|
||||
git push origin "$BACKPORT_BRANCH"
|
||||
|
||||
- name: Create backport PR
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
VERSION: ${{ steps.parse.outputs.version }}
|
||||
PR_TITLE: ${{ steps.check.outputs.pr_title }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
BACKPORT_BRANCH: ${{ steps.apply.outputs.backport_branch }}
|
||||
run: |
|
||||
TITLE="[${VERSION}] ${PR_TITLE}"
|
||||
BODY="Backport of #${PR_NUMBER} to \`${TARGET_BRANCH}\`."
|
||||
|
||||
PR_URL=$(gh pr create \
|
||||
--base "$TARGET_BRANCH" \
|
||||
--head "$BACKPORT_BRANCH" \
|
||||
--title "$TITLE" \
|
||||
--body "$BODY")
|
||||
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="✅ Backport PR created: ${PR_URL}"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# Trigger 2: PR merged → process any queued cherrypick/* labels
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
handle-merge:
|
||||
if: >
|
||||
github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.merged == true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Collect cherry-pick labels
|
||||
id: labels
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
LABELS=$(gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/labels \
|
||||
--jq '[.[] | select(.name | startswith("cherrypick/")) | .name] | join(",")')
|
||||
|
||||
if [ -z "$LABELS" ]; then
|
||||
echo "No cherrypick labels found, nothing to do."
|
||||
echo "has_labels=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "Found labels: $LABELS"
|
||||
echo "has_labels=true" >> "$GITHUB_OUTPUT"
|
||||
echo "labels=$LABELS" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
if: steps.labels.outputs.has_labels == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download PR patch
|
||||
if: steps.labels.outputs.has_labels == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/pulls/${PR_NUMBER} \
|
||||
-H "Accept: application/vnd.github.v3.patch" > /tmp/pr.patch
|
||||
|
||||
- name: Process each cherry-pick label
|
||||
if: steps.labels.outputs.has_labels == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
LABELS: ${{ steps.labels.outputs.labels }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
IFS=',' read -ra LABEL_ARRAY <<< "$LABELS"
|
||||
for LABEL in "${LABEL_ARRAY[@]}"; do
|
||||
VERSION="${LABEL#cherrypick/}"
|
||||
TARGET_BRANCH="release/$VERSION"
|
||||
|
||||
echo "=== Processing backport to $TARGET_BRANCH ==="
|
||||
|
||||
# Verify target branch exists
|
||||
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null 2>&1; then
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="❌ Cannot cherry-pick to \`$TARGET_BRANCH\`: branch does not exist."
|
||||
continue
|
||||
fi
|
||||
|
||||
BACKPORT_BRANCH="backport/${PR_NUMBER}-to-${TARGET_BRANCH//\//-}"
|
||||
|
||||
git checkout "$TARGET_BRANCH"
|
||||
git checkout -b "$BACKPORT_BRANCH"
|
||||
|
||||
# Apply the patch
|
||||
HAS_CONFLICTS="false"
|
||||
CONFLICTED_FILES=""
|
||||
|
||||
if git apply --check /tmp/pr.patch 2>/dev/null; then
|
||||
git apply /tmp/pr.patch
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}"
|
||||
elif git apply --3way /tmp/pr.patch; then
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}" || true
|
||||
else
|
||||
HAS_CONFLICTS="true"
|
||||
CONFLICTED_FILES=$(git diff --name-only --diff-filter=U | tr '\n' ',' | sed 's/,$//')
|
||||
git diff --name-only --diff-filter=U | while read -r file; do
|
||||
git checkout --theirs -- "$file"
|
||||
done
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH} (conflicts)" || true
|
||||
fi
|
||||
|
||||
git push origin "$BACKPORT_BRANCH"
|
||||
|
||||
# Build PR title and body
|
||||
TITLE="[${VERSION}] ${PR_TITLE}"
|
||||
BODY="Backport of #${PR_NUMBER} to \`${TARGET_BRANCH}\`."
|
||||
|
||||
PR_URL=$(gh pr create \
|
||||
--base "$TARGET_BRANCH" \
|
||||
--head "$BACKPORT_BRANCH" \
|
||||
--title "$TITLE" \
|
||||
--body "$BODY")
|
||||
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="✅ Backport PR to \`$TARGET_BRANCH\` created: ${PR_URL}"
|
||||
|
||||
# Clean up for next iteration
|
||||
git checkout "$TARGET_BRANCH"
|
||||
git branch -D "$BACKPORT_BRANCH"
|
||||
done
|
||||
28
.github/workflows/tests.yaml
vendored
28
.github/workflows/tests.yaml
vendored
@@ -250,6 +250,13 @@ jobs:
|
||||
hauler store save --filename store.tar.zst
|
||||
# verify via save with filename and platform (amd64)
|
||||
hauler store save --filename store-amd64.tar.zst --platform linux/amd64
|
||||
# verify via save with chunk-size (splits into haul-chunked_0.tar.zst, haul-chunked_1.tar.zst, ...)
|
||||
hauler store save --filename haul-chunked.tar.zst --chunk-size 50M
|
||||
# verify chunk files exist and original is removed
|
||||
ls haul-chunked_*.tar.zst
|
||||
! test -f haul-chunked.tar.zst
|
||||
# verify at least two chunks were produced
|
||||
[ $(ls haul-chunked_*.tar.zst | wc -l) -ge 2 ]
|
||||
|
||||
- name: Remove Hauler Store Contents
|
||||
run: |
|
||||
@@ -269,6 +276,14 @@ jobs:
|
||||
hauler store load --filename store.tar.zst --tempdir /opt
|
||||
# verify via load with filename and platform (amd64)
|
||||
hauler store load --filename store-amd64.tar.zst
|
||||
# verify via load from chunks using explicit first chunk
|
||||
rm -rf store
|
||||
hauler store load --filename haul-chunked_0.tar.zst
|
||||
hauler store info
|
||||
# verify via load from chunks using base filename (auto-detect)
|
||||
rm -rf store
|
||||
hauler store load --filename haul-chunked.tar.zst
|
||||
hauler store info
|
||||
|
||||
- name: Verify Hauler Store Contents
|
||||
run: |
|
||||
@@ -291,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Remove Hauler Store Contents
|
||||
run: |
|
||||
rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst
|
||||
rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst haul-chunked_*.tar.zst
|
||||
hauler store info
|
||||
|
||||
- name: Verify - hauler store sync
|
||||
@@ -303,6 +318,17 @@ jobs:
|
||||
hauler store sync --filename testdata/hauler-manifest-pipeline.yaml --filename testdata/hauler-manifest.yaml
|
||||
# need more tests here
|
||||
|
||||
- name: Verify - hauler store sync (image list)
|
||||
run: |
|
||||
# verify via local image list file
|
||||
hauler store sync --image-txt testdata/images.txt
|
||||
# verify via multiple image list files
|
||||
hauler store sync --image-txt testdata/images.txt --image-txt testdata/images.txt
|
||||
# verify via remote image list file
|
||||
hauler store sync --image-txt https://raw.githubusercontent.com/hauler-dev/hauler/main/testdata/images.txt
|
||||
# confirm images are present in the store
|
||||
hauler store info | grep 'busybox'
|
||||
|
||||
- name: Verify - hauler store serve
|
||||
run: |
|
||||
hauler store serve --help
|
||||
|
||||
@@ -133,6 +133,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
|
||||
}
|
||||
|
||||
if rewrite != "" {
|
||||
rawRewrite := rewrite
|
||||
rewrite = strings.TrimPrefix(rewrite, "/")
|
||||
if !strings.Contains(rewrite, ":") {
|
||||
if tag, ok := r.(name.Tag); ok {
|
||||
@@ -146,7 +147,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err)
|
||||
}
|
||||
if err := rewriteReference(ctx, s, r, newRef); err != nil {
|
||||
if err := rewriteReference(ctx, s, r, newRef, rawRewrite); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -155,7 +156,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
|
||||
return nil
|
||||
}
|
||||
|
||||
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference) error {
|
||||
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference, rawRewrite string) error {
|
||||
l := log.FromContext(ctx)
|
||||
|
||||
if err := s.OCI.LoadIndex(); err != nil {
|
||||
@@ -184,8 +185,9 @@ func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Referenc
|
||||
newRegistry := newRefContext.RegistryStr()
|
||||
// If user omitted a registry in the rewrite string, go-containerregistry defaults to
|
||||
// index.docker.io. Preserve the original registry when the source is non-docker.
|
||||
if newRegistry == "index.docker.io" && oldRegistry != "index.docker.io" {
|
||||
if newRegistry == "index.docker.io" && !strings.HasPrefix(rawRewrite, "docker.io") && !strings.HasPrefix(rawRewrite, "index.docker.io") {
|
||||
newRegistry = oldRegistry
|
||||
newRepo = strings.TrimPrefix(newRepo, "library/") //if rewrite has library/ prefix in path it is stripped off unless registry specified in rewrite
|
||||
}
|
||||
oldTotal := oldRepo + ":" + oldTag
|
||||
newTotal := newRepo + ":" + newTag
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
"github.com/google/go-containerregistry/pkg/registry"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
helmchart "helm.sh/helm/v3/pkg/chart"
|
||||
|
||||
"hauler.dev/go/hauler/internal/flags"
|
||||
@@ -248,7 +249,9 @@ func TestRewriteReference(t *testing.T) {
|
||||
t.Fatalf("parse newRef: %v", err)
|
||||
}
|
||||
|
||||
if err := rewriteReference(ctx, s, oldRef, newRef); err != nil {
|
||||
rawRewrite := newRef.String()
|
||||
|
||||
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
|
||||
t.Fatalf("rewriteReference: %v", err)
|
||||
}
|
||||
|
||||
@@ -259,8 +262,9 @@ func TestRewriteReference(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
oldRef, _ := name.NewTag("docker.io/missing/repo:v1")
|
||||
newRef, _ := name.NewTag("docker.io/new/repo:v2")
|
||||
rawRewrite := newRef.String()
|
||||
|
||||
err := rewriteReference(ctx, s, oldRef, newRef)
|
||||
err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
@@ -268,6 +272,86 @@ func TestRewriteReference(t *testing.T) {
|
||||
t.Errorf("expected 'could not find' in error, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Tests for the registry-preservation / library/-stripping logic (lines 188-191).
|
||||
// go-containerregistry normalises bare single-name Docker Hub refs (e.g. "nginx:latest")
|
||||
// to "index.docker.io/library/nginx:latest". When the rewrite string omits a registry,
|
||||
// rewriteReference must (a) preserve the source registry and (b) strip the injected
|
||||
// "library/" prefix so that the stored ref looks like "nginx:v2", not "library/nginx:v2".
|
||||
|
||||
t.Run("path-only rewrite strips library/ prefix from docker hub official image", func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedStoreDescriptor(t, s, map[string]string{
|
||||
ocispec.AnnotationRefName: "library/nginx:latest",
|
||||
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
|
||||
})
|
||||
|
||||
oldRef, _ := name.NewTag("nginx:latest") // → index.docker.io/library/nginx:latest
|
||||
newRef, _ := name.NewTag("nginx:v2") // → index.docker.io/library/nginx:v2
|
||||
rawRewrite := "nginx:v2"
|
||||
|
||||
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
|
||||
t.Fatalf("rewriteReference: %v", err)
|
||||
}
|
||||
// library/ must be stripped; registry stays index.docker.io
|
||||
assertAnnotationsInStore(t, s, "nginx:v2", "index.docker.io/nginx:v2")
|
||||
})
|
||||
|
||||
t.Run("explicit docker.io rewrite preserves library/ prefix", func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedStoreDescriptor(t, s, map[string]string{
|
||||
ocispec.AnnotationRefName: "library/nginx:latest",
|
||||
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
|
||||
})
|
||||
|
||||
oldRef, _ := name.NewTag("nginx:latest")
|
||||
newRef, _ := name.NewTag("docker.io/nginx:v2") // → index.docker.io/library/nginx:v2
|
||||
rawRewrite := "docker.io/nginx:v2"
|
||||
|
||||
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
|
||||
t.Fatalf("rewriteReference: %v", err)
|
||||
}
|
||||
// rawRewrite starts with "docker.io" → condition must NOT fire → library/ preserved
|
||||
assertAnnotationsInStore(t, s, "library/nginx:v2", "index.docker.io/library/nginx:v2")
|
||||
})
|
||||
|
||||
t.Run("explicit index.docker.io rewrite preserves library/ prefix", func(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
seedStoreDescriptor(t, s, map[string]string{
|
||||
ocispec.AnnotationRefName: "library/nginx:latest",
|
||||
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
|
||||
})
|
||||
|
||||
oldRef, _ := name.NewTag("nginx:latest")
|
||||
newRef, _ := name.NewTag("index.docker.io/nginx:v2") // → index.docker.io/library/nginx:v2
|
||||
rawRewrite := "index.docker.io/nginx:v2"
|
||||
|
||||
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
|
||||
t.Fatalf("rewriteReference: %v", err)
|
||||
}
|
||||
// rawRewrite starts with "index.docker.io" → condition must NOT fire → library/ preserved
|
||||
assertAnnotationsInStore(t, s, "library/nginx:v2", "index.docker.io/library/nginx:v2")
|
||||
})
|
||||
|
||||
t.Run("non-docker source with path-only rewrite preserves original registry", func(t *testing.T) {
|
||||
host, rOpts := newTestRegistry(t)
|
||||
seedImage(t, host, "src/repo", "v1", rOpts...)
|
||||
|
||||
s := newTestStore(t)
|
||||
if err := s.AddImage(ctx, host+"/src/repo:v1", "", rOpts...); err != nil {
|
||||
t.Fatalf("AddImage: %v", err)
|
||||
}
|
||||
|
||||
oldRef, _ := name.NewTag(host+"/src/repo:v1", name.Insecure)
|
||||
newRef, _ := name.NewTag("newrepo/img:v2") // defaults to index.docker.io
|
||||
rawRewrite := "newrepo/img:v2"
|
||||
|
||||
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
|
||||
t.Fatalf("rewriteReference: %v", err)
|
||||
}
|
||||
// condition fires → registry reverts to host, no library/ to strip
|
||||
assertAnnotationsInStore(t, s, "newrepo/img:v2", host+"/newrepo/img:v2")
|
||||
})
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
@@ -39,8 +39,9 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r
|
||||
l.Debugf("using temporary directory at [%s]", tempDir)
|
||||
|
||||
for _, fileName := range o.FileName {
|
||||
l.Infof("loading haul [%s] to [%s]", fileName, o.StoreDir)
|
||||
err := unarchiveLayoutTo(ctx, fileName, o.StoreDir, tempDir)
|
||||
resolved := resolveHaulPath(fileName)
|
||||
l.Infof("loading haul [%s] to [%s]", resolved, o.StoreDir)
|
||||
err := unarchiveLayoutTo(ctx, resolved, o.StoreDir, tempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -85,6 +86,13 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
|
||||
}
|
||||
}
|
||||
|
||||
// reassemble chunk files if haulPath matches the chunk naming pattern
|
||||
joined, err := archives.JoinChunks(ctx, haulPath, tempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
haulPath = joined
|
||||
|
||||
if err := archives.Unarchive(ctx, haulPath, tempDir); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -107,6 +115,9 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
|
||||
if _, exists := idx.Manifests[i].Annotations[consts.KindAnnotationName]; !exists {
|
||||
idx.Manifests[i].Annotations[consts.KindAnnotationName] = consts.KindAnnotationImage
|
||||
}
|
||||
// Translate legacy dev.cosignproject.cosign values to dev.hauler equivalents.
|
||||
kind := idx.Manifests[i].Annotations[consts.KindAnnotationName]
|
||||
idx.Manifests[i].Annotations[consts.KindAnnotationName] = consts.NormalizeLegacyKind(kind)
|
||||
if ref, ok := idx.Manifests[i].Annotations[consts.ContainerdImageNameKey]; ok {
|
||||
if slash := strings.Index(ref, "/"); slash != -1 {
|
||||
ref = ref[slash+1:]
|
||||
@@ -139,6 +150,29 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
|
||||
return err
|
||||
}
|
||||
|
||||
// resolveHaulPath returns path as-is if it exists or is a URL. If the file is
|
||||
// not found, it globs for chunk files matching <base>_*<ext> in the same
|
||||
// directory and returns the first match so JoinChunks can reassemble them.
|
||||
func resolveHaulPath(path string) string {
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
return path
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path
|
||||
}
|
||||
base := path
|
||||
ext := ""
|
||||
for filepath.Ext(base) != "" {
|
||||
ext = filepath.Ext(base) + ext
|
||||
base = strings.TrimSuffix(base, filepath.Ext(base))
|
||||
}
|
||||
matches, err := filepath.Glob(base + "_*" + ext)
|
||||
if err != nil || len(matches) == 0 {
|
||||
return path
|
||||
}
|
||||
return matches[0]
|
||||
}
|
||||
|
||||
func clearDir(path string) error {
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
|
||||
@@ -287,6 +287,99 @@ func TestUnarchiveLayoutTo_AnnotationBackfill(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// TestUnarchiveLayoutTo_LegacyKindMigration
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// TestUnarchiveLayoutTo_LegacyKindMigration crafts a haul archive whose
|
||||
// index.json contains old dev.cosignproject.cosign kind values, then verifies
|
||||
// that unarchiveLayoutTo translates them to dev.hauler equivalents.
|
||||
func TestUnarchiveLayoutTo_LegacyKindMigration(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
|
||||
// Step 1: Extract the real test archive to obtain a valid OCI layout on disk.
|
||||
extractDir := t.TempDir()
|
||||
if err := archives.Unarchive(ctx, testHaulArchive, extractDir); err != nil {
|
||||
t.Fatalf("Unarchive: %v", err)
|
||||
}
|
||||
|
||||
// Step 2: Read index.json and inject old dev.cosignproject.cosign kind values.
|
||||
indexPath := filepath.Join(extractDir, "index.json")
|
||||
data, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read index.json: %v", err)
|
||||
}
|
||||
|
||||
var idx ocispec.Index
|
||||
if err := json.Unmarshal(data, &idx); err != nil {
|
||||
t.Fatalf("unmarshal index.json: %v", err)
|
||||
}
|
||||
if len(idx.Manifests) == 0 {
|
||||
t.Skip("testdata/haul.tar.zst has no top-level manifests — cannot test legacy kind migration")
|
||||
}
|
||||
|
||||
// Replace all kind annotations with old-prefix equivalents so we can verify
|
||||
// that unarchiveLayoutTo normalizes them to the new dev.hauler prefix.
|
||||
const legacyPrefix = "dev.cosignproject.cosign"
|
||||
const newPrefix = "dev.hauler"
|
||||
for i := range idx.Manifests {
|
||||
if idx.Manifests[i].Annotations == nil {
|
||||
idx.Manifests[i].Annotations = make(map[string]string)
|
||||
}
|
||||
kind := idx.Manifests[i].Annotations[consts.KindAnnotationName]
|
||||
if kind == "" {
|
||||
kind = consts.KindAnnotationImage
|
||||
}
|
||||
// Rewrite dev.hauler/* → dev.cosignproject.cosign/* to simulate legacy archive.
|
||||
if strings.HasPrefix(kind, newPrefix) {
|
||||
kind = legacyPrefix + kind[len(newPrefix):]
|
||||
}
|
||||
idx.Manifests[i].Annotations[consts.KindAnnotationName] = kind
|
||||
}
|
||||
|
||||
out, err := json.MarshalIndent(idx, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal legacy index.json: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(indexPath, out, 0644); err != nil {
|
||||
t.Fatalf("write legacy index.json: %v", err)
|
||||
}
|
||||
|
||||
// Step 3: Re-archive with files at the archive root (no subdir prefix).
|
||||
legacyArchive := filepath.Join(t.TempDir(), "legacy.tar.zst")
|
||||
if err := createRootLevelArchive(extractDir, legacyArchive); err != nil {
|
||||
t.Fatalf("createRootLevelArchive: %v", err)
|
||||
}
|
||||
|
||||
// Step 4: Load the legacy archive.
|
||||
destDir := t.TempDir()
|
||||
tempDir := t.TempDir()
|
||||
if err := unarchiveLayoutTo(ctx, legacyArchive, destDir, tempDir); err != nil {
|
||||
t.Fatalf("unarchiveLayoutTo legacy: %v", err)
|
||||
}
|
||||
|
||||
// Step 5: Every descriptor in the dest store must now have a dev.hauler kind.
|
||||
s, err := store.NewLayout(destDir)
|
||||
if err != nil {
|
||||
t.Fatalf("store.NewLayout(destDir): %v", err)
|
||||
}
|
||||
|
||||
if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error {
|
||||
kind := desc.Annotations[consts.KindAnnotationName]
|
||||
if strings.HasPrefix(kind, legacyPrefix) {
|
||||
t.Errorf("descriptor %s still has legacy kind %q; expected dev.hauler prefix",
|
||||
desc.Digest, kind)
|
||||
}
|
||||
if !strings.HasPrefix(kind, newPrefix) {
|
||||
t.Errorf("descriptor %s has unexpected kind %q; expected dev.hauler prefix",
|
||||
desc.Digest, kind)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// TestClearDir
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
@@ -4,10 +4,13 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
referencev3 "github.com/distribution/distribution/v3/reference"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
@@ -72,10 +75,64 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, rso *flags.StoreRootOpts, r
|
||||
return err
|
||||
}
|
||||
|
||||
l.Infof("saving store [%s] to archive [%s]", o.StoreDir, o.FileName)
|
||||
if o.ChunkSize != "" {
|
||||
if o.ContainerdCompatibility == true {
|
||||
l.Warnf("compatibility warning... stores split by chunk size must be imported using `hauler store load` to rejoin before import to containerd")
|
||||
}
|
||||
maxBytes, err := parseChunkSize(o.ChunkSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunks, err := archives.SplitArchive(ctx, absOutputfile, maxBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range chunks {
|
||||
l.Infof("saving store [%s] to chunk [%s]", o.StoreDir, filepath.Base(c))
|
||||
}
|
||||
} else {
|
||||
l.Infof("saving store [%s] to archive [%s]", o.StoreDir, o.FileName)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseChunkSize parses a human-readable byte size string (e.g. "1G", "500M", "2GB")
|
||||
// into a byte count. Suffixes are treated as binary units (1K = 1024).
|
||||
func parseChunkSize(s string) (int64, error) {
|
||||
units := map[string]int64{
|
||||
"K": 1 << 10, "KB": 1 << 10,
|
||||
"M": 1 << 20, "MB": 1 << 20,
|
||||
"G": 1 << 30, "GB": 1 << 30,
|
||||
"T": 1 << 40, "TB": 1 << 40,
|
||||
}
|
||||
s = strings.ToUpper(strings.TrimSpace(s))
|
||||
var result int64
|
||||
matched := false
|
||||
for suffix, mult := range units {
|
||||
if strings.HasSuffix(s, suffix) {
|
||||
n, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid chunk size %q", s)
|
||||
}
|
||||
result = n * mult
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid chunk size %q: %w", s, err)
|
||||
}
|
||||
result = n
|
||||
}
|
||||
if result <= 0 {
|
||||
return 0, fmt.Errorf("chunk size must be greater than zero, received %q", s)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type exports struct {
|
||||
digests []string
|
||||
records map[string]tarball.Descriptor
|
||||
|
||||
@@ -221,3 +221,99 @@ func TestSaveCmd_EmptyStore(t *testing.T) {
|
||||
t.Fatalf("archive not created for empty store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// parseChunkSize unit tests
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
func TestParseChunkSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want int64
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "kilobytes", input: "1K", want: 1 << 10},
|
||||
{name: "kilobytes long", input: "1KB", want: 1 << 10},
|
||||
{name: "megabytes", input: "500M", want: 500 << 20},
|
||||
{name: "megabytes long", input: "500MB", want: 500 << 20},
|
||||
{name: "gigabytes", input: "2G", want: 2 << 30},
|
||||
{name: "gigabytes long", input: "2GB", want: 2 << 30},
|
||||
{name: "terabytes", input: "1T", want: 1 << 40},
|
||||
{name: "terabytes long", input: "1TB", want: 1 << 40},
|
||||
{name: "plain bytes", input: "1024", want: 1024},
|
||||
{name: "lowercase", input: "1g", want: 1 << 30},
|
||||
{name: "whitespace trimmed", input: " 1G ", want: 1 << 30},
|
||||
{name: "zero is invalid", input: "0", wantErr: true},
|
||||
{name: "zero with suffix", input: "0M", wantErr: true},
|
||||
{name: "negative bytes", input: "-1", wantErr: true},
|
||||
{name: "negative with suffix", input: "-1G", wantErr: true},
|
||||
{name: "empty string", input: "", wantErr: true},
|
||||
{name: "invalid suffix", input: "1X", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseChunkSize(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("parseChunkSize(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
if !tt.wantErr && got != tt.want {
|
||||
t.Errorf("parseChunkSize(%q) = %d, want %d", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// SaveCmd chunk-size integration tests
|
||||
// Do NOT use t.Parallel() — SaveCmd calls os.Chdir.
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
func TestSaveCmd_ChunkSize(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
host, _ := newLocalhostRegistry(t)
|
||||
seedImage(t, host, "test/chunksave", "v1")
|
||||
|
||||
s := newTestStore(t)
|
||||
if err := s.AddImage(ctx, host+"/test/chunksave:v1", ""); err != nil {
|
||||
t.Fatalf("AddImage: %v", err)
|
||||
}
|
||||
|
||||
archiveDir := t.TempDir()
|
||||
archivePath := filepath.Join(archiveDir, "haul-chunked.tar.zst")
|
||||
o := newSaveOpts(s.Root, archivePath)
|
||||
o.ChunkSize = "1K"
|
||||
|
||||
if err := SaveCmd(ctx, o, defaultRootOpts(s.Root), defaultCliOpts()); err != nil {
|
||||
t.Fatalf("SaveCmd with chunk-size: %v", err)
|
||||
}
|
||||
|
||||
// original archive must be replaced by chunk files
|
||||
if _, err := os.Stat(archivePath); !os.IsNotExist(err) {
|
||||
t.Error("original archive should be removed after chunking")
|
||||
}
|
||||
|
||||
// at least one chunk must exist
|
||||
matches, err := filepath.Glob(filepath.Join(archiveDir, "haul-chunked_*.tar.zst"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob chunks: %v", err)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatal("expected at least one chunk file, found none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveCmd_ChunkSize_Invalid(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
if err := s.SaveIndex(); err != nil {
|
||||
t.Fatalf("SaveIndex: %v", err)
|
||||
}
|
||||
|
||||
o := newSaveOpts(s.Root, filepath.Join(t.TempDir(), "haul.tar.zst"))
|
||||
o.ChunkSize = "0"
|
||||
|
||||
if err := SaveCmd(ctx, o, defaultRootOpts(s.Root), defaultCliOpts()); err == nil {
|
||||
t.Fatal("SaveCmd: expected error for chunk-size=0, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,54 +81,108 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
|
||||
l.Infof("processing completed successfully")
|
||||
}
|
||||
|
||||
// If passed a local manifest, process it
|
||||
for _, fileName := range o.FileName {
|
||||
l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
|
||||
// If passed a hauler manifest, process it
|
||||
if len(o.FileName) != 0 {
|
||||
for _, fileName := range o.FileName {
|
||||
l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
|
||||
|
||||
haulPath := fileName
|
||||
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
|
||||
l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
|
||||
haulPath := fileName
|
||||
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
|
||||
l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
|
||||
|
||||
h := getter.NewHttp()
|
||||
parsedURL, err := url.Parse(haulPath)
|
||||
h := getter.NewHttp()
|
||||
parsedURL, err := url.Parse(haulPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rc, err := h.Open(ctx, parsedURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
fileName := h.Name(parsedURL)
|
||||
if fileName == "" {
|
||||
fileName = filepath.Base(parsedURL.Path)
|
||||
}
|
||||
haulPath = filepath.Join(tempDir, fileName)
|
||||
|
||||
out, err := os.Create(haulPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
if _, err = io.Copy(out, rc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fi, err := os.Open(haulPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rc, err := h.Open(ctx, parsedURL)
|
||||
defer fi.Close()
|
||||
|
||||
err = processContent(ctx, fi, o, s, rso, ro)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
fileName := h.Name(parsedURL)
|
||||
if fileName == "" {
|
||||
fileName = filepath.Base(parsedURL.Path)
|
||||
l.Infof("processing completed successfully")
|
||||
}
|
||||
}
|
||||
|
||||
// If passed an image.txt file, process it
|
||||
if len(o.ImageTxt) != 0 {
|
||||
for _, imageTxt := range o.ImageTxt {
|
||||
l.Infof("processing image.txt [%s] to store [%s]", imageTxt, o.StoreDir)
|
||||
|
||||
haulPath := imageTxt
|
||||
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
|
||||
l.Debugf("detected remote image.txt... starting download... [%s]", haulPath)
|
||||
|
||||
h := getter.NewHttp()
|
||||
parsedURL, err := url.Parse(haulPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rc, err := h.Open(ctx, parsedURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
fileName := h.Name(parsedURL)
|
||||
if fileName == "" {
|
||||
fileName = filepath.Base(parsedURL.Path)
|
||||
}
|
||||
haulPath = filepath.Join(tempDir, fileName)
|
||||
|
||||
out, err := os.Create(haulPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
if _, err = io.Copy(out, rc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
haulPath = filepath.Join(tempDir, fileName)
|
||||
|
||||
out, err := os.Create(haulPath)
|
||||
fi, err := os.Open(haulPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
defer fi.Close()
|
||||
|
||||
if _, err = io.Copy(out, rc); err != nil {
|
||||
err = processImageTxt(ctx, fi, o, s, rso, ro)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fi, err := os.Open(haulPath)
|
||||
if err != nil {
|
||||
return err
|
||||
l.Infof("processing completed successfully")
|
||||
}
|
||||
defer fi.Close()
|
||||
|
||||
err = processContent(ctx, fi, o, s, rso, ro)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
l.Infof("processing completed successfully")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -363,3 +417,21 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func processImageTxt(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
|
||||
l := log.FromContext(ctx)
|
||||
l.Infof("syncing images from [%s] to store", filepath.Base(fi.Name()))
|
||||
scanner := bufio.NewScanner(fi)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
img := v1.Image{Name: line}
|
||||
l.Infof("adding image [%s] to the store [%s]", line, o.StoreDir)
|
||||
if err := storeImage(ctx, s, img, o.Platform, rso, ro, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
@@ -256,6 +256,157 @@ spec:
|
||||
assertArtifactInStore(t, s, "synced-local.sh")
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// processImageTxt tests
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// writeImageTxtFile writes lines to a temp file and returns it seeked to the
|
||||
// start, ready for processImageTxt to consume.
|
||||
func writeImageTxtFile(t *testing.T, lines string) *os.File {
|
||||
t.Helper()
|
||||
fi, err := os.CreateTemp(t.TempDir(), "images-*.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("writeImageTxtFile CreateTemp: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { fi.Close() })
|
||||
if _, err := fi.WriteString(lines); err != nil {
|
||||
t.Fatalf("writeImageTxtFile WriteString: %v", err)
|
||||
}
|
||||
if _, err := fi.Seek(0, io.SeekStart); err != nil {
|
||||
t.Fatalf("writeImageTxtFile Seek: %v", err)
|
||||
}
|
||||
return fi
|
||||
}
|
||||
|
||||
func TestProcessImageTxt_SingleImage(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
|
||||
host, _ := newLocalhostRegistry(t)
|
||||
seedImage(t, host, "myorg/txtimage", "v1")
|
||||
|
||||
fi := writeImageTxtFile(t, fmt.Sprintf("%s/myorg/txtimage:v1\n", host))
|
||||
o := newSyncOpts(s.Root)
|
||||
ro := defaultCliOpts()
|
||||
|
||||
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
|
||||
t.Fatalf("processImageTxt single image: %v", err)
|
||||
}
|
||||
assertArtifactInStore(t, s, "myorg/txtimage")
|
||||
}
|
||||
|
||||
func TestProcessImageTxt_MultipleImages(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
|
||||
host, _ := newLocalhostRegistry(t)
|
||||
seedImage(t, host, "myorg/alpha", "v1")
|
||||
seedImage(t, host, "myorg/beta", "v2")
|
||||
|
||||
content := fmt.Sprintf("%s/myorg/alpha:v1\n%s/myorg/beta:v2\n", host, host)
|
||||
fi := writeImageTxtFile(t, content)
|
||||
o := newSyncOpts(s.Root)
|
||||
ro := defaultCliOpts()
|
||||
|
||||
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
|
||||
t.Fatalf("processImageTxt multiple images: %v", err)
|
||||
}
|
||||
assertArtifactInStore(t, s, "myorg/alpha")
|
||||
assertArtifactInStore(t, s, "myorg/beta")
|
||||
}
|
||||
|
||||
func TestProcessImageTxt_SkipsBlankLinesAndComments(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
|
||||
host, _ := newLocalhostRegistry(t)
|
||||
seedImage(t, host, "myorg/commenttest", "v1")
|
||||
|
||||
content := fmt.Sprintf("# this is a comment\n\n%s/myorg/commenttest:v1\n\n# another comment\n", host)
|
||||
fi := writeImageTxtFile(t, content)
|
||||
o := newSyncOpts(s.Root)
|
||||
ro := defaultCliOpts()
|
||||
|
||||
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
|
||||
t.Fatalf("processImageTxt skip blanks/comments: %v", err)
|
||||
}
|
||||
assertArtifactInStore(t, s, "myorg/commenttest")
|
||||
if n := countArtifactsInStore(t, s); n != 1 {
|
||||
t.Errorf("expected 1 artifact, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessImageTxt_EmptyFile(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
|
||||
fi := writeImageTxtFile(t, "")
|
||||
o := newSyncOpts(s.Root)
|
||||
ro := defaultCliOpts()
|
||||
|
||||
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
|
||||
t.Fatalf("processImageTxt empty file: %v", err)
|
||||
}
|
||||
if n := countArtifactsInStore(t, s); n != 0 {
|
||||
t.Errorf("expected 0 artifacts for empty file, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// SyncCmd --image-txt integration tests
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
func TestSyncCmd_ImageTxt_LocalFile(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
|
||||
host, _ := newLocalhostRegistry(t)
|
||||
seedImage(t, host, "myorg/syncedtxt", "v1")
|
||||
|
||||
txtFile, err := os.CreateTemp(t.TempDir(), "images-*.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTemp: %v", err)
|
||||
}
|
||||
txtPath := txtFile.Name()
|
||||
fmt.Fprintf(txtFile, "%s/myorg/syncedtxt:v1\n", host)
|
||||
txtFile.Close()
|
||||
|
||||
o := newSyncOpts(s.Root)
|
||||
o.ImageTxt = []string{txtPath}
|
||||
rso := defaultRootOpts(s.Root)
|
||||
ro := defaultCliOpts()
|
||||
|
||||
if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
|
||||
t.Fatalf("SyncCmd ImageTxt LocalFile: %v", err)
|
||||
}
|
||||
assertArtifactInStore(t, s, "myorg/syncedtxt")
|
||||
}
|
||||
|
||||
func TestSyncCmd_ImageTxt_RemoteFile(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
|
||||
host, _ := newLocalhostRegistry(t)
|
||||
seedImage(t, host, "myorg/remotetxt", "v1")
|
||||
|
||||
imageListContent := fmt.Sprintf("%s/myorg/remotetxt:v1\n", host)
|
||||
imageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
io.WriteString(w, imageListContent) //nolint:errcheck
|
||||
}))
|
||||
t.Cleanup(imageSrv.Close)
|
||||
|
||||
o := newSyncOpts(s.Root)
|
||||
o.ImageTxt = []string{imageSrv.URL + "/images.txt"}
|
||||
rso := defaultRootOpts(s.Root)
|
||||
ro := defaultCliOpts()
|
||||
|
||||
if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
|
||||
t.Fatalf("SyncCmd ImageTxt RemoteFile: %v", err)
|
||||
}
|
||||
assertArtifactInStore(t, s, "myorg/remotetxt")
|
||||
}
|
||||
|
||||
func TestSyncCmd_RemoteManifest(t *testing.T) {
|
||||
ctx := newTestContext(t)
|
||||
s := newTestStore(t)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
"github.com/google/go-containerregistry/pkg/v1/static"
|
||||
gvtypes "github.com/google/go-containerregistry/pkg/v1/types"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/rs/zerolog"
|
||||
"helm.sh/helm/v3/pkg/action"
|
||||
@@ -282,6 +283,41 @@ func seedOCI11Referrer(t *testing.T, host, repo string, baseImg gcrv1.Image, opt
|
||||
}
|
||||
}
|
||||
|
||||
// seedStoreDescriptor injects a descriptor with the given annotations directly
|
||||
// into the store index without requiring a real registry or blob. This is used
|
||||
// to pre-populate the store for rewriteReference unit tests.
|
||||
func seedStoreDescriptor(t *testing.T, s *store.Layout, annotations map[string]string) {
|
||||
t.Helper()
|
||||
desc := ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageManifest,
|
||||
Digest: digest.Digest("sha256:" + strings.Repeat("a", 64)),
|
||||
Size: 1,
|
||||
Annotations: annotations,
|
||||
}
|
||||
if err := s.OCI.AddIndex(desc); err != nil {
|
||||
t.Fatalf("seedStoreDescriptor: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// assertAnnotationsInStore walks the store and fails if no descriptor has both
|
||||
// AnnotationRefName == refName AND ContainerdImageNameKey == containerdName.
|
||||
func assertAnnotationsInStore(t *testing.T, s *store.Layout, refName, containerdName string) {
|
||||
t.Helper()
|
||||
found := false
|
||||
if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error {
|
||||
if desc.Annotations[ocispec.AnnotationRefName] == refName &&
|
||||
desc.Annotations[consts.ContainerdImageNameKey] == containerdName {
|
||||
found = true
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("assertAnnotationsInStore walk: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no artifact with AnnotationRefName=%q and ContainerdImageNameKey=%q found in store", refName, containerdName)
|
||||
}
|
||||
}
|
||||
|
||||
// assertReferrerInStore walks the store and fails if no descriptor has a kind
|
||||
// annotation with the KindAnnotationReferrers prefix and a ref containing refSubstring.
|
||||
func assertReferrerInStore(t *testing.T, s *store.Layout, refSubstring string) {
|
||||
|
||||
4
go.mod
4
go.mod
@@ -143,7 +143,7 @@ require (
|
||||
github.com/go-errors/errors v1.4.2 // indirect
|
||||
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-openapi/analysis v0.24.1 // indirect
|
||||
@@ -324,7 +324,7 @@ require (
|
||||
google.golang.org/api v0.267.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/grpc v1.78.0 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
|
||||
8
go.sum
8
go.sum
@@ -378,8 +378,8 @@ github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs
|
||||
github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
@@ -1294,8 +1294,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
|
||||
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
||||
@@ -10,6 +10,7 @@ type SaveOpts struct {
|
||||
FileName string
|
||||
Platform string
|
||||
ContainerdCompatibility bool
|
||||
ChunkSize string
|
||||
}
|
||||
|
||||
func (o *SaveOpts) AddFlags(cmd *cobra.Command) {
|
||||
@@ -18,5 +19,6 @@ func (o *SaveOpts) AddFlags(cmd *cobra.Command) {
|
||||
f.StringVarP(&o.FileName, "filename", "f", consts.DefaultHaulerArchiveName, "(Optional) Specify the name of outputted haul")
|
||||
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform for runtime imports... i.e. linux/amd64 (unspecified implies all)")
|
||||
f.BoolVar(&o.ContainerdCompatibility, "containerd", false, "(Optional) Enable import compatibility with containerd... removes oci-layout from the haul")
|
||||
f.StringVar(&o.ChunkSize, "chunk-size", "", "(Optional) Split the output archive into chunks of the specified size (e.g. 1G, 500M, 2048M)")
|
||||
|
||||
}
|
||||
|
||||
@@ -2,12 +2,12 @@ package flags
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"hauler.dev/go/hauler/pkg/consts"
|
||||
)
|
||||
|
||||
type SyncOpts struct {
|
||||
*StoreRootOpts
|
||||
FileName []string
|
||||
ImageTxt []string
|
||||
Key string
|
||||
CertOidcIssuer string
|
||||
CertOidcIssuerRegexp string
|
||||
@@ -25,7 +25,8 @@ type SyncOpts struct {
|
||||
func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
|
||||
f := cmd.Flags()
|
||||
|
||||
f.StringSliceVarP(&o.FileName, "filename", "f", []string{consts.DefaultHaulerManifestName}, "Specify the name of manifest(s) to sync")
|
||||
f.StringSliceVarP(&o.FileName, "filename", "f", []string{}, "Specify the name of manifest(s) to sync")
|
||||
f.StringSliceVarP(&o.ImageTxt, "image-txt", "i", []string{}, "Specify local or remote image.txt file(s) to sync images")
|
||||
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification")
|
||||
f.StringVar(&o.CertIdentity, "certificate-identity", "", "(Optional) Cosign certificate-identity (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")
|
||||
f.StringVar(&o.CertIdentityRegexp, "certificate-identity-regexp", "", "(Optional) Cosign certificate-identity-regexp (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")
|
||||
|
||||
@@ -3,8 +3,10 @@ package archives
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
"hauler.dev/go/hauler/pkg/log"
|
||||
@@ -102,3 +104,85 @@ func Archive(ctx context.Context, dir, outfile string, compression archives.Comp
|
||||
l.Debugf("archive created successfully [%s]", outfile)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SplitArchive splits an existing archive into chunks of at most maxBytes each.
|
||||
// Chunks are named <base>_0<ext>, <base>_1<ext>, ... where base is the archive
|
||||
// path with all extensions stripped, and ext is the compound extension (e.g. .tar.zst).
|
||||
// The original archive is removed after successful splitting.
|
||||
func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]string, error) {
|
||||
l := log.FromContext(ctx)
|
||||
|
||||
// derive base path and compound extension by stripping all extensions
|
||||
base := archivePath
|
||||
ext := ""
|
||||
for filepath.Ext(base) != "" {
|
||||
ext = filepath.Ext(base) + ext
|
||||
base = strings.TrimSuffix(base, filepath.Ext(base))
|
||||
}
|
||||
|
||||
f, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open archive for splitting: %w", err)
|
||||
}
|
||||
|
||||
var chunks []string
|
||||
buf := make([]byte, 32*1024)
|
||||
chunkIdx := 0
|
||||
var written int64
|
||||
var outf *os.File
|
||||
|
||||
for {
|
||||
if outf == nil {
|
||||
chunkPath := fmt.Sprintf("%s_%d%s", base, chunkIdx, ext)
|
||||
outf, err = os.Create(chunkPath)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("failed to create chunk %d: %w", chunkIdx, err)
|
||||
}
|
||||
chunks = append(chunks, chunkPath)
|
||||
l.Debugf("creating chunk [%s]", chunkPath)
|
||||
written = 0
|
||||
chunkIdx++
|
||||
}
|
||||
|
||||
remaining := maxBytes - written
|
||||
readSize := int64(len(buf))
|
||||
if readSize > remaining {
|
||||
readSize = remaining
|
||||
}
|
||||
|
||||
n, readErr := f.Read(buf[:readSize])
|
||||
if n > 0 {
|
||||
if _, writeErr := outf.Write(buf[:n]); writeErr != nil {
|
||||
outf.Close()
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("failed to write to chunk: %w", writeErr)
|
||||
}
|
||||
written += int64(n)
|
||||
}
|
||||
|
||||
if readErr == io.EOF {
|
||||
outf.Close()
|
||||
outf = nil
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
outf.Close()
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("failed to read archive: %w", readErr)
|
||||
}
|
||||
|
||||
if written >= maxBytes {
|
||||
outf.Close()
|
||||
outf = nil
|
||||
}
|
||||
}
|
||||
|
||||
f.Close()
|
||||
if err := os.Remove(archivePath); err != nil {
|
||||
return nil, fmt.Errorf("failed to remove original archive after splitting: %w", err)
|
||||
}
|
||||
|
||||
l.Infof("split archive [%s] into %d chunk(s)", filepath.Base(archivePath), len(chunks))
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package archives
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -162,3 +164,278 @@ func TestSecurePath(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// chunkInfo
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
func TestChunkInfo(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantBase string
|
||||
wantExt string
|
||||
wantIndex int
|
||||
wantOk bool
|
||||
}{
|
||||
{
|
||||
name: "compound extension",
|
||||
path: "/tmp/haul_3.tar.zst",
|
||||
wantBase: "/tmp/haul",
|
||||
wantExt: ".tar.zst",
|
||||
wantIndex: 3,
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "single extension",
|
||||
path: "/tmp/archive_0.zst",
|
||||
wantBase: "/tmp/archive",
|
||||
wantExt: ".zst",
|
||||
wantIndex: 0,
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "large index",
|
||||
path: "/tmp/haul_42.tar.zst",
|
||||
wantBase: "/tmp/haul",
|
||||
wantExt: ".tar.zst",
|
||||
wantIndex: 42,
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "no numeric suffix",
|
||||
path: "/tmp/haul.tar.zst",
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "alphabetic suffix",
|
||||
path: "/tmp/haul_abc.tar.zst",
|
||||
wantOk: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
base, ext, index, ok := chunkInfo(tt.path)
|
||||
if ok != tt.wantOk {
|
||||
t.Fatalf("chunkInfo() ok = %v, want %v", ok, tt.wantOk)
|
||||
}
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if base != tt.wantBase {
|
||||
t.Errorf("chunkInfo() base = %q, want %q", base, tt.wantBase)
|
||||
}
|
||||
if ext != tt.wantExt {
|
||||
t.Errorf("chunkInfo() ext = %q, want %q", ext, tt.wantExt)
|
||||
}
|
||||
if index != tt.wantIndex {
|
||||
t.Errorf("chunkInfo() index = %d, want %d", index, tt.wantIndex)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// SplitArchive
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
func TestSplitArchive(t *testing.T) {
|
||||
ctx := testContext(t)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
dataSize int
|
||||
maxBytes int64
|
||||
}{
|
||||
{name: "splits into multiple chunks", dataSize: 100, maxBytes: 30},
|
||||
{name: "single chunk when data fits", dataSize: 50, maxBytes: 100},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
archivePath := filepath.Join(dir, "haul.tar.zst")
|
||||
data := make([]byte, tt.dataSize)
|
||||
for i := range data {
|
||||
data[i] = byte(i % 256)
|
||||
}
|
||||
if err := os.WriteFile(archivePath, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
chunks, err := SplitArchive(ctx, archivePath, tt.maxBytes)
|
||||
if err != nil {
|
||||
t.Fatalf("SplitArchive() error = %v", err)
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
t.Fatal("SplitArchive() returned no chunks")
|
||||
}
|
||||
|
||||
// original archive must be removed
|
||||
if _, err := os.Stat(archivePath); !os.IsNotExist(err) {
|
||||
t.Error("original archive should be removed after splitting")
|
||||
}
|
||||
|
||||
// chunks must follow <base>_N<ext> naming
|
||||
for i, chunk := range chunks {
|
||||
expected := filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i))
|
||||
if chunk != expected {
|
||||
t.Errorf("chunk[%d] = %s, want %s", i, chunk, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// concatenating chunks must reproduce the original data
|
||||
var combined []byte
|
||||
for _, chunk := range chunks {
|
||||
b, err := os.ReadFile(chunk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
combined = append(combined, b...)
|
||||
}
|
||||
if !bytes.Equal(combined, data) {
|
||||
t.Error("combined chunks do not match original data")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitArchive_MissingFile(t *testing.T) {
|
||||
ctx := testContext(t)
|
||||
dir := t.TempDir()
|
||||
_, err := SplitArchive(ctx, filepath.Join(dir, "nonexistent.tar.zst"), 1<<30)
|
||||
if err == nil {
|
||||
t.Fatal("SplitArchive() expected error for missing file, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// JoinChunks
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
func TestJoinChunks(t *testing.T) {
|
||||
ctx := testContext(t)
|
||||
|
||||
t.Run("joins multiple chunks in order", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tempDir := t.TempDir()
|
||||
for i, content := range []string{"chunk0-data", "chunk1-data", "chunk2-data"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("haul_%d.tar.zst", i)), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir)
|
||||
if err != nil {
|
||||
t.Fatalf("JoinChunks() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []byte("chunk0-datachunk1-datachunk2-data"); !bytes.Equal(data, want) {
|
||||
t.Errorf("JoinChunks() content = %q, want %q", data, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("any chunk triggers full assembly", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tempDir := t.TempDir()
|
||||
for i, content := range []string{"aaa", "bbb"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("data_%d.tar.zst", i)), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// pass chunk_1, not chunk_0 — should still assemble from chunk_0
|
||||
got, err := JoinChunks(ctx, filepath.Join(dir, "data_1.tar.zst"), tempDir)
|
||||
if err != nil {
|
||||
t.Fatalf("JoinChunks() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []byte("aaabbb"); !bytes.Equal(data, want) {
|
||||
t.Errorf("JoinChunks() content = %q, want %q", data, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-chunk file returned unchanged", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
nonChunk := filepath.Join(dir, "haul.tar.zst")
|
||||
if err := os.WriteFile(nonChunk, []byte("not-a-chunk"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := JoinChunks(ctx, nonChunk, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("JoinChunks() error = %v", err)
|
||||
}
|
||||
if got != nonChunk {
|
||||
t.Errorf("JoinChunks() = %s, want %s (unchanged)", got, nonChunk)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-numeric suffix files excluded", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tempDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "haul_0.tar.zst"), []byte("valid"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// glob matches this but chunkInfo rejects it
|
||||
if err := os.WriteFile(filepath.Join(dir, "haul_foo.tar.zst"), []byte("invalid"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := JoinChunks(ctx, filepath.Join(dir, "haul_0.tar.zst"), tempDir)
|
||||
if err != nil {
|
||||
t.Fatalf("JoinChunks() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(data, []byte("valid")) {
|
||||
t.Errorf("JoinChunks() included non-numeric suffix file; content = %q", data)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// SplitArchive + JoinChunks round-trip
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
func TestSplitJoinChunks_RoundTrip(t *testing.T) {
|
||||
ctx := testContext(t)
|
||||
|
||||
original := make([]byte, 1000)
|
||||
for i := range original {
|
||||
original[i] = byte(i % 256)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
archivePath := filepath.Join(dir, "haul.tar.zst")
|
||||
if err := os.WriteFile(archivePath, original, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
chunks, err := SplitArchive(ctx, archivePath, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("SplitArchive() error = %v", err)
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
t.Fatal("SplitArchive() returned no chunks")
|
||||
}
|
||||
|
||||
joined, err := JoinChunks(ctx, chunks[0], t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("JoinChunks() error = %v", err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(joined)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, original) {
|
||||
t.Error("round-trip: joined data does not match original")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,9 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
@@ -156,3 +159,86 @@ func Unarchive(ctx context.Context, tarball, dst string) error {
|
||||
l.Infof("unarchiving completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
var chunkSuffixRe = regexp.MustCompile(`^(.+)_(\d+)$`)
|
||||
|
||||
// chunkInfo checks whether archivePath matches the chunk naming pattern (<base>_N<ext>).
|
||||
// Returns the base path (without index), compound extension, numeric index, and whether it matched.
|
||||
func chunkInfo(archivePath string) (base, ext string, index int, ok bool) {
|
||||
dir := filepath.Dir(archivePath)
|
||||
name := filepath.Base(archivePath)
|
||||
|
||||
// strip compound extension (e.g. .tar.zst)
|
||||
nameBase := name
|
||||
nameExt := ""
|
||||
for filepath.Ext(nameBase) != "" {
|
||||
nameExt = filepath.Ext(nameBase) + nameExt
|
||||
nameBase = strings.TrimSuffix(nameBase, filepath.Ext(nameBase))
|
||||
}
|
||||
|
||||
m := chunkSuffixRe.FindStringSubmatch(nameBase)
|
||||
if m == nil {
|
||||
return "", "", 0, false
|
||||
}
|
||||
|
||||
idx, _ := strconv.Atoi(m[2])
|
||||
return filepath.Join(dir, m[1]), nameExt, idx, true
|
||||
}
|
||||
|
||||
// JoinChunks detects whether archivePath is a chunk file and, if so, finds all
|
||||
// sibling chunks, concatenates them in numeric order into a single file in tempDir,
|
||||
// and returns the path to the joined file. If archivePath is not a chunk, it is
|
||||
// returned unchanged.
|
||||
func JoinChunks(ctx context.Context, archivePath, tempDir string) (string, error) {
|
||||
l := log.FromContext(ctx)
|
||||
|
||||
base, ext, _, ok := chunkInfo(archivePath)
|
||||
if !ok {
|
||||
return archivePath, nil
|
||||
}
|
||||
|
||||
all, err := filepath.Glob(base + "_*" + ext)
|
||||
if err != nil {
|
||||
return archivePath, nil
|
||||
}
|
||||
var matches []string
|
||||
for _, m := range all {
|
||||
if _, _, _, ok := chunkInfo(m); ok {
|
||||
matches = append(matches, m)
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return archivePath, nil
|
||||
}
|
||||
|
||||
sort.Slice(matches, func(i, j int) bool {
|
||||
_, _, idxI, _ := chunkInfo(matches[i])
|
||||
_, _, idxJ, _ := chunkInfo(matches[j])
|
||||
return idxI < idxJ
|
||||
})
|
||||
|
||||
l.Debugf("joining %d chunk(s) for [%s]", len(matches), base)
|
||||
|
||||
joinedPath := filepath.Join(tempDir, filepath.Base(base)+ext)
|
||||
outf, err := os.Create(joinedPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create joined archive: %w", err)
|
||||
}
|
||||
defer outf.Close()
|
||||
|
||||
for _, chunk := range matches {
|
||||
l.Debugf("joining chunk [%s]", chunk)
|
||||
cf, err := os.Open(chunk)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open chunk [%s]: %w", chunk, err)
|
||||
}
|
||||
if _, err := io.Copy(outf, cf); err != nil {
|
||||
cf.Close()
|
||||
return "", fmt.Errorf("failed to copy chunk [%s]: %w", chunk, err)
|
||||
}
|
||||
cf.Close()
|
||||
}
|
||||
|
||||
l.Infof("joined %d chunk(s) into [%s]", len(matches), filepath.Base(joinedPath))
|
||||
return joinedPath, nil
|
||||
}
|
||||
|
||||
@@ -44,16 +44,16 @@ const (
|
||||
// annotation keys
|
||||
ContainerdImageNameKey = "io.containerd.image.name"
|
||||
KindAnnotationName = "kind"
|
||||
KindAnnotationImage = "dev.cosignproject.cosign/image"
|
||||
KindAnnotationIndex = "dev.cosignproject.cosign/imageIndex"
|
||||
KindAnnotationSigs = "dev.cosignproject.cosign/sigs"
|
||||
KindAnnotationAtts = "dev.cosignproject.cosign/atts"
|
||||
KindAnnotationSboms = "dev.cosignproject.cosign/sboms"
|
||||
KindAnnotationImage = "dev.hauler/image"
|
||||
KindAnnotationIndex = "dev.hauler/imageIndex"
|
||||
KindAnnotationSigs = "dev.hauler/sigs"
|
||||
KindAnnotationAtts = "dev.hauler/atts"
|
||||
KindAnnotationSboms = "dev.hauler/sboms"
|
||||
// KindAnnotationReferrers is the kind prefix for OCI 1.1 referrer manifests (cosign v3
|
||||
// new-bundle-format). Each referrer gets a unique kind with the referrer manifest digest
|
||||
// appended (e.g. "dev.cosignproject.cosign/referrers/sha256hex") so multiple referrers
|
||||
// for the same base image coexist in the OCI index.
|
||||
KindAnnotationReferrers = "dev.cosignproject.cosign/referrers"
|
||||
// appended (e.g. "dev.hauler/referrers/sha256hex") so multiple referrers for the same
|
||||
// base image coexist in the OCI index.
|
||||
KindAnnotationReferrers = "dev.hauler/referrers"
|
||||
|
||||
// Sigstore / OCI 1.1 artifact media types used by cosign v3 new-bundle-format.
|
||||
SigstoreBundleMediaType = "application/vnd.dev.sigstore.bundle.v0.3+json"
|
||||
|
||||
19
pkg/consts/migrate.go
Normal file
19
pkg/consts/migrate.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package consts
|
||||
|
||||
import "strings"
|
||||
|
||||
// NormalizeLegacyKind translates old dev.cosignproject.cosign kind annotation
|
||||
// values to their dev.hauler equivalents. Returns the input unchanged if it is
|
||||
// already a current value or empty.
|
||||
//
|
||||
// This handles all cases including the dynamic referrer suffix:
|
||||
//
|
||||
// dev.cosignproject.cosign/referrers/<sha256hex> → dev.hauler/referrers/<sha256hex>
|
||||
func NormalizeLegacyKind(kind string) string {
|
||||
const oldPrefix = "dev.cosignproject.cosign"
|
||||
const newPrefix = "dev.hauler"
|
||||
if strings.HasPrefix(kind, oldPrefix) {
|
||||
return newPrefix + kind[len(oldPrefix):]
|
||||
}
|
||||
return kind
|
||||
}
|
||||
30
pkg/consts/migrate_test.go
Normal file
30
pkg/consts/migrate_test.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package consts
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeLegacyKind(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
// Old dev.cosignproject.cosign values → new dev.hauler equivalents
|
||||
{"dev.cosignproject.cosign/image", "dev.hauler/image"},
|
||||
{"dev.cosignproject.cosign/imageIndex", "dev.hauler/imageIndex"},
|
||||
{"dev.cosignproject.cosign/sigs", "dev.hauler/sigs"},
|
||||
{"dev.cosignproject.cosign/atts", "dev.hauler/atts"},
|
||||
{"dev.cosignproject.cosign/sboms", "dev.hauler/sboms"},
|
||||
{"dev.cosignproject.cosign/referrers/abc123def456", "dev.hauler/referrers/abc123def456"},
|
||||
// Already-new values pass through unchanged
|
||||
{"dev.hauler/image", "dev.hauler/image"},
|
||||
{"dev.hauler/imageIndex", "dev.hauler/imageIndex"},
|
||||
{"dev.hauler/referrers/abc123", "dev.hauler/referrers/abc123"},
|
||||
// Empty string passes through unchanged
|
||||
{"", ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := NormalizeLegacyKind(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("NormalizeLegacyKind(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -91,12 +92,21 @@ func (o *OCI) LoadIndex() error {
|
||||
continue
|
||||
}
|
||||
|
||||
// Set default kind if missing
|
||||
// Set default kind if missing; normalize legacy dev.cosignproject.cosign values.
|
||||
kind := desc.Annotations[consts.KindAnnotationName]
|
||||
kind = consts.NormalizeLegacyKind(kind)
|
||||
if kind == "" {
|
||||
kind = consts.KindAnnotationImage
|
||||
}
|
||||
|
||||
// Write the normalized kind back into a copy of the annotations map so
|
||||
// that Walk() callers receive descriptors with dev.hauler/... values.
|
||||
// We copy the map to avoid mutating the slice element's shared map.
|
||||
normalized := make(map[string]string, len(desc.Annotations)+1)
|
||||
maps.Copy(normalized, desc.Annotations)
|
||||
normalized[consts.KindAnnotationName] = kind
|
||||
desc.Annotations = normalized
|
||||
|
||||
if strings.TrimSpace(key.String()) != "--" {
|
||||
switch key.(type) {
|
||||
case name.Digest:
|
||||
@@ -130,7 +140,7 @@ func (o *OCI) SaveIndex() error {
|
||||
kindI := descs[i].Annotations["kind"]
|
||||
kindJ := descs[j].Annotations["kind"]
|
||||
|
||||
// Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top.
|
||||
// Objects with the prefix of KindAnnotationImage should be at the top.
|
||||
if strings.HasPrefix(kindI, consts.KindAnnotationImage) && !strings.HasPrefix(kindJ, consts.KindAnnotationImage) {
|
||||
return true
|
||||
} else if !strings.HasPrefix(kindI, consts.KindAnnotationImage) && strings.HasPrefix(kindJ, consts.KindAnnotationImage) {
|
||||
@@ -299,11 +309,18 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr
|
||||
if err := p.oci.LoadIndex(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Use compound key format: "reference-kind"
|
||||
// Use compound key format: "reference-kind"; normalize legacy values.
|
||||
kind := d.Annotations[consts.KindAnnotationName]
|
||||
kind = consts.NormalizeLegacyKind(kind)
|
||||
if kind == "" {
|
||||
kind = consts.KindAnnotationImage
|
||||
}
|
||||
// Copy annotations map to avoid mutating the caller's descriptor,
|
||||
// then write the normalized kind so Walk() callers see dev.hauler/... values.
|
||||
normalizedAnnotations := make(map[string]string, len(d.Annotations)+1)
|
||||
maps.Copy(normalizedAnnotations, d.Annotations)
|
||||
normalizedAnnotations[consts.KindAnnotationName] = kind
|
||||
d.Annotations = normalizedAnnotations
|
||||
key := fmt.Sprintf("%s-%s", p.ref, kind)
|
||||
p.oci.nameMap.Store(key, d)
|
||||
if err := p.oci.SaveIndex(); err != nil {
|
||||
|
||||
281
pkg/content/oci_test.go
Normal file
281
pkg/content/oci_test.go
Normal file
@@ -0,0 +1,281 @@
|
||||
package content
|
||||
|
||||
// oci_test.go covers the annotation-normalization correctness of LoadIndex()
|
||||
// and ociPusher.Push(). Specifically, it verifies that descriptors returned
|
||||
// by Walk() carry the normalized dev.hauler/... kind annotation value, not the
|
||||
// legacy dev.cosignproject.cosign/... value that may be present on disk.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
"github.com/opencontainers/image-spec/specs-go"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
||||
"hauler.dev/go/hauler/pkg/consts"
|
||||
)
|
||||
|
||||
// buildMinimalOCILayout writes the smallest valid OCI layout (oci-layout marker
|
||||
// + index.json with the supplied descriptors) into dir. No blobs are written;
|
||||
// this is sufficient for testing LoadIndex/Walk without a full store.
|
||||
func buildMinimalOCILayout(t *testing.T, dir string, manifests []ocispec.Descriptor) {
|
||||
t.Helper()
|
||||
|
||||
// oci-layout marker
|
||||
layoutMarker := map[string]string{"imageLayoutVersion": "1.0.0"}
|
||||
markerData, err := json.Marshal(layoutMarker)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal oci-layout: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, ocispec.ImageLayoutFile), markerData, 0644); err != nil {
|
||||
t.Fatalf("write oci-layout: %v", err)
|
||||
}
|
||||
|
||||
// index.json
|
||||
idx := ocispec.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ocispec.MediaTypeImageIndex,
|
||||
Manifests: manifests,
|
||||
}
|
||||
data, err := json.MarshalIndent(idx, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal index.json: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, ocispec.ImageIndexFile), data, 0644); err != nil {
|
||||
t.Fatalf("write index.json: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeDigest returns a syntactically valid digest string that can be used in
|
||||
// test descriptors without any real blob.
|
||||
func fakeDigest(hex string) digest.Digest {
|
||||
// pad hex to 64 chars
|
||||
for len(hex) < 64 {
|
||||
hex += "0"
|
||||
}
|
||||
return digest.Digest("sha256:" + hex)
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations verifies that
|
||||
// after LoadIndex() (called implicitly by Walk()), every descriptor returned
|
||||
// by Walk carries a normalized dev.hauler/... kind annotation, not the legacy
|
||||
// dev.cosignproject.cosign/... value stored on disk.
|
||||
func TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
legacyKinds := []string{
|
||||
"dev.cosignproject.cosign/image",
|
||||
"dev.cosignproject.cosign/imageIndex",
|
||||
"dev.cosignproject.cosign/sigs",
|
||||
"dev.cosignproject.cosign/atts",
|
||||
"dev.cosignproject.cosign/sboms",
|
||||
}
|
||||
|
||||
var manifests []ocispec.Descriptor
|
||||
for i, legacyKind := range legacyKinds {
|
||||
d := ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageManifest,
|
||||
Digest: fakeDigest(strings.Repeat(string(rune('a'+i)), 1)),
|
||||
Size: 100,
|
||||
Annotations: map[string]string{
|
||||
ocispec.AnnotationRefName: "example.com/repo:tag" + strings.Repeat(string(rune('a'+i)), 1),
|
||||
consts.KindAnnotationName: legacyKind,
|
||||
},
|
||||
}
|
||||
manifests = append(manifests, d)
|
||||
}
|
||||
|
||||
buildMinimalOCILayout(t, dir, manifests)
|
||||
|
||||
o, err := NewOCI(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewOCI: %v", err)
|
||||
}
|
||||
|
||||
var walked []ocispec.Descriptor
|
||||
if err := o.Walk(func(_ string, desc ocispec.Descriptor) error {
|
||||
walked = append(walked, desc)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
|
||||
if len(walked) == 0 {
|
||||
t.Fatal("Walk returned no descriptors")
|
||||
}
|
||||
|
||||
const legacyPrefix = "dev.cosignproject.cosign"
|
||||
const newPrefix = "dev.hauler"
|
||||
for _, desc := range walked {
|
||||
kind := desc.Annotations[consts.KindAnnotationName]
|
||||
if strings.HasPrefix(kind, legacyPrefix) {
|
||||
t.Errorf("descriptor %s: Walk returned legacy kind %q; want normalized dev.hauler/... value",
|
||||
desc.Digest, kind)
|
||||
}
|
||||
if !strings.HasPrefix(kind, newPrefix) {
|
||||
t.Errorf("descriptor %s: Walk returned unexpected kind %q; want dev.hauler/... prefix",
|
||||
desc.Digest, kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// TestLoadIndex_DoesNotMutateOnDiskAnnotations
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// TestLoadIndex_DoesNotMutateOnDiskAnnotations verifies that the normalization
|
||||
// performed by LoadIndex() is in-memory only: the index.json on disk must
|
||||
// still carry the original (legacy) annotation values after a Walk() call.
|
||||
func TestLoadIndex_DoesNotMutateOnDiskAnnotations(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
legacyKind := "dev.cosignproject.cosign/image"
|
||||
manifests := []ocispec.Descriptor{
|
||||
{
|
||||
MediaType: ocispec.MediaTypeImageManifest,
|
||||
Digest: fakeDigest("b"),
|
||||
Size: 100,
|
||||
Annotations: map[string]string{
|
||||
ocispec.AnnotationRefName: "example.com/repo:tagb",
|
||||
consts.KindAnnotationName: legacyKind,
|
||||
},
|
||||
},
|
||||
}
|
||||
buildMinimalOCILayout(t, dir, manifests)
|
||||
|
||||
o, err := NewOCI(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewOCI: %v", err)
|
||||
}
|
||||
// Trigger LoadIndex via Walk.
|
||||
if err := o.Walk(func(_ string, _ ocispec.Descriptor) error { return nil }); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
|
||||
// Re-read index.json from disk and verify the annotation is unchanged.
|
||||
data, err := os.ReadFile(filepath.Join(dir, ocispec.ImageIndexFile))
|
||||
if err != nil {
|
||||
t.Fatalf("read index.json: %v", err)
|
||||
}
|
||||
var idx ocispec.Index
|
||||
if err := json.Unmarshal(data, &idx); err != nil {
|
||||
t.Fatalf("unmarshal index.json: %v", err)
|
||||
}
|
||||
for _, desc := range idx.Manifests {
|
||||
got := desc.Annotations[consts.KindAnnotationName]
|
||||
if got != legacyKind {
|
||||
t.Errorf("on-disk kind was mutated: got %q, want %q", got, legacyKind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// TestPush_NormalizesLegacyKindInStoredDescriptor
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// TestPush_NormalizesLegacyKindInStoredDescriptor verifies that after a Push()
|
||||
// that matches the root digest, the descriptor stored in nameMap (and therefore
|
||||
// returned by subsequent Walk() calls) carries the normalized dev.hauler/...
|
||||
// kind annotation rather than the legacy value.
|
||||
func TestPush_NormalizesLegacyKindInStoredDescriptor(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
buildMinimalOCILayout(t, dir, nil) // start with empty index
|
||||
|
||||
o, err := NewOCI(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewOCI: %v", err)
|
||||
}
|
||||
|
||||
// Build a minimal manifest blob so Push() can write it to disk.
|
||||
manifest := ocispec.Manifest{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ocispec.MediaTypeImageManifest,
|
||||
Config: ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageConfig,
|
||||
Digest: fakeDigest("config0"),
|
||||
Size: 2,
|
||||
},
|
||||
}
|
||||
manifestData, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal manifest: %v", err)
|
||||
}
|
||||
manifestDigest := digest.FromBytes(manifestData)
|
||||
|
||||
// Ensure the blobs directory exists so Push can write.
|
||||
blobsDir := filepath.Join(dir, ocispec.ImageBlobsDir, "sha256")
|
||||
if err := os.MkdirAll(blobsDir, 0755); err != nil {
|
||||
t.Fatalf("mkdir blobs: %v", err)
|
||||
}
|
||||
|
||||
legacyKind := "dev.cosignproject.cosign/sigs"
|
||||
baseRef := "example.com/repo:tagsig"
|
||||
|
||||
pusher, err := o.Pusher(context.Background(), baseRef+"@"+manifestDigest.String())
|
||||
if err != nil {
|
||||
t.Fatalf("Pusher: %v", err)
|
||||
}
|
||||
|
||||
desc := ocispec.Descriptor{
|
||||
MediaType: ocispec.MediaTypeImageManifest,
|
||||
Digest: manifestDigest,
|
||||
Size: int64(len(manifestData)),
|
||||
Annotations: map[string]string{
|
||||
ocispec.AnnotationRefName: baseRef,
|
||||
consts.KindAnnotationName: legacyKind,
|
||||
},
|
||||
}
|
||||
|
||||
w, err := pusher.Push(context.Background(), desc)
|
||||
if err != nil {
|
||||
t.Fatalf("Push: %v", err)
|
||||
}
|
||||
if _, err := w.Write(manifestData); err != nil {
|
||||
t.Fatalf("Write manifest: %v", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatalf("Close writer: %v", err)
|
||||
}
|
||||
|
||||
// Now Walk and verify the descriptor in nameMap has the normalized kind.
|
||||
// We need a fresh OCI instance so Walk calls LoadIndex (which reads SaveIndex output).
|
||||
o2, err := NewOCI(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewOCI second: %v", err)
|
||||
}
|
||||
|
||||
const legacyPrefix = "dev.cosignproject.cosign"
|
||||
const newPrefix = "dev.hauler"
|
||||
var found bool
|
||||
if err := o2.Walk(func(_ string, d ocispec.Descriptor) error {
|
||||
found = true
|
||||
kind := d.Annotations[consts.KindAnnotationName]
|
||||
if strings.HasPrefix(kind, legacyPrefix) {
|
||||
t.Errorf("Push stored descriptor with legacy kind %q; want normalized dev.hauler/... value", kind)
|
||||
}
|
||||
if !strings.HasPrefix(kind, newPrefix) {
|
||||
t.Errorf("Push stored descriptor with unexpected kind %q; want dev.hauler/... prefix", kind)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("Walk returned no descriptors after Push")
|
||||
}
|
||||
|
||||
// Also verify the caller's original descriptor map was NOT mutated.
|
||||
if desc.Annotations[consts.KindAnnotationName] != legacyKind {
|
||||
t.Errorf("Push mutated caller's descriptor annotations: got %q, want %q",
|
||||
desc.Annotations[consts.KindAnnotationName], legacyKind)
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,10 @@ func NewRegistryTarget(host string, opts RegistryOptions) *RegistryTarget {
|
||||
)
|
||||
|
||||
hosts := func(h string) ([]cdocker.RegistryHost, error) {
|
||||
host, err := cdocker.DefaultHost(h)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scheme := "https"
|
||||
if opts.PlainHTTP || opts.Insecure {
|
||||
scheme = "http"
|
||||
@@ -55,7 +59,7 @@ func NewRegistryTarget(host string, opts RegistryOptions) *RegistryTarget {
|
||||
Client: http.DefaultClient,
|
||||
Authorizer: authorizer,
|
||||
Scheme: scheme,
|
||||
Host: h,
|
||||
Host: host,
|
||||
Path: "/v2",
|
||||
Capabilities: cdocker.HostCapabilityPull | cdocker.HostCapabilityResolve | cdocker.HostCapabilityPush,
|
||||
}}, nil
|
||||
|
||||
4
testdata/images.txt
vendored
Normal file
4
testdata/images.txt
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# hauler image list
|
||||
# one image reference per line; blank lines and comments are ignored
|
||||
ghcr.io/hauler-dev/library/busybox
|
||||
ghcr.io/hauler-dev/library/busybox:stable
|
||||
Reference in New Issue
Block a user