mirror of
https://github.com/hauler-dev/hauler.git
synced 2026-03-17 09:00:18 +00:00
Compare commits
1 Commits
chunk-the-
...
cherrypick
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e59beb08d0 |
298
.github/workflows/cherrypick.yml
vendored
Normal file
298
.github/workflows/cherrypick.yml
vendored
Normal file
@@ -0,0 +1,298 @@
|
||||
name: Cherry-pick to release branch
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request:
|
||||
types: [closed]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# Trigger 1: /cherrypick-X.Y comment on a PR
|
||||
# - If already merged → run cherry-pick immediately
|
||||
# - If not yet merged → add label, cherry-pick will run on merge
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
handle-comment:
|
||||
if: >
|
||||
github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request &&
|
||||
startsWith(github.event.comment.body, '/cherrypick-')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check commenter permissions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COMMENTER: ${{ github.event.comment.user.login }}
|
||||
run: |
|
||||
PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/${COMMENTER}/permission \
|
||||
--jq '.permission')
|
||||
echo "Permission level for $COMMENTER: $PERMISSION"
|
||||
if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" && "$PERMISSION" != "write" ]]; then
|
||||
echo "::warning::User $COMMENTER does not have write access, ignoring cherry-pick request"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Parse version from comment
|
||||
id: parse
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: |
|
||||
VERSION=$(echo "$COMMENT_BODY" | head -1 | grep -oP '(?<=/cherrypick-)\d+\.\d+')
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::Could not parse version from comment"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "target_branch=release/$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "label=cherrypick/$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: React to comment
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
|
||||
-f content='+1'
|
||||
|
||||
- name: Check if PR is merged
|
||||
id: check
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }})
|
||||
MERGED=$(echo "$PR_JSON" | jq -r '.merged')
|
||||
echo "merged=$MERGED" >> "$GITHUB_OUTPUT"
|
||||
echo "pr_title=$(echo "$PR_JSON" | jq -r '.title')" >> "$GITHUB_OUTPUT"
|
||||
echo "base_sha=$(echo "$PR_JSON" | jq -r '.base.sha')" >> "$GITHUB_OUTPUT"
|
||||
echo "head_sha=$(echo "$PR_JSON" | jq -r '.head.sha')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Add cherry-pick label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
LABEL: ${{ steps.parse.outputs.label }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/labels \
|
||||
-f name="$LABEL" -f color="fbca04" -f description="Queued for cherry-pick" 2>/dev/null || true
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/labels \
|
||||
-f "labels[]=$LABEL"
|
||||
|
||||
- name: Notify if queued (not yet merged)
|
||||
if: steps.check.outputs.merged != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
LABEL: ${{ steps.parse.outputs.label }}
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="🏷️ Labeled \`$LABEL\` — backport to \`$TARGET_BRANCH\` will be created automatically when this PR is merged."
|
||||
|
||||
- name: Checkout repository
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Verify target branch exists
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
env:
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null 2>&1; then
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="❌ Cannot cherry-pick: branch \`$TARGET_BRANCH\` does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Apply PR diff and push
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
id: apply
|
||||
env:
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BACKPORT_BRANCH="backport/${PR_NUMBER}-to-${TARGET_BRANCH//\//-}"
|
||||
echo "backport_branch=$BACKPORT_BRANCH" >> "$GITHUB_OUTPUT"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
git checkout "$TARGET_BRANCH"
|
||||
git checkout -b "$BACKPORT_BRANCH"
|
||||
|
||||
# Download the PR's patch from GitHub (pure diff of the PR's changes)
|
||||
gh api repos/${{ github.repository }}/pulls/${PR_NUMBER} \
|
||||
-H "Accept: application/vnd.github.v3.patch" > /tmp/pr.patch
|
||||
|
||||
# Apply the patch
|
||||
HAS_CONFLICTS="false"
|
||||
CONFLICTED_FILES=""
|
||||
|
||||
if git apply --check /tmp/pr.patch 2>/dev/null; then
|
||||
# Clean apply
|
||||
git apply /tmp/pr.patch
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}"
|
||||
elif git apply --3way /tmp/pr.patch; then
|
||||
# Applied with 3-way merge (auto-resolved)
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}" || true
|
||||
else
|
||||
# Has real conflicts — apply what we can
|
||||
HAS_CONFLICTS="true"
|
||||
CONFLICTED_FILES=$(git diff --name-only --diff-filter=U | tr '\n' ',' | sed 's/,$//')
|
||||
# Take the incoming version for conflicted files
|
||||
git diff --name-only --diff-filter=U | while read -r file; do
|
||||
git checkout --theirs -- "$file"
|
||||
done
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH} (conflicts)" || true
|
||||
fi
|
||||
|
||||
echo "has_conflicts=$HAS_CONFLICTS" >> "$GITHUB_OUTPUT"
|
||||
echo "conflicted_files=$CONFLICTED_FILES" >> "$GITHUB_OUTPUT"
|
||||
|
||||
git push origin "$BACKPORT_BRANCH"
|
||||
|
||||
- name: Create backport PR
|
||||
if: steps.check.outputs.merged == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TARGET_BRANCH: ${{ steps.parse.outputs.target_branch }}
|
||||
VERSION: ${{ steps.parse.outputs.version }}
|
||||
PR_TITLE: ${{ steps.check.outputs.pr_title }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
BACKPORT_BRANCH: ${{ steps.apply.outputs.backport_branch }}
|
||||
run: |
|
||||
TITLE="[${VERSION}] ${PR_TITLE}"
|
||||
BODY="Backport of #${PR_NUMBER} to \`${TARGET_BRANCH}\`."
|
||||
|
||||
PR_URL=$(gh pr create \
|
||||
--base "$TARGET_BRANCH" \
|
||||
--head "$BACKPORT_BRANCH" \
|
||||
--title "$TITLE" \
|
||||
--body "$BODY")
|
||||
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="✅ Backport PR created: ${PR_URL}"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# Trigger 2: PR merged → process any queued cherrypick/* labels
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
handle-merge:
|
||||
if: >
|
||||
github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.merged == true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Collect cherry-pick labels
|
||||
id: labels
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
LABELS=$(gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/labels \
|
||||
--jq '[.[] | select(.name | startswith("cherrypick/")) | .name] | join(",")')
|
||||
|
||||
if [ -z "$LABELS" ]; then
|
||||
echo "No cherrypick labels found, nothing to do."
|
||||
echo "has_labels=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "Found labels: $LABELS"
|
||||
echo "has_labels=true" >> "$GITHUB_OUTPUT"
|
||||
echo "labels=$LABELS" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
if: steps.labels.outputs.has_labels == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download PR patch
|
||||
if: steps.labels.outputs.has_labels == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
gh api repos/${{ github.repository }}/pulls/${PR_NUMBER} \
|
||||
-H "Accept: application/vnd.github.v3.patch" > /tmp/pr.patch
|
||||
|
||||
- name: Process each cherry-pick label
|
||||
if: steps.labels.outputs.has_labels == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
LABELS: ${{ steps.labels.outputs.labels }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
IFS=',' read -ra LABEL_ARRAY <<< "$LABELS"
|
||||
for LABEL in "${LABEL_ARRAY[@]}"; do
|
||||
VERSION="${LABEL#cherrypick/}"
|
||||
TARGET_BRANCH="release/$VERSION"
|
||||
|
||||
echo "=== Processing backport to $TARGET_BRANCH ==="
|
||||
|
||||
# Verify target branch exists
|
||||
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null 2>&1; then
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="❌ Cannot cherry-pick to \`$TARGET_BRANCH\`: branch does not exist."
|
||||
continue
|
||||
fi
|
||||
|
||||
BACKPORT_BRANCH="backport/${PR_NUMBER}-to-${TARGET_BRANCH//\//-}"
|
||||
|
||||
git checkout "$TARGET_BRANCH"
|
||||
git checkout -b "$BACKPORT_BRANCH"
|
||||
|
||||
# Apply the patch
|
||||
HAS_CONFLICTS="false"
|
||||
CONFLICTED_FILES=""
|
||||
|
||||
if git apply --check /tmp/pr.patch 2>/dev/null; then
|
||||
git apply /tmp/pr.patch
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}"
|
||||
elif git apply --3way /tmp/pr.patch; then
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH}" || true
|
||||
else
|
||||
HAS_CONFLICTS="true"
|
||||
CONFLICTED_FILES=$(git diff --name-only --diff-filter=U | tr '\n' ',' | sed 's/,$//')
|
||||
git diff --name-only --diff-filter=U | while read -r file; do
|
||||
git checkout --theirs -- "$file"
|
||||
done
|
||||
git add -A
|
||||
git commit -m "Backport PR #${PR_NUMBER} to ${TARGET_BRANCH} (conflicts)" || true
|
||||
fi
|
||||
|
||||
git push origin "$BACKPORT_BRANCH"
|
||||
|
||||
# Build PR title and body
|
||||
TITLE="[${VERSION}] ${PR_TITLE}"
|
||||
BODY="Backport of #${PR_NUMBER} to \`${TARGET_BRANCH}\`."
|
||||
|
||||
PR_URL=$(gh pr create \
|
||||
--base "$TARGET_BRANCH" \
|
||||
--head "$BACKPORT_BRANCH" \
|
||||
--title "$TITLE" \
|
||||
--body "$BODY")
|
||||
|
||||
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
|
||||
-f body="✅ Backport PR to \`$TARGET_BRANCH\` created: ${PR_URL}"
|
||||
|
||||
# Clean up for next iteration
|
||||
git checkout "$TARGET_BRANCH"
|
||||
git branch -D "$BACKPORT_BRANCH"
|
||||
done
|
||||
17
.github/workflows/tests.yaml
vendored
17
.github/workflows/tests.yaml
vendored
@@ -250,13 +250,6 @@ jobs:
|
||||
hauler store save --filename store.tar.zst
|
||||
# verify via save with filename and platform (amd64)
|
||||
hauler store save --filename store-amd64.tar.zst --platform linux/amd64
|
||||
# verify via save with chunk-size (splits into haul-chunked_0.tar.zst, haul-chunked_1.tar.zst, ...)
|
||||
hauler store save --filename haul-chunked.tar.zst --chunk-size 50M
|
||||
# verify chunk files exist and original is removed
|
||||
ls haul-chunked_*.tar.zst
|
||||
! test -f haul-chunked.tar.zst
|
||||
# verify at least two chunks were produced
|
||||
[ $(ls haul-chunked_*.tar.zst | wc -l) -ge 2 ]
|
||||
|
||||
- name: Remove Hauler Store Contents
|
||||
run: |
|
||||
@@ -276,14 +269,6 @@ jobs:
|
||||
hauler store load --filename store.tar.zst --tempdir /opt
|
||||
# verify via load with filename and platform (amd64)
|
||||
hauler store load --filename store-amd64.tar.zst
|
||||
# verify via load from chunks using explicit first chunk
|
||||
rm -rf store
|
||||
hauler store load --filename haul-chunked_0.tar.zst
|
||||
hauler store info
|
||||
# verify via load from chunks using base filename (auto-detect)
|
||||
rm -rf store
|
||||
hauler store load --filename haul-chunked.tar.zst
|
||||
hauler store info
|
||||
|
||||
- name: Verify Hauler Store Contents
|
||||
run: |
|
||||
@@ -306,7 +291,7 @@ jobs:
|
||||
|
||||
- name: Remove Hauler Store Contents
|
||||
run: |
|
||||
rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst haul-chunked_*.tar.zst
|
||||
rm -rf store haul.tar.zst store.tar.zst store-amd64.tar.zst
|
||||
hauler store info
|
||||
|
||||
- name: Verify - hauler store sync
|
||||
|
||||
@@ -39,9 +39,8 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r
|
||||
l.Debugf("using temporary directory at [%s]", tempDir)
|
||||
|
||||
for _, fileName := range o.FileName {
|
||||
resolved := resolveHaulPath(fileName)
|
||||
l.Infof("loading haul [%s] to [%s]", resolved, o.StoreDir)
|
||||
err := unarchiveLayoutTo(ctx, resolved, o.StoreDir, tempDir)
|
||||
l.Infof("loading haul [%s] to [%s]", fileName, o.StoreDir)
|
||||
err := unarchiveLayoutTo(ctx, fileName, o.StoreDir, tempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -86,13 +85,6 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
|
||||
}
|
||||
}
|
||||
|
||||
// reassemble chunk files if haulPath matches the chunk naming pattern
|
||||
joined, err := archives.JoinChunks(ctx, haulPath, tempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
haulPath = joined
|
||||
|
||||
if err := archives.Unarchive(ctx, haulPath, tempDir); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -147,29 +139,6 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
|
||||
return err
|
||||
}
|
||||
|
||||
// resolveHaulPath returns path as-is if it exists or is a URL. If the file is
|
||||
// not found, it globs for chunk files matching <base>_*<ext> in the same
|
||||
// directory and returns the first match so JoinChunks can reassemble them.
|
||||
func resolveHaulPath(path string) string {
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
return path
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path
|
||||
}
|
||||
base := path
|
||||
ext := ""
|
||||
for filepath.Ext(base) != "" {
|
||||
ext = filepath.Ext(base) + ext
|
||||
base = strings.TrimSuffix(base, filepath.Ext(base))
|
||||
}
|
||||
matches, err := filepath.Glob(base + "_*" + ext)
|
||||
if err != nil || len(matches) == 0 {
|
||||
return path
|
||||
}
|
||||
return matches[0]
|
||||
}
|
||||
|
||||
func clearDir(path string) error {
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,13 +4,10 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
referencev3 "github.com/distribution/distribution/v3/reference"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
@@ -75,61 +72,10 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, rso *flags.StoreRootOpts, r
|
||||
return err
|
||||
}
|
||||
|
||||
if o.ChunkSize != "" {
|
||||
maxBytes, err := parseChunkSize(o.ChunkSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunks, err := archives.SplitArchive(ctx, absOutputfile, maxBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range chunks {
|
||||
l.Infof("saving store [%s] to chunk [%s]", o.StoreDir, filepath.Base(c))
|
||||
}
|
||||
} else {
|
||||
l.Infof("saving store [%s] to archive [%s]", o.StoreDir, o.FileName)
|
||||
}
|
||||
|
||||
l.Infof("saving store [%s] to archive [%s]", o.StoreDir, o.FileName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseChunkSize parses a human-readable byte size string (e.g. "1G", "500M", "2GB")
|
||||
// into a byte count. Suffixes are treated as binary units (1K = 1024).
|
||||
func parseChunkSize(s string) (int64, error) {
|
||||
units := map[string]int64{
|
||||
"K": 1 << 10, "KB": 1 << 10,
|
||||
"M": 1 << 20, "MB": 1 << 20,
|
||||
"G": 1 << 30, "GB": 1 << 30,
|
||||
"T": 1 << 40, "TB": 1 << 40,
|
||||
}
|
||||
s = strings.ToUpper(strings.TrimSpace(s))
|
||||
var result int64
|
||||
matched := false
|
||||
for suffix, mult := range units {
|
||||
if strings.HasSuffix(s, suffix) {
|
||||
n, err := strconv.ParseInt(strings.TrimSuffix(s, suffix), 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid chunk size %q", s)
|
||||
}
|
||||
result = n * mult
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid chunk size %q: %w", s, err)
|
||||
}
|
||||
result = n
|
||||
}
|
||||
if result <= 0 {
|
||||
return 0, fmt.Errorf("chunk size must be greater than zero, received %q", s)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type exports struct {
|
||||
digests []string
|
||||
records map[string]tarball.Descriptor
|
||||
|
||||
@@ -10,7 +10,6 @@ type SaveOpts struct {
|
||||
FileName string
|
||||
Platform string
|
||||
ContainerdCompatibility bool
|
||||
ChunkSize string
|
||||
}
|
||||
|
||||
func (o *SaveOpts) AddFlags(cmd *cobra.Command) {
|
||||
@@ -19,6 +18,5 @@ func (o *SaveOpts) AddFlags(cmd *cobra.Command) {
|
||||
f.StringVarP(&o.FileName, "filename", "f", consts.DefaultHaulerArchiveName, "(Optional) Specify the name of outputted haul")
|
||||
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform for runtime imports... i.e. linux/amd64 (unspecified implies all)")
|
||||
f.BoolVar(&o.ContainerdCompatibility, "containerd", false, "(Optional) Enable import compatibility with containerd... removes oci-layout from the haul")
|
||||
f.StringVar(&o.ChunkSize, "chunk-size", "", "(Optional) Split the output archive into chunks of the specified size (e.g. 1G, 500M, 2048M)")
|
||||
|
||||
}
|
||||
|
||||
@@ -3,10 +3,8 @@ package archives
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
"hauler.dev/go/hauler/pkg/log"
|
||||
@@ -104,85 +102,3 @@ func Archive(ctx context.Context, dir, outfile string, compression archives.Comp
|
||||
l.Debugf("archive created successfully [%s]", outfile)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SplitArchive splits an existing archive into chunks of at most maxBytes each.
|
||||
// Chunks are named <base>_0<ext>, <base>_1<ext>, ... where base is the archive
|
||||
// path with all extensions stripped, and ext is the compound extension (e.g. .tar.zst).
|
||||
// The original archive is removed after successful splitting.
|
||||
func SplitArchive(ctx context.Context, archivePath string, maxBytes int64) ([]string, error) {
|
||||
l := log.FromContext(ctx)
|
||||
|
||||
// derive base path and compound extension by stripping all extensions
|
||||
base := archivePath
|
||||
ext := ""
|
||||
for filepath.Ext(base) != "" {
|
||||
ext = filepath.Ext(base) + ext
|
||||
base = strings.TrimSuffix(base, filepath.Ext(base))
|
||||
}
|
||||
|
||||
f, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open archive for splitting: %w", err)
|
||||
}
|
||||
|
||||
var chunks []string
|
||||
buf := make([]byte, 32*1024)
|
||||
chunkIdx := 0
|
||||
var written int64
|
||||
var outf *os.File
|
||||
|
||||
for {
|
||||
if outf == nil {
|
||||
chunkPath := fmt.Sprintf("%s_%d%s", base, chunkIdx, ext)
|
||||
outf, err = os.Create(chunkPath)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("failed to create chunk %d: %w", chunkIdx, err)
|
||||
}
|
||||
chunks = append(chunks, chunkPath)
|
||||
l.Debugf("creating chunk [%s]", chunkPath)
|
||||
written = 0
|
||||
chunkIdx++
|
||||
}
|
||||
|
||||
remaining := maxBytes - written
|
||||
readSize := int64(len(buf))
|
||||
if readSize > remaining {
|
||||
readSize = remaining
|
||||
}
|
||||
|
||||
n, readErr := f.Read(buf[:readSize])
|
||||
if n > 0 {
|
||||
if _, writeErr := outf.Write(buf[:n]); writeErr != nil {
|
||||
outf.Close()
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("failed to write to chunk: %w", writeErr)
|
||||
}
|
||||
written += int64(n)
|
||||
}
|
||||
|
||||
if readErr == io.EOF {
|
||||
outf.Close()
|
||||
outf = nil
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
outf.Close()
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("failed to read archive: %w", readErr)
|
||||
}
|
||||
|
||||
if written >= maxBytes {
|
||||
outf.Close()
|
||||
outf = nil
|
||||
}
|
||||
}
|
||||
|
||||
f.Close()
|
||||
if err := os.Remove(archivePath); err != nil {
|
||||
return nil, fmt.Errorf("failed to remove original archive after splitting: %w", err)
|
||||
}
|
||||
|
||||
l.Infof("split archive [%s] into %d chunk(s)", filepath.Base(archivePath), len(chunks))
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
@@ -6,9 +6,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
@@ -159,86 +156,3 @@ func Unarchive(ctx context.Context, tarball, dst string) error {
|
||||
l.Infof("unarchiving completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
var chunkSuffixRe = regexp.MustCompile(`^(.+)_(\d+)$`)
|
||||
|
||||
// chunkInfo checks whether archivePath matches the chunk naming pattern (<base>_N<ext>).
|
||||
// Returns the base path (without index), compound extension, numeric index, and whether it matched.
|
||||
func chunkInfo(archivePath string) (base, ext string, index int, ok bool) {
|
||||
dir := filepath.Dir(archivePath)
|
||||
name := filepath.Base(archivePath)
|
||||
|
||||
// strip compound extension (e.g. .tar.zst)
|
||||
nameBase := name
|
||||
nameExt := ""
|
||||
for filepath.Ext(nameBase) != "" {
|
||||
nameExt = filepath.Ext(nameBase) + nameExt
|
||||
nameBase = strings.TrimSuffix(nameBase, filepath.Ext(nameBase))
|
||||
}
|
||||
|
||||
m := chunkSuffixRe.FindStringSubmatch(nameBase)
|
||||
if m == nil {
|
||||
return "", "", 0, false
|
||||
}
|
||||
|
||||
idx, _ := strconv.Atoi(m[2])
|
||||
return filepath.Join(dir, m[1]), nameExt, idx, true
|
||||
}
|
||||
|
||||
// JoinChunks detects whether archivePath is a chunk file and, if so, finds all
|
||||
// sibling chunks, concatenates them in numeric order into a single file in tempDir,
|
||||
// and returns the path to the joined file. If archivePath is not a chunk, it is
|
||||
// returned unchanged.
|
||||
func JoinChunks(ctx context.Context, archivePath, tempDir string) (string, error) {
|
||||
l := log.FromContext(ctx)
|
||||
|
||||
base, ext, _, ok := chunkInfo(archivePath)
|
||||
if !ok {
|
||||
return archivePath, nil
|
||||
}
|
||||
|
||||
all, err := filepath.Glob(base + "_*" + ext)
|
||||
if err != nil {
|
||||
return archivePath, nil
|
||||
}
|
||||
var matches []string
|
||||
for _, m := range all {
|
||||
if _, _, _, ok := chunkInfo(m); ok {
|
||||
matches = append(matches, m)
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return archivePath, nil
|
||||
}
|
||||
|
||||
sort.Slice(matches, func(i, j int) bool {
|
||||
_, _, idxI, _ := chunkInfo(matches[i])
|
||||
_, _, idxJ, _ := chunkInfo(matches[j])
|
||||
return idxI < idxJ
|
||||
})
|
||||
|
||||
l.Debugf("joining %d chunk(s) for [%s]", len(matches), base)
|
||||
|
||||
joinedPath := filepath.Join(tempDir, filepath.Base(base)+ext)
|
||||
outf, err := os.Create(joinedPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create joined archive: %w", err)
|
||||
}
|
||||
defer outf.Close()
|
||||
|
||||
for _, chunk := range matches {
|
||||
l.Debugf("joining chunk [%s]", chunk)
|
||||
cf, err := os.Open(chunk)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open chunk [%s]: %w", chunk, err)
|
||||
}
|
||||
if _, err := io.Copy(outf, cf); err != nil {
|
||||
cf.Close()
|
||||
return "", fmt.Errorf("failed to copy chunk [%s]: %w", chunk, err)
|
||||
}
|
||||
cf.Close()
|
||||
}
|
||||
|
||||
l.Infof("joined %d chunk(s) into [%s]", len(matches), filepath.Base(joinedPath))
|
||||
return joinedPath, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user