Compare commits

..

4 Commits

Author SHA1 Message Date
CamrynCarter
3abbe77924 test updates 2026-03-22 13:36:01 -07:00
CamrynCarter
748deb3421 updated test 2026-03-22 11:23:29 -07:00
CamrynCarter
9051a06810 trim library/ 2026-03-20 14:51:00 -07:00
CamrynCarter
e084eb3e1f fixed keep registry logic 2026-03-19 11:13:26 -07:00
8 changed files with 165 additions and 282 deletions

View File

@@ -303,17 +303,6 @@ jobs:
hauler store sync --filename testdata/hauler-manifest-pipeline.yaml --filename testdata/hauler-manifest.yaml
# need more tests here
- name: Verify - hauler store sync (image list)
run: |
# verify via local image list file
hauler store sync --image-txt testdata/images.txt
# verify via multiple image list files
hauler store sync --image-txt testdata/images.txt --image-txt testdata/images.txt
# verify via remote image list file
hauler store sync --image-txt https://raw.githubusercontent.com/hauler-dev/hauler/main/testdata/images.txt
# confirm images are present in the store
hauler store info | grep 'busybox'
- name: Verify - hauler store serve
run: |
hauler store serve --help

View File

@@ -133,6 +133,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
}
if rewrite != "" {
rawRewrite := rewrite
rewrite = strings.TrimPrefix(rewrite, "/")
if !strings.Contains(rewrite, ":") {
if tag, ok := r.(name.Tag); ok {
@@ -146,7 +147,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
if err != nil {
return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err)
}
if err := rewriteReference(ctx, s, r, newRef); err != nil {
if err := rewriteReference(ctx, s, r, newRef, rawRewrite); err != nil {
return err
}
}
@@ -155,7 +156,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
return nil
}
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference) error {
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference, rawRewrite string) error {
l := log.FromContext(ctx)
if err := s.OCI.LoadIndex(); err != nil {
@@ -184,8 +185,9 @@ func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Referenc
newRegistry := newRefContext.RegistryStr()
// If user omitted a registry in the rewrite string, go-containerregistry defaults to
// index.docker.io. Preserve the original registry when the source is non-docker.
if newRegistry == "index.docker.io" && oldRegistry != "index.docker.io" {
if newRegistry == "index.docker.io" && !strings.HasPrefix(rawRewrite, "docker.io") && !strings.HasPrefix(rawRewrite, "index.docker.io") {
newRegistry = oldRegistry
newRepo = strings.TrimPrefix(newRepo, "library/") //if rewrite has library/ prefix in path it is stripped off unless registry specified in rewrite
}
oldTotal := oldRepo + ":" + oldTag
newTotal := newRepo + ":" + newTag

View File

@@ -12,6 +12,7 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/registry"
"github.com/google/go-containerregistry/pkg/v1/remote"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
helmchart "helm.sh/helm/v3/pkg/chart"
"hauler.dev/go/hauler/internal/flags"
@@ -248,7 +249,9 @@ func TestRewriteReference(t *testing.T) {
t.Fatalf("parse newRef: %v", err)
}
if err := rewriteReference(ctx, s, oldRef, newRef); err != nil {
rawRewrite := newRef.String()
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
@@ -259,8 +262,9 @@ func TestRewriteReference(t *testing.T) {
s := newTestStore(t)
oldRef, _ := name.NewTag("docker.io/missing/repo:v1")
newRef, _ := name.NewTag("docker.io/new/repo:v2")
rawRewrite := newRef.String()
err := rewriteReference(ctx, s, oldRef, newRef)
err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite)
if err == nil {
t.Fatal("expected error, got nil")
}
@@ -268,6 +272,86 @@ func TestRewriteReference(t *testing.T) {
t.Errorf("expected 'could not find' in error, got: %v", err)
}
})
// Tests for the registry-preservation / library/-stripping logic (lines 188-191).
// go-containerregistry normalises bare single-name Docker Hub refs (e.g. "nginx:latest")
// to "index.docker.io/library/nginx:latest". When the rewrite string omits a registry,
// rewriteReference must (a) preserve the source registry and (b) strip the injected
// "library/" prefix so that the stored ref looks like "nginx:v2", not "library/nginx:v2".
t.Run("path-only rewrite strips library/ prefix from docker hub official image", func(t *testing.T) {
s := newTestStore(t)
seedStoreDescriptor(t, s, map[string]string{
ocispec.AnnotationRefName: "library/nginx:latest",
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
})
oldRef, _ := name.NewTag("nginx:latest") // → index.docker.io/library/nginx:latest
newRef, _ := name.NewTag("nginx:v2") // → index.docker.io/library/nginx:v2
rawRewrite := "nginx:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// library/ must be stripped; registry stays index.docker.io
assertAnnotationsInStore(t, s, "nginx:v2", "index.docker.io/nginx:v2")
})
t.Run("explicit docker.io rewrite preserves library/ prefix", func(t *testing.T) {
s := newTestStore(t)
seedStoreDescriptor(t, s, map[string]string{
ocispec.AnnotationRefName: "library/nginx:latest",
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
})
oldRef, _ := name.NewTag("nginx:latest")
newRef, _ := name.NewTag("docker.io/nginx:v2") // → index.docker.io/library/nginx:v2
rawRewrite := "docker.io/nginx:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// rawRewrite starts with "docker.io" → condition must NOT fire → library/ preserved
assertAnnotationsInStore(t, s, "library/nginx:v2", "index.docker.io/library/nginx:v2")
})
t.Run("explicit index.docker.io rewrite preserves library/ prefix", func(t *testing.T) {
s := newTestStore(t)
seedStoreDescriptor(t, s, map[string]string{
ocispec.AnnotationRefName: "library/nginx:latest",
consts.ContainerdImageNameKey: "index.docker.io/library/nginx:latest",
})
oldRef, _ := name.NewTag("nginx:latest")
newRef, _ := name.NewTag("index.docker.io/nginx:v2") // → index.docker.io/library/nginx:v2
rawRewrite := "index.docker.io/nginx:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// rawRewrite starts with "index.docker.io" → condition must NOT fire → library/ preserved
assertAnnotationsInStore(t, s, "library/nginx:v2", "index.docker.io/library/nginx:v2")
})
t.Run("non-docker source with path-only rewrite preserves original registry", func(t *testing.T) {
host, rOpts := newTestRegistry(t)
seedImage(t, host, "src/repo", "v1", rOpts...)
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/src/repo:v1", "", rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
oldRef, _ := name.NewTag(host+"/src/repo:v1", name.Insecure)
newRef, _ := name.NewTag("newrepo/img:v2") // defaults to index.docker.io
rawRewrite := "newrepo/img:v2"
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// condition fires → registry reverts to host, no library/ to strip
assertAnnotationsInStore(t, s, "newrepo/img:v2", host+"/newrepo/img:v2")
})
}
// --------------------------------------------------------------------------

View File

@@ -81,108 +81,54 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
l.Infof("processing completed successfully")
}
// If passed a hauler manifest, process it
if len(o.FileName) != 0 {
for _, fileName := range o.FileName {
l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
// If passed a local manifest, process it
for _, fileName := range o.FileName {
l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
haulPath := fileName
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
haulPath := fileName
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
h := getter.NewHttp()
parsedURL, err := url.Parse(haulPath)
if err != nil {
return err
}
rc, err := h.Open(ctx, parsedURL)
if err != nil {
return err
}
defer rc.Close()
fileName := h.Name(parsedURL)
if fileName == "" {
fileName = filepath.Base(parsedURL.Path)
}
haulPath = filepath.Join(tempDir, fileName)
out, err := os.Create(haulPath)
if err != nil {
return err
}
defer out.Close()
if _, err = io.Copy(out, rc); err != nil {
return err
}
}
fi, err := os.Open(haulPath)
h := getter.NewHttp()
parsedURL, err := url.Parse(haulPath)
if err != nil {
return err
}
defer fi.Close()
err = processContent(ctx, fi, o, s, rso, ro)
rc, err := h.Open(ctx, parsedURL)
if err != nil {
return err
}
defer rc.Close()
l.Infof("processing completed successfully")
fileName := h.Name(parsedURL)
if fileName == "" {
fileName = filepath.Base(parsedURL.Path)
}
haulPath = filepath.Join(tempDir, fileName)
out, err := os.Create(haulPath)
if err != nil {
return err
}
defer out.Close()
if _, err = io.Copy(out, rc); err != nil {
return err
}
}
}
// If passed an image.txt file, process it
if len(o.ImageTxt) != 0 {
for _, imageTxt := range o.ImageTxt {
l.Infof("processing image.txt [%s] to store [%s]", imageTxt, o.StoreDir)
haulPath := imageTxt
if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
l.Debugf("detected remote image.txt... starting download... [%s]", haulPath)
h := getter.NewHttp()
parsedURL, err := url.Parse(haulPath)
if err != nil {
return err
}
rc, err := h.Open(ctx, parsedURL)
if err != nil {
return err
}
defer rc.Close()
fileName := h.Name(parsedURL)
if fileName == "" {
fileName = filepath.Base(parsedURL.Path)
}
haulPath = filepath.Join(tempDir, fileName)
out, err := os.Create(haulPath)
if err != nil {
return err
}
defer out.Close()
if _, err = io.Copy(out, rc); err != nil {
return err
}
}
fi, err := os.Open(haulPath)
if err != nil {
return err
}
defer fi.Close()
err = processImageTxt(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
l.Infof("processing completed successfully")
fi, err := os.Open(haulPath)
if err != nil {
return err
}
defer fi.Close()
err = processContent(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
l.Infof("processing completed successfully")
}
return nil
@@ -417,21 +363,3 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
}
return nil
}
func processImageTxt(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
l.Infof("syncing images from [%s] to store", filepath.Base(fi.Name()))
scanner := bufio.NewScanner(fi)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
img := v1.Image{Name: line}
l.Infof("adding image [%s] to the store [%s]", line, o.StoreDir)
if err := storeImage(ctx, s, img, o.Platform, rso, ro, ""); err != nil {
return err
}
}
return scanner.Err()
}

View File

@@ -256,157 +256,6 @@ spec:
assertArtifactInStore(t, s, "synced-local.sh")
}
// --------------------------------------------------------------------------
// processImageTxt tests
// --------------------------------------------------------------------------
// writeImageTxtFile writes lines to a temp file and returns it seeked to the
// start, ready for processImageTxt to consume.
func writeImageTxtFile(t *testing.T, lines string) *os.File {
t.Helper()
fi, err := os.CreateTemp(t.TempDir(), "images-*.txt")
if err != nil {
t.Fatalf("writeImageTxtFile CreateTemp: %v", err)
}
t.Cleanup(func() { fi.Close() })
if _, err := fi.WriteString(lines); err != nil {
t.Fatalf("writeImageTxtFile WriteString: %v", err)
}
if _, err := fi.Seek(0, io.SeekStart); err != nil {
t.Fatalf("writeImageTxtFile Seek: %v", err)
}
return fi
}
func TestProcessImageTxt_SingleImage(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/txtimage", "v1")
fi := writeImageTxtFile(t, fmt.Sprintf("%s/myorg/txtimage:v1\n", host))
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt single image: %v", err)
}
assertArtifactInStore(t, s, "myorg/txtimage")
}
func TestProcessImageTxt_MultipleImages(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/alpha", "v1")
seedImage(t, host, "myorg/beta", "v2")
content := fmt.Sprintf("%s/myorg/alpha:v1\n%s/myorg/beta:v2\n", host, host)
fi := writeImageTxtFile(t, content)
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt multiple images: %v", err)
}
assertArtifactInStore(t, s, "myorg/alpha")
assertArtifactInStore(t, s, "myorg/beta")
}
func TestProcessImageTxt_SkipsBlankLinesAndComments(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/commenttest", "v1")
content := fmt.Sprintf("# this is a comment\n\n%s/myorg/commenttest:v1\n\n# another comment\n", host)
fi := writeImageTxtFile(t, content)
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt skip blanks/comments: %v", err)
}
assertArtifactInStore(t, s, "myorg/commenttest")
if n := countArtifactsInStore(t, s); n != 1 {
t.Errorf("expected 1 artifact, got %d", n)
}
}
func TestProcessImageTxt_EmptyFile(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
fi := writeImageTxtFile(t, "")
o := newSyncOpts(s.Root)
ro := defaultCliOpts()
if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
t.Fatalf("processImageTxt empty file: %v", err)
}
if n := countArtifactsInStore(t, s); n != 0 {
t.Errorf("expected 0 artifacts for empty file, got %d", n)
}
}
// --------------------------------------------------------------------------
// SyncCmd --image-txt integration tests
// --------------------------------------------------------------------------
func TestSyncCmd_ImageTxt_LocalFile(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/syncedtxt", "v1")
txtFile, err := os.CreateTemp(t.TempDir(), "images-*.txt")
if err != nil {
t.Fatalf("CreateTemp: %v", err)
}
txtPath := txtFile.Name()
fmt.Fprintf(txtFile, "%s/myorg/syncedtxt:v1\n", host)
txtFile.Close()
o := newSyncOpts(s.Root)
o.ImageTxt = []string{txtPath}
rso := defaultRootOpts(s.Root)
ro := defaultCliOpts()
if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
t.Fatalf("SyncCmd ImageTxt LocalFile: %v", err)
}
assertArtifactInStore(t, s, "myorg/syncedtxt")
}
func TestSyncCmd_ImageTxt_RemoteFile(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/remotetxt", "v1")
imageListContent := fmt.Sprintf("%s/myorg/remotetxt:v1\n", host)
imageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
io.WriteString(w, imageListContent) //nolint:errcheck
}))
t.Cleanup(imageSrv.Close)
o := newSyncOpts(s.Root)
o.ImageTxt = []string{imageSrv.URL + "/images.txt"}
rso := defaultRootOpts(s.Root)
ro := defaultCliOpts()
if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
t.Fatalf("SyncCmd ImageTxt RemoteFile: %v", err)
}
assertArtifactInStore(t, s, "myorg/remotetxt")
}
func TestSyncCmd_RemoteManifest(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)

View File

@@ -22,6 +22,7 @@ import (
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/static"
gvtypes "github.com/google/go-containerregistry/pkg/v1/types"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
"helm.sh/helm/v3/pkg/action"
@@ -282,6 +283,41 @@ func seedOCI11Referrer(t *testing.T, host, repo string, baseImg gcrv1.Image, opt
}
}
// seedStoreDescriptor injects a descriptor with the given annotations directly
// into the store index without requiring a real registry or blob. This is used
// to pre-populate the store for rewriteReference unit tests.
func seedStoreDescriptor(t *testing.T, s *store.Layout, annotations map[string]string) {
t.Helper()
desc := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Digest: digest.Digest("sha256:" + strings.Repeat("a", 64)),
Size: 1,
Annotations: annotations,
}
if err := s.OCI.AddIndex(desc); err != nil {
t.Fatalf("seedStoreDescriptor: %v", err)
}
}
// assertAnnotationsInStore walks the store and fails if no descriptor has both
// AnnotationRefName == refName AND ContainerdImageNameKey == containerdName.
func assertAnnotationsInStore(t *testing.T, s *store.Layout, refName, containerdName string) {
t.Helper()
found := false
if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error {
if desc.Annotations[ocispec.AnnotationRefName] == refName &&
desc.Annotations[consts.ContainerdImageNameKey] == containerdName {
found = true
}
return nil
}); err != nil {
t.Fatalf("assertAnnotationsInStore walk: %v", err)
}
if !found {
t.Errorf("no artifact with AnnotationRefName=%q and ContainerdImageNameKey=%q found in store", refName, containerdName)
}
}
// assertReferrerInStore walks the store and fails if no descriptor has a kind
// annotation with the KindAnnotationReferrers prefix and a ref containing refSubstring.
func assertReferrerInStore(t *testing.T, s *store.Layout, refSubstring string) {

View File

@@ -2,12 +2,12 @@ package flags
import (
"github.com/spf13/cobra"
"hauler.dev/go/hauler/pkg/consts"
)
type SyncOpts struct {
*StoreRootOpts
FileName []string
ImageTxt []string
Key string
CertOidcIssuer string
CertOidcIssuerRegexp string
@@ -25,8 +25,7 @@ type SyncOpts struct {
func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringSliceVarP(&o.FileName, "filename", "f", []string{}, "Specify the name of manifest(s) to sync")
f.StringSliceVarP(&o.ImageTxt, "image-txt", "i", []string{}, "Specify local or remote image.txt file(s) to sync images")
f.StringSliceVarP(&o.FileName, "filename", "f", []string{consts.DefaultHaulerManifestName}, "Specify the name of manifest(s) to sync")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification")
f.StringVar(&o.CertIdentity, "certificate-identity", "", "(Optional) Cosign certificate-identity (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")
f.StringVar(&o.CertIdentityRegexp, "certificate-identity-regexp", "", "(Optional) Cosign certificate-identity-regexp (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")

4
testdata/images.txt vendored
View File

@@ -1,4 +0,0 @@
# hauler image list
# one image reference per line; blank lines and comments are ignored
ghcr.io/hauler-dev/library/busybox
ghcr.io/hauler-dev/library/busybox:stable