images.txt

.github/workflows/tests.yaml

@@ -303,6 +303,17 @@ jobs:
           hauler store sync --filename testdata/hauler-manifest-pipeline.yaml --filename testdata/hauler-manifest.yaml
           # need more tests here
 
+      - name: Verify - hauler store sync (image list)
+        run: |
+          # verify via local image list file
+          hauler store sync --image-txt testdata/images.txt
+          # verify via multiple image list files
+          hauler store sync --image-txt testdata/images.txt --image-txt testdata/images.txt
+          # verify via remote image list file
+          hauler store sync --image-txt https://raw.githubusercontent.com/hauler-dev/hauler/main/testdata/images.txt
+          # confirm images are present in the store
+          hauler store info | grep 'busybox'
+
       - name: Verify - hauler store serve
         run: |
           hauler store serve --help

cmd/hauler/cli/store/sync.go

@@ -81,54 +81,108 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
 		l.Infof("processing completed successfully")
 	}
 
-	// If passed a local manifest, process it
-	for _, fileName := range o.FileName {
-		l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
+	// If passed a hauler manifest, process it
+	if len(o.FileName) != 0 {
+		for _, fileName := range o.FileName {
+			l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
 
-		haulPath := fileName
-		if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
-			l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
+			haulPath := fileName
+			if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
+				l.Debugf("detected remote manifest... starting download... [%s]", haulPath)
 
-			h := getter.NewHttp()
-			parsedURL, err := url.Parse(haulPath)
+				h := getter.NewHttp()
+				parsedURL, err := url.Parse(haulPath)
+				if err != nil {
+					return err
+				}
+				rc, err := h.Open(ctx, parsedURL)
+				if err != nil {
+					return err
+				}
+				defer rc.Close()
+
+				fileName := h.Name(parsedURL)
+				if fileName == "" {
+					fileName = filepath.Base(parsedURL.Path)
+				}
+				haulPath = filepath.Join(tempDir, fileName)
+
+				out, err := os.Create(haulPath)
+				if err != nil {
+					return err
+				}
+				defer out.Close()
+
+				if _, err = io.Copy(out, rc); err != nil {
+					return err
+				}
+			}
+
+			fi, err := os.Open(haulPath)
 			if err != nil {
 				return err
 			}
-			rc, err := h.Open(ctx, parsedURL)
+			defer fi.Close()
+
+			err = processContent(ctx, fi, o, s, rso, ro)
 			if err != nil {
 				return err
 			}
-			defer rc.Close()
 
-			fileName := h.Name(parsedURL)
-			if fileName == "" {
-				fileName = filepath.Base(parsedURL.Path)
+			l.Infof("processing completed successfully")
+		}
+	}
+
+	// If passed an image.txt file, process it
+	if len(o.ImageTxt) != 0 {
+		for _, imageTxt := range o.ImageTxt {
+			l.Infof("processing image.txt [%s] to store [%s]", imageTxt, o.StoreDir)
+
+			haulPath := imageTxt
+			if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") {
+				l.Debugf("detected remote image.txt... starting download... [%s]", haulPath)
+
+				h := getter.NewHttp()
+				parsedURL, err := url.Parse(haulPath)
+				if err != nil {
+					return err
+				}
+				rc, err := h.Open(ctx, parsedURL)
+				if err != nil {
+					return err
+				}
+				defer rc.Close()
+
+				fileName := h.Name(parsedURL)
+				if fileName == "" {
+					fileName = filepath.Base(parsedURL.Path)
+				}
+				haulPath = filepath.Join(tempDir, fileName)
+
+				out, err := os.Create(haulPath)
+				if err != nil {
+					return err
+				}
+				defer out.Close()
+
+				if _, err = io.Copy(out, rc); err != nil {
+					return err
+				}
 			}
-			haulPath = filepath.Join(tempDir, fileName)
 
-			out, err := os.Create(haulPath)
+			fi, err := os.Open(haulPath)
 			if err != nil {
 				return err
 			}
-			defer out.Close()
+			defer fi.Close()
 
-			if _, err = io.Copy(out, rc); err != nil {
+			err = processImageTxt(ctx, fi, o, s, rso, ro)
+			if err != nil {
 				return err
 			}
-		}
 
-		fi, err := os.Open(haulPath)
-		if err != nil {
-			return err
+			l.Infof("processing completed successfully")
 		}
-		defer fi.Close()
-
-		err = processContent(ctx, fi, o, s, rso, ro)
-		if err != nil {
-			return err
-		}
-
-		l.Infof("processing completed successfully")
 	}
 
 	return nil
@@ -363,3 +417,21 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 	}
 	return nil
 }
+
+func processImageTxt(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
+	l := log.FromContext(ctx)
+	l.Infof("syncing images from [%s] to store", filepath.Base(fi.Name()))
+	scanner := bufio.NewScanner(fi)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		img := v1.Image{Name: line}
+		l.Infof("adding image [%s] to the store [%s]", line, o.StoreDir)
+		if err := storeImage(ctx, s, img, o.Platform, rso, ro, ""); err != nil {
+			return err
+		}
+	}
+	return scanner.Err()
+}

cmd/hauler/cli/store/sync_test.go

@@ -256,6 +256,157 @@ spec:
 	assertArtifactInStore(t, s, "synced-local.sh")
 }
 
+// --------------------------------------------------------------------------
+// processImageTxt tests
+// --------------------------------------------------------------------------
+
+// writeImageTxtFile writes lines to a temp file and returns it seeked to the
+// start, ready for processImageTxt to consume.
+func writeImageTxtFile(t *testing.T, lines string) *os.File {
+	t.Helper()
+	fi, err := os.CreateTemp(t.TempDir(), "images-*.txt")
+	if err != nil {
+		t.Fatalf("writeImageTxtFile CreateTemp: %v", err)
+	}
+	t.Cleanup(func() { fi.Close() })
+	if _, err := fi.WriteString(lines); err != nil {
+		t.Fatalf("writeImageTxtFile WriteString: %v", err)
+	}
+	if _, err := fi.Seek(0, io.SeekStart); err != nil {
+		t.Fatalf("writeImageTxtFile Seek: %v", err)
+	}
+	return fi
+}
+
+func TestProcessImageTxt_SingleImage(t *testing.T) {
+	ctx := newTestContext(t)
+	s := newTestStore(t)
+
+	host, _ := newLocalhostRegistry(t)
+	seedImage(t, host, "myorg/txtimage", "v1")
+
+	fi := writeImageTxtFile(t, fmt.Sprintf("%s/myorg/txtimage:v1\n", host))
+	o := newSyncOpts(s.Root)
+	ro := defaultCliOpts()
+
+	if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
+		t.Fatalf("processImageTxt single image: %v", err)
+	}
+	assertArtifactInStore(t, s, "myorg/txtimage")
+}
+
+func TestProcessImageTxt_MultipleImages(t *testing.T) {
+	ctx := newTestContext(t)
+	s := newTestStore(t)
+
+	host, _ := newLocalhostRegistry(t)
+	seedImage(t, host, "myorg/alpha", "v1")
+	seedImage(t, host, "myorg/beta", "v2")
+
+	content := fmt.Sprintf("%s/myorg/alpha:v1\n%s/myorg/beta:v2\n", host, host)
+	fi := writeImageTxtFile(t, content)
+	o := newSyncOpts(s.Root)
+	ro := defaultCliOpts()
+
+	if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
+		t.Fatalf("processImageTxt multiple images: %v", err)
+	}
+	assertArtifactInStore(t, s, "myorg/alpha")
+	assertArtifactInStore(t, s, "myorg/beta")
+}
+
+func TestProcessImageTxt_SkipsBlankLinesAndComments(t *testing.T) {
+	ctx := newTestContext(t)
+	s := newTestStore(t)
+
+	host, _ := newLocalhostRegistry(t)
+	seedImage(t, host, "myorg/commenttest", "v1")
+
+	content := fmt.Sprintf("# this is a comment\n\n%s/myorg/commenttest:v1\n\n# another comment\n", host)
+	fi := writeImageTxtFile(t, content)
+	o := newSyncOpts(s.Root)
+	ro := defaultCliOpts()
+
+	if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
+		t.Fatalf("processImageTxt skip blanks/comments: %v", err)
+	}
+	assertArtifactInStore(t, s, "myorg/commenttest")
+	if n := countArtifactsInStore(t, s); n != 1 {
+		t.Errorf("expected 1 artifact, got %d", n)
+	}
+}
+
+func TestProcessImageTxt_EmptyFile(t *testing.T) {
+	ctx := newTestContext(t)
+	s := newTestStore(t)
+
+	fi := writeImageTxtFile(t, "")
+	o := newSyncOpts(s.Root)
+	ro := defaultCliOpts()
+
+	if err := processImageTxt(ctx, fi, o, s, o.StoreRootOpts, ro); err != nil {
+		t.Fatalf("processImageTxt empty file: %v", err)
+	}
+	if n := countArtifactsInStore(t, s); n != 0 {
+		t.Errorf("expected 0 artifacts for empty file, got %d", n)
+	}
+}
+
+// --------------------------------------------------------------------------
+// SyncCmd --image-txt integration tests
+// --------------------------------------------------------------------------
+
+func TestSyncCmd_ImageTxt_LocalFile(t *testing.T) {
+	ctx := newTestContext(t)
+	s := newTestStore(t)
+
+	host, _ := newLocalhostRegistry(t)
+	seedImage(t, host, "myorg/syncedtxt", "v1")
+
+	txtFile, err := os.CreateTemp(t.TempDir(), "images-*.txt")
+	if err != nil {
+		t.Fatalf("CreateTemp: %v", err)
+	}
+	txtPath := txtFile.Name()
+	fmt.Fprintf(txtFile, "%s/myorg/syncedtxt:v1\n", host)
+	txtFile.Close()
+
+	o := newSyncOpts(s.Root)
+	o.ImageTxt = []string{txtPath}
+	rso := defaultRootOpts(s.Root)
+	ro := defaultCliOpts()
+
+	if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
+		t.Fatalf("SyncCmd ImageTxt LocalFile: %v", err)
+	}
+	assertArtifactInStore(t, s, "myorg/syncedtxt")
+}
+
+func TestSyncCmd_ImageTxt_RemoteFile(t *testing.T) {
+	ctx := newTestContext(t)
+	s := newTestStore(t)
+
+	host, _ := newLocalhostRegistry(t)
+	seedImage(t, host, "myorg/remotetxt", "v1")
+
+	imageListContent := fmt.Sprintf("%s/myorg/remotetxt:v1\n", host)
+	imageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		io.WriteString(w, imageListContent) //nolint:errcheck
+	}))
+	t.Cleanup(imageSrv.Close)
+
+	o := newSyncOpts(s.Root)
+	o.ImageTxt = []string{imageSrv.URL + "/images.txt"}
+	rso := defaultRootOpts(s.Root)
+	ro := defaultCliOpts()
+
+	if err := SyncCmd(ctx, o, s, rso, ro); err != nil {
+		t.Fatalf("SyncCmd ImageTxt RemoteFile: %v", err)
+	}
+	assertArtifactInStore(t, s, "myorg/remotetxt")
+}
+
 func TestSyncCmd_RemoteManifest(t *testing.T) {
 	ctx := newTestContext(t)
 	s := newTestStore(t)

internal/flags/sync.go

@@ -2,12 +2,12 @@ package flags
 
 import (
 	"github.com/spf13/cobra"
-	"hauler.dev/go/hauler/pkg/consts"
 )
 
 type SyncOpts struct {
 	*StoreRootOpts
 	FileName                     []string
+	ImageTxt                     []string
 	Key                          string
 	CertOidcIssuer               string
 	CertOidcIssuerRegexp         string
@@ -25,7 +25,8 @@ type SyncOpts struct {
 func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
 
-	f.StringSliceVarP(&o.FileName, "filename", "f", []string{consts.DefaultHaulerManifestName}, "Specify the name of manifest(s) to sync")
+	f.StringSliceVarP(&o.FileName, "filename", "f", []string{}, "Specify the name of manifest(s) to sync")
+	f.StringSliceVarP(&o.ImageTxt, "image-txt", "i", []string{}, "Specify local or remote image.txt file(s) to sync images")
 	f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification")
 	f.StringVar(&o.CertIdentity, "certificate-identity", "", "(Optional) Cosign certificate-identity (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")
 	f.StringVar(&o.CertIdentityRegexp, "certificate-identity-regexp", "", "(Optional) Cosign certificate-identity-regexp (either --certificate-identity or --certificate-identity-regexp required for keyless verification)")

testdata/images.txt

@@ -0,0 +1,4 @@
+# hauler image list
+# one image reference per line; blank lines and comments are ignored
+ghcr.io/hauler-dev/library/busybox
+ghcr.io/hauler-dev/library/busybox:stable