Add ImageTxt collection + storing logic

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/opencontainers/image-spec v1.0.2
 	github.com/pkg/errors v0.9.1
-	github.com/rancherfederal/ocil v0.1.7
+	github.com/rancherfederal/ocil v0.1.8
 	github.com/rs/zerolog v1.26.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1

go.sum

@@ -869,8 +869,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rancherfederal/ocil v0.1.7 h1:qIEaMEoLPbNSKZpmllZnIAj7YuD1xq1f0tVFL+32IIs=
-github.com/rancherfederal/ocil v0.1.7/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY=
+github.com/rancherfederal/ocil v0.1.8 h1:jVYD/AY7ipXgKepdZDDG1mxMOxCk/KIDdZw2qsseR+c=
+github.com/rancherfederal/ocil v0.1.8/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

pkg/collection/imagetxt/imagetxt.go

@@ -0,0 +1,232 @@
+package imagetxt
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/rancherfederal/hauler/pkg/log"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	artifact "github.com/rancherfederal/ocil/pkg/artifacts"
+	"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
+	"github.com/rancherfederal/ocil/pkg/artifacts/image"
+)
+
+type ImageTxt struct {
+	Ref            string
+	IncludeSources map[string]bool
+	ExcludeSources map[string]bool
+
+	lock     *sync.Mutex
+	client   *getter.Client
+	computed bool
+	contents map[string]artifact.OCI
+}
+
+var _ artifact.OCICollection = (*ImageTxt)(nil)
+
+type Option interface {
+	Apply(*ImageTxt) error
+}
+
+type withIncludeSources []string
+
+func (o withIncludeSources) Apply(it *ImageTxt) error {
+	if it.IncludeSources == nil {
+		it.IncludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.IncludeSources[s] = true
+	}
+	return nil
+}
+
+func WithIncludeSources(include ...string) Option {
+	return withIncludeSources(include)
+}
+
+type withExcludeSources []string
+
+func (o withExcludeSources) Apply(it *ImageTxt) error {
+	if it.ExcludeSources == nil {
+		it.ExcludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.ExcludeSources[s] = true
+	}
+	return nil
+}
+
+func WithExcludeSources(exclude ...string) Option {
+	return withExcludeSources(exclude)
+}
+
+func New(ref string, opts ...Option) (*ImageTxt, error) {
+	it := &ImageTxt{
+		Ref: ref,
+
+		client: getter.NewClient(getter.ClientOptions{}),
+		lock:   &sync.Mutex{},
+	}
+
+	for i, o := range opts {
+		if err := o.Apply(it); err != nil {
+			return nil, fmt.Errorf("invalid option %d: %v", i, err)
+		}
+	}
+
+	return it, nil
+}
+
+func (it *ImageTxt) Contents() (map[string]artifact.OCI, error) {
+	it.lock.Lock()
+	defer it.lock.Unlock()
+	if !it.computed {
+		if err := it.compute(); err != nil {
+			return nil, fmt.Errorf("compute OCI layout: %v", err)
+		}
+		it.computed = true
+	}
+	return it.contents, nil
+}
+
+func (it *ImageTxt) compute() error {
+	// TODO - pass in logger from context
+	l := log.NewLogger(os.Stdout)
+
+	it.contents = make(map[string]artifact.OCI)
+
+	ctx := context.TODO()
+
+	rc, err := it.client.ContentFrom(ctx, it.Ref)
+	if err != nil {
+		return fmt.Errorf("fetch image.txt ref %s: %w", it.Ref, err)
+	}
+	defer rc.Close()
+
+	entries, err := splitImagesTxt(rc)
+	if err != nil {
+		return fmt.Errorf("parse image.txt ref %s: %v", it.Ref, err)
+	}
+
+	foundSources := make(map[string]bool)
+	for _, e := range entries {
+		for s := range e.Sources {
+			foundSources[s] = true
+		}
+	}
+
+	var pullAll bool
+	var targetSources map[string]bool
+
+	if len(foundSources) == 0 || (len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0) {
+		// pull all found images
+		pullAll = true
+
+		if len(foundSources) == 0 {
+			l.Infof("image txt file appears to have no sources; pulling all found images")
+			if len(it.IncludeSources) != 0 || len(it.ExcludeSources) != 0 {
+				l.Warnf("ImageTxt provided include or exclude sources; ignoring")
+			}
+		} else if len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0 {
+			l.Infof("image-sources txt file not filtered; pulling all found images")
+		}
+	} else {
+		// determine sources to pull
+		if len(it.IncludeSources) != 0 && len(it.ExcludeSources) != 0 {
+			l.Warnf("ImageTxt provided include and exclude sources; using only include sources")
+		}
+
+		if len(it.IncludeSources) != 0 {
+			targetSources = it.IncludeSources
+		} else {
+			for s := range foundSources {
+				targetSources[s] = true
+			}
+			for s := range it.ExcludeSources {
+				delete(targetSources, s)
+			}
+		}
+		var targetSourcesArr []string
+		for s := range targetSources {
+			targetSourcesArr = append(targetSourcesArr, s)
+		}
+		l.Infof("pulling images covering sources %s", strings.Join(targetSourcesArr, ", "))
+	}
+
+	for _, e := range entries {
+		var matchesSourceFilter bool
+		if pullAll {
+			l.Infof("pulling image %s", e.Reference)
+		} else {
+			for s := range e.Sources {
+				if targetSources[s] {
+					matchesSourceFilter = true
+					l.Infof("pulling image %s (matched source %s)", e.Reference, s)
+					break
+				}
+			}
+		}
+
+		if pullAll || matchesSourceFilter {
+			curImage, err := image.NewImage(e.Reference.String())
+			if err != nil {
+				return fmt.Errorf("pull image %s: %v", e.Reference, err)
+			}
+			it.contents[e.Reference.String()] = curImage
+		}
+	}
+
+	return nil
+}
+
+type imageTxtEntry struct {
+	Reference name.Reference
+	Sources   map[string]bool
+}
+
+func splitImagesTxt(r io.Reader) ([]imageTxtEntry, error) {
+	var entries []imageTxtEntry
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		curEntry := imageTxtEntry{
+			Sources: make(map[string]bool),
+		}
+
+		lineContent := scanner.Text()
+		if lineContent == "" || strings.HasPrefix(lineContent, "#") {
+			// skip past empty and commented lines
+			continue
+		}
+		splitContent := strings.Split(lineContent, " ")
+		if len(splitContent) > 2 {
+			return nil, fmt.Errorf(
+				"invalid image.txt format: must contain only an image reference and sources separated by space; invalid line: %q",
+				lineContent)
+		}
+
+		curRef, err := name.ParseReference(splitContent[0])
+		if err != nil {
+			return nil, fmt.Errorf("invalid reference %s: %v", splitContent[0], err)
+		}
+		curEntry.Reference = curRef
+
+		if len(splitContent) == 2 {
+			for _, source := range strings.Split(splitContent[1], ",") {
+				curEntry.Sources[source] = true
+			}
+		}
+
+		entries = append(entries, curEntry)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("scan contents: %v", err)
+	}
+
+	return entries, nil
+}