Wrap errors

[0.2] Backport auth, insecure, plain HTTP flags

cmd/hauler/cli/cli.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -39,7 +40,7 @@ func New() *cobra.Command {
 	}
 
 	pf := cmd.PersistentFlags()
-	pf.StringVarP(&ro.logLevel, "log-level", "l", "info", "")
+	pf.StringVarP(&ro.logLevel, "log-level", "l", "info", `Verbosity of logs ("debug", info", "warn", "error")`)
 	pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to $XDG_CACHE_HOME/hauler)")
 	pf.StringVarP(&ro.storeDir, "store", "s", "", "Location to create store at (defaults to $PWD/store)")
 
@@ -57,33 +58,28 @@ func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
 
 	if dir == "" {
 		l.Debugf("no store path specified, defaulting to $PWD/store")
-		pwd, err := os.Getwd()
-		if err != nil {
-			return nil, err
-		}
-
-		dir = filepath.Join(pwd, defaultStoreLocation)
+		dir = defaultStoreLocation
 	}
 
 	abs, err := filepath.Abs(dir)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("resolve store absolute path: %w", err)
 	}
 
 	l.Debugf("using store at %s", abs)
 	if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) {
 		err := os.Mkdir(abs, os.ModePerm)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("create store directory %s: %w", abs, err)
 		}
 	} else if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("stat %s: %w", abs, err)
 	}
 
 	// TODO: Do we want this to be configurable?
 	c, err := o.getCache(ctx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("get cache: %w", err)
 	}
 
 	s := store.NewStore(ctx, abs, store.WithCache(c))
@@ -94,18 +90,17 @@ func (o *rootOpts) getCache(ctx context.Context) (cache.Cache, error) {
 	dir := o.cacheDir
 
 	if dir == "" {
-		// Default to $XDG_CACHE_HOME
-		cachedir, err := os.UserCacheDir()
+		// Default to $XDG_CACHE_HOME/hauler
+		userCacheDir, err := os.UserCacheDir()
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("get default cache directory: %w", err)
 		}
+		dir = filepath.Join(userCacheDir, "hauler")
+	}
 
-		abs, _ := filepath.Abs(filepath.Join(cachedir, "hauler"))
-		if err := os.MkdirAll(abs, os.ModePerm); err != nil {
-			return nil, err
-		}
-
-		dir = abs
+	abs, _ := filepath.Abs(dir)
+	if err := os.MkdirAll(abs, os.ModePerm); err != nil {
+		return nil, fmt.Errorf("create cache directory %s: %w", abs, err)
 	}
 
 	c := cache.NewFilesystem(dir)

cmd/hauler/cli/download/download.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"path"
 
 	"github.com/containerd/containerd/remotes/docker"
@@ -18,18 +19,26 @@ import (
 
 	"github.com/rancherfederal/hauler/pkg/artifact/types"
 	"github.com/rancherfederal/hauler/pkg/log"
+	"github.com/rancherfederal/hauler/pkg/version"
 )
 
 type Opts struct {
 	DestinationDir string
-	OutputFile     string
+
+	Username  string
+	Password  string
+	Insecure  bool
+	PlainHTTP bool
 }
 
 func (o *Opts) AddArgs(cmd *cobra.Command) {
 	f := cmd.Flags()
 
-	f.StringVar(&o.DestinationDir, "dir", "", "Directory to save contents to (defaults to current directory)")
-	f.StringVarP(&o.OutputFile, "output", "o", "", "(Optional) Override name of file to save.")
+	f.StringVarP(&o.DestinationDir, "output", "o", "", "Directory to save contents to (defaults to current directory)")
+	f.StringVarP(&o.Username, "username", "u", "", "Username when copying to an authenticated remote registry")
+	f.StringVarP(&o.Password, "password", "p", "", "Password when copying to an authenticated remote registry")
+	f.BoolVar(&o.Insecure, "insecure", false, "Toggle allowing insecure connections when copying to a remote registry")
+	f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry")
 }
 
 func Cmd(ctx context.Context, o *Opts, reference string) error {
@@ -38,12 +47,71 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 	cs := content.NewFileStore(o.DestinationDir)
 	defer cs.Close()
 
-	ref, err := name.ParseReference(reference)
+	// build + configure oras client
+	var refOpts []name.Option
+	remoteOpts := []remote.Option{
+		remote.WithAuthFromKeychain(authn.DefaultKeychain),
+	}
+
+	if o.PlainHTTP {
+		refOpts = append(refOpts, name.Insecure)
+	}
+
+	if o.Username != "" || o.Password != "" {
+		basicAuth := &authn.Basic{
+			Username: o.Username,
+			Password: o.Password,
+		}
+		remoteOpts = append(remoteOpts, remote.WithAuth(basicAuth))
+	}
+
+	if o.Insecure {
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig.InsecureSkipVerify = true
+
+		remoteOpts = append(remoteOpts, remote.WithTransport(transport))
+	}
+
+	// build + configure containerd client
+	var registryOpts []docker.RegistryOpt
+
+	if o.PlainHTTP {
+		registryOpts = append(registryOpts, docker.WithPlainHTTP(docker.MatchAllHosts))
+	}
+
+	if o.Username != "" || o.Password != "" {
+		creds := func(string) (string, string, error) {
+			return o.Username, o.Password, nil
+		}
+		authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(creds))
+		registryOpts = append(registryOpts, docker.WithAuthorizer(authorizer))
+	}
+
+	if o.Insecure {
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig.InsecureSkipVerify = true
+
+		httpClient := &http.Client{
+			Transport: transport,
+		}
+		registryOpts = append(registryOpts, docker.WithClient(httpClient))
+	}
+
+	resolverOpts := docker.ResolverOptions{
+		Hosts:   docker.ConfigureDefaultRegistries(registryOpts...),
+		Headers: http.Header{},
+	}
+	resolverOpts.Headers.Set("User-Agent", "hauler/"+version.GitVersion)
+
+	resolver := docker.NewResolver(resolverOpts)
+
+	// begin dowloading target
+	ref, err := name.ParseReference(reference, refOpts...)
 	if err != nil {
 		return err
 	}
 
-	desc, err := remote.Get(ref)
+	desc, err := remote.Get(ref, remoteOpts...)
 	if err != nil {
 		return err
 	}
@@ -62,15 +130,12 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 	switch manifest.Config.MediaType {
 	case types.DockerConfigJSON, types.OCIManifestSchema1:
 		l.Debugf("identified [image] (%s) content", manifest.Config.MediaType)
-		img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+		img, err := remote.Image(ref, remoteOpts...)
 		if err != nil {
 			return err
 		}
 
-		outputFile := o.OutputFile
-		if outputFile == "" {
-			outputFile = fmt.Sprintf("%s:%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier())
-		}
+		outputFile := fmt.Sprintf("%s_%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier())
 
 		if err := tarball.WriteToFile(outputFile, ref, img); err != nil {
 			return err
@@ -83,7 +148,7 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 
 		fs := content.NewFileStore(o.DestinationDir)
 
-		resolver := docker.NewResolver(docker.ResolverOptions{})
+		// TODO - additional accepted media types
 		_, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
 		if err != nil {
 			return err
@@ -100,7 +165,7 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 
 		fs := content.NewFileStore(o.DestinationDir)
 
-		resolver := docker.NewResolver(docker.ResolverOptions{})
+		// TODO - additional accepted media types
 		_, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
 		if err != nil {
 			return err

cmd/hauler/cli/store/sync.go

@@ -11,7 +11,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
+	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha2"
 	tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
+	"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
 	"github.com/rancherfederal/hauler/pkg/collection/k3s"
 	"github.com/rancherfederal/hauler/pkg/content"
 	"github.com/rancherfederal/hauler/pkg/log"
@@ -70,79 +72,121 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 
 			l.Infof("syncing [%s] to [%s]", obj.GroupVersionKind().String(), s.DataDir)
 
-			// TODO: Should type switch instead...
-			switch obj.GroupVersionKind().Kind {
-			case v1alpha1.FilesContentKind:
-				var cfg v1alpha1.Files
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
+			gvk := obj.GroupVersionKind()
 
-				for _, f := range cfg.Spec.Files {
-					err := storeFile(ctx, s, f)
+			switch {
+			// content.hauler.cattle.io/v1alpha1
+			case gvk.GroupVersion() == v1alpha1.ContentGroupVersion:
+				l.Warnf(
+					"API version %s is deprecated in v0.3; ok to use in v0.2, use %s instead in v0.3",
+					gvk.GroupVersion().String(),
+					v1alpha2.ContentGroupVersion.String(),
+				)
+				switch gvk.Kind {
+				// content.hauler.cattle.io/v1alpha1 Files
+				case v1alpha1.FilesContentKind:
+					var cfg v1alpha1.Files
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					for _, f := range cfg.Spec.Files {
+						err := storeFile(ctx, s, f)
+						if err != nil {
+							return err
+						}
+					}
+				// content.hauler.cattle.io/v1alpha1 Images
+				case v1alpha1.ImagesContentKind:
+					var cfg v1alpha1.Images
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					for _, i := range cfg.Spec.Images {
+						err := storeImage(ctx, s, i)
+						if err != nil {
+							return err
+						}
+					}
+				// content.hauler.cattle.io/v1alpha1 Charts
+				case v1alpha1.ChartsContentKind:
+					var cfg v1alpha1.Charts
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					for _, ch := range cfg.Spec.Charts {
+						err := storeChart(ctx, s, ch)
+						if err != nil {
+							return err
+						}
+					}
+				// collection.hauler.cattle.io/v1alpha1 unknown
+				default:
+					return fmt.Errorf("unsupported Kind %s for %s", obj.GroupVersionKind().Kind, obj.GroupVersionKind().GroupVersion().String())
+				}
+			// collection.hauler.cattle.io/v1alpha1
+			case gvk.GroupVersion() == v1alpha1.CollectionGroupVersion:
+				l.Warnf(
+					"API version %s is deprecated in v0.3; ok to use in v0.2, use %s instead in v0.3",
+					gvk.GroupVersion().String(),
+					v1alpha2.CollectionGroupVersion.String(),
+				)
+				switch gvk.Kind {
+				// collection.hauler.cattle.io/v1alpha1 K3s
+				case v1alpha1.K3sCollectionKind:
+					var cfg v1alpha1.K3s
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					k, err := k3s.NewK3s(cfg.Spec.Version)
 					if err != nil {
 						return err
 					}
-				}
-
-			case v1alpha1.ImagesContentKind:
-				var cfg v1alpha1.Images
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				for _, i := range cfg.Spec.Images {
-					err := storeImage(ctx, s, i)
-					if err != nil {
+					if _, err := s.AddCollection(ctx, k); err != nil {
 						return err
 					}
-				}
-
-			case v1alpha1.ChartsContentKind:
-				var cfg v1alpha1.Charts
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				for _, ch := range cfg.Spec.Charts {
-					err := storeChart(ctx, s, ch)
-					if err != nil {
+				// collection.hauler.cattle.io/v1alpha1 ThickCharts
+				case v1alpha1.ChartsCollectionKind:
+					var cfg v1alpha1.ThickCharts
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
 						return err
 					}
-				}
-
-			case v1alpha1.K3sCollectionKind:
-				var cfg v1alpha1.K3s
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				k, err := k3s.NewK3s(cfg.Spec.Version)
-				if err != nil {
-					return err
-				}
-
-				if _, err := s.AddCollection(ctx, k); err != nil {
-					return err
-				}
-
-			case v1alpha1.ChartsCollectionKind:
-				var cfg v1alpha1.ThickCharts
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				for _, cfg := range cfg.Spec.Charts {
-					tc, err := tchart.NewChart(cfg.Name, cfg.RepoURL, cfg.Version)
-					if err != nil {
+					for _, cfg := range cfg.Spec.Charts {
+						tc, err := tchart.NewChart(cfg.Name, cfg.RepoURL, cfg.Version)
+						if err != nil {
+							return err
+						}
+						if _, err := s.AddCollection(ctx, tc); err != nil {
+							return err
+						}
+					}
+				// collection.hauler.cattle.io/v1alpha1 ImageTxts
+				case v1alpha1.ImageTxtsContentKind:
+					var cfg v1alpha1.ImageTxts
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
 						return err
 					}
 
-					if _, err := s.AddCollection(ctx, tc); err != nil {
-						return err
-					}
-				}
+					for _, cfgIt := range cfg.Spec.ImageTxts {
+						it, err := imagetxt.New(cfgIt.Ref,
+							imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
+							imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
+						)
+						if err != nil {
+							return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
+						}
 
+						if _, err := s.AddCollection(ctx, it); err != nil {
+							return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
+						}
+					}
+				// collection.hauler.cattle.io/v1alpha1 unknown
+				default:
+					return fmt.Errorf("unsupported Kind %s for %s", gvk.Kind, gvk.GroupVersion().String())
+				}
+			// content.hauler.cattle.io/v1alpha2 + collection.hauler.cattle.io/v1alpha2
+			case gvk.GroupVersion() == v1alpha2.ContentGroupVersion || gvk.GroupVersion() == v1alpha2.CollectionGroupVersion:
+				return fmt.Errorf("API group + version %s not yet supported", gvk.GroupVersion().String())
+			// unknown
 			default:
 				return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
 			}

pkg/apis/hauler.cattle.io/v1alpha1/imagetxt.go

@@ -0,0 +1,30 @@
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ImageTxtsContentKind = "ImageTxts"
+)
+
+type ImageTxts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ImageTxtsSpec `json:"spec,omitempty"`
+}
+
+type ImageTxtsSpec struct {
+	ImageTxts []ImageTxt `json:"imageTxts,omitempty"`
+}
+
+type ImageTxt struct {
+	Ref     string          `json:"ref,omitempty"`
+	Sources ImageTxtSources `json:"sources,omitempty"`
+}
+
+type ImageTxtSources struct {
+	Include []string `json:"include,omitempty"`
+	Exclude []string `json:"exclude,omitempty"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/chart.go

@@ -0,0 +1,47 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ChartsContentKind    = "Charts"
+	ChartsCollectionKind = "ThickCharts"
+)
+
+type Charts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ChartSpec `json:"spec,omitempty"`
+}
+
+type ChartSpec struct {
+	Charts []Chart `json:"charts,omitempty"`
+}
+
+type Chart struct {
+	Name    string `json:"name"`
+	RepoURL string `json:"repoURL"`
+	Version string `json:"version"`
+}
+
+type ThickCharts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ChartSpec `json:"spec,omitempty"`
+}
+
+type ThickChartSpec struct {
+	Charts []ThickChart `json:"charts,omitempty"`
+}
+
+type ThickChart struct {
+	Chart       `json:",inline,omitempty"`
+	ExtraImages []ChartImage `json:"extraImages,omitempty"`
+}
+
+type ChartImage struct {
+	Reference string `json:"ref"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/driver.go

@@ -0,0 +1,21 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	DriverContentKind = "Driver"
+)
+
+type Driver struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec DriverSpec `json:"spec"`
+}
+
+type DriverSpec struct {
+	Type    string `json:"type"`
+	Version string `json:"version"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/file.go

@@ -0,0 +1,27 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const FilesContentKind = "Files"
+
+type Files struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec FileSpec `json:"spec,omitempty"`
+}
+
+type FileSpec struct {
+	Files []File `json:"files,omitempty"`
+}
+
+type File struct {
+	// Path is the path to the file contents, can be a local or remote path
+	Path string `json:"path"`
+
+	// Name is an optional field specifying the name of the file. When specified,
+	// it will override any dynamic name discovery from Path
+	Name string `json:"name,omitempty"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/groupversion_info.go

@@ -0,0 +1,19 @@
+package v1alpha2
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+const (
+	Version         = "v1alpha2"
+	ContentGroup    = "content.hauler.cattle.io"
+	CollectionGroup = "collection.hauler.cattle.io"
+)
+
+var (
+	ContentGroupVersion = schema.GroupVersion{Group: ContentGroup, Version: Version}
+	SchemeBuilder       = &scheme.Builder{GroupVersion: ContentGroupVersion}
+
+	CollectionGroupVersion = schema.GroupVersion{Group: CollectionGroup, Version: Version}
+)

pkg/apis/hauler.cattle.io/v1alpha2/image.go

@@ -0,0 +1,22 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const ImagesContentKind = "Images"
+
+type Images struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ImageSpec `json:"spec,omitempty"`
+}
+
+type ImageSpec struct {
+	Images []Image `json:"images,omitempty"`
+}
+
+type Image struct {
+	Name string `json:"name"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/imagetxt.go

@@ -0,0 +1,30 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ImageTxtsContentKind = "ImageTxts"
+)
+
+type ImageTxts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ImageTxtsSpec `json:"spec,omitempty"`
+}
+
+type ImageTxtsSpec struct {
+	ImageTxts []ImageTxt `json:"imageTxts,omitempty"`
+}
+
+type ImageTxt struct {
+	Name    string          `json:"name,omitempty"`
+	Sources ImageTxtSources `json:"sources,omitempty"`
+}
+
+type ImageTxtSources struct {
+	Include []string `json:"include,omitempty"`
+	Exclude []string `json:"exclude,omitempty"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/k3s.go

@@ -0,0 +1,17 @@
+package v1alpha2
+
+import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+const K3sCollectionKind = "K3s"
+
+type K3s struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec K3sSpec `json:"spec,omitempty"`
+}
+
+type K3sSpec struct {
+	Version string `json:"version"`
+	Arch    string `json:"arch"`
+}

pkg/artifact/local/layer.go

@@ -9,8 +9,6 @@ import (
 	"github.com/rancherfederal/hauler/pkg/artifact/types"
 )
 
-type Opener func() (io.ReadCloser, error)
-
 func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 	var err error
 

pkg/artifact/local/opener.go

@@ -0,0 +1,25 @@
+package local
+
+import (
+	"io"
+	"net/http"
+	"os"
+)
+
+type Opener func() (io.ReadCloser, error)
+
+func LocalOpener(path string) Opener {
+	return func() (io.ReadCloser, error) {
+		return os.Open(path)
+	}
+}
+
+func RemoteOpener(url string) Opener {
+	return func() (io.ReadCloser, error) {
+		resp, err := http.Get(url)
+		if err != nil {
+			return nil, err
+		}
+		return resp.Body, nil
+	}
+}

pkg/collection/imagetxt/imagetxt.go

@@ -0,0 +1,240 @@
+package imagetxt
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/rancherfederal/hauler/pkg/artifact"
+	"github.com/rancherfederal/hauler/pkg/artifact/local"
+	"github.com/rancherfederal/hauler/pkg/content/image"
+	"github.com/rancherfederal/hauler/pkg/log"
+
+	"github.com/google/go-containerregistry/pkg/name"
+)
+
+type ImageTxt struct {
+	Ref            string
+	IncludeSources map[string]bool
+	ExcludeSources map[string]bool
+
+	lock     *sync.Mutex
+	getter   local.Opener
+	computed bool
+	contents map[name.Reference]artifact.OCI
+}
+
+var _ artifact.Collection = (*ImageTxt)(nil)
+
+type Option interface {
+	Apply(*ImageTxt) error
+}
+
+type withIncludeSources []string
+
+func (o withIncludeSources) Apply(it *ImageTxt) error {
+	if it.IncludeSources == nil {
+		it.IncludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.IncludeSources[s] = true
+	}
+	return nil
+}
+
+func WithIncludeSources(include ...string) Option {
+	return withIncludeSources(include)
+}
+
+type withExcludeSources []string
+
+func (o withExcludeSources) Apply(it *ImageTxt) error {
+	if it.ExcludeSources == nil {
+		it.ExcludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.ExcludeSources[s] = true
+	}
+	return nil
+}
+
+func WithExcludeSources(exclude ...string) Option {
+	return withExcludeSources(exclude)
+}
+
+func New(ref string, opts ...Option) (*ImageTxt, error) {
+	it := &ImageTxt{
+		Ref: ref,
+
+		lock: &sync.Mutex{},
+	}
+
+	if strings.HasPrefix(ref, "http") || strings.HasPrefix(ref, "https") {
+		it.getter = local.RemoteOpener(ref)
+	} else {
+		it.getter = local.LocalOpener(ref)
+	}
+
+	for i, o := range opts {
+		if err := o.Apply(it); err != nil {
+			return nil, fmt.Errorf("invalid option %d: %v", i, err)
+		}
+	}
+
+	return it, nil
+}
+
+func (it *ImageTxt) Contents() (map[name.Reference]artifact.OCI, error) {
+	it.lock.Lock()
+	defer it.lock.Unlock()
+	if !it.computed {
+		if err := it.compute(); err != nil {
+			return nil, fmt.Errorf("compute OCI layout: %v", err)
+		}
+		it.computed = true
+	}
+	return it.contents, nil
+}
+
+func (it *ImageTxt) compute() error {
+	// TODO - pass in logger from context
+	l := log.NewLogger(os.Stdout)
+
+	it.contents = make(map[name.Reference]artifact.OCI)
+
+	r, err := it.getter()
+	if err != nil {
+		return fmt.Errorf("fetch image.txt ref %s: %v", it.Ref, err)
+	}
+	defer r.Close()
+
+	buf := &bytes.Buffer{}
+	if _, err := io.Copy(buf, r); err != nil {
+		return fmt.Errorf("read image.txt ref %s: %v", it.Ref, err)
+	}
+
+	entries, err := splitImagesTxt(buf)
+	if err != nil {
+		return fmt.Errorf("parse image.txt ref %s: %v", it.Ref, err)
+	}
+
+	foundSources := make(map[string]bool)
+	for _, e := range entries {
+		for s := range e.Sources {
+			foundSources[s] = true
+		}
+	}
+
+	var pullAll bool
+	targetSources := make(map[string]bool)
+
+	if len(foundSources) == 0 || (len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0) {
+		// pull all found images
+		pullAll = true
+
+		if len(foundSources) == 0 {
+			l.Infof("image txt file appears to have no sources; pulling all found images")
+			if len(it.IncludeSources) != 0 || len(it.ExcludeSources) != 0 {
+				l.Warnf("ImageTxt provided include or exclude sources; ignoring")
+			}
+		} else if len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0 {
+			l.Infof("image-sources txt file not filtered; pulling all found images")
+		}
+	} else {
+		// determine sources to pull
+		if len(it.IncludeSources) != 0 && len(it.ExcludeSources) != 0 {
+			l.Warnf("ImageTxt provided include and exclude sources; using only include sources")
+		}
+
+		if len(it.IncludeSources) != 0 {
+			targetSources = it.IncludeSources
+		} else {
+			for s := range foundSources {
+				targetSources[s] = true
+			}
+			for s := range it.ExcludeSources {
+				delete(targetSources, s)
+			}
+		}
+		var targetSourcesArr []string
+		for s := range targetSources {
+			targetSourcesArr = append(targetSourcesArr, s)
+		}
+		l.Infof("pulling images covering sources %s", strings.Join(targetSourcesArr, ", "))
+	}
+
+	for _, e := range entries {
+		var matchesSourceFilter bool
+		if pullAll {
+			l.Infof("pulling image %s", e.Reference)
+		} else {
+			for s := range e.Sources {
+				if targetSources[s] {
+					matchesSourceFilter = true
+					l.Infof("pulling image %s (matched source %s)", e.Reference, s)
+					break
+				}
+			}
+		}
+
+		if pullAll || matchesSourceFilter {
+			curImage, err := image.NewImage(e.Reference.String())
+			if err != nil {
+				return fmt.Errorf("pull image %s: %v", e.Reference, err)
+			}
+			it.contents[e.Reference] = curImage
+		}
+	}
+
+	return nil
+}
+
+type imageTxtEntry struct {
+	Reference name.Reference
+	Sources   map[string]bool
+}
+
+func splitImagesTxt(r io.Reader) ([]imageTxtEntry, error) {
+	var entries []imageTxtEntry
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		curEntry := imageTxtEntry{
+			Sources: make(map[string]bool),
+		}
+
+		lineContent := scanner.Text()
+		if lineContent == "" || strings.HasPrefix(lineContent, "#") {
+			// skip past empty and commented lines
+			continue
+		}
+		splitContent := strings.Split(lineContent, " ")
+		if len(splitContent) > 2 {
+			return nil, fmt.Errorf(
+				"invalid image.txt format: must contain only an image reference and sources separated by space; invalid line: %q",
+				lineContent)
+		}
+
+		curRef, err := name.ParseReference(splitContent[0])
+		if err != nil {
+			return nil, fmt.Errorf("invalid reference %s: %v", splitContent[0], err)
+		}
+		curEntry.Reference = curRef
+
+		if len(splitContent) == 2 {
+			for _, source := range strings.Split(splitContent[1], ",") {
+				curEntry.Sources[source] = true
+			}
+		}
+
+		entries = append(entries, curEntry)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("scan contents: %v", err)
+	}
+
+	return entries, nil
+}

pkg/collection/imagetxt/imagetxt_test.go

@@ -0,0 +1,216 @@
+package imagetxt
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/name"
+
+	artifacts "github.com/rancherfederal/hauler/pkg/artifact"
+	"github.com/rancherfederal/hauler/pkg/content/image"
+)
+
+var (
+	ErrInvalidRef     = errors.New("invalid reference")
+	ErrRefNotFound    = errors.New("ref not found")
+	ErrRefNotImage    = errors.New("ref is not image")
+	ErrExtraRefsFound = errors.New("extra refs found in contents")
+)
+
+var (
+	testServer *httptest.Server
+)
+
+func TestMain(m *testing.M) {
+	setup()
+	code := m.Run()
+	teardown()
+	os.Exit(code)
+}
+
+func setup() {
+	dir := http.Dir("./testdata/http/")
+	h := http.FileServer(dir)
+	testServer = httptest.NewServer(h)
+}
+
+func teardown() {
+	if testServer != nil {
+		testServer.Close()
+	}
+}
+
+type failKind string
+
+const (
+	failKindNew      = failKind("New")
+	failKindContents = failKind("Contents")
+)
+
+func checkError(checkedFailKind failKind) func(*testing.T, error, bool, failKind) {
+	return func(cet *testing.T, err error, testShouldFail bool, testFailKind failKind) {
+		if err != nil {
+			// if error should not have happened at all OR error should have happened
+			// at a different point, test failed
+			if !testShouldFail || testFailKind != checkedFailKind {
+				cet.Fatalf("unexpected error at %s: %v", checkedFailKind, err)
+			}
+			// test should fail at this point, test passed
+			return
+		}
+		// if no error occurred but error should have happened at this point, test
+		// failed
+		if testShouldFail && testFailKind == checkedFailKind {
+			cet.Fatalf("unexpected nil error at %s", checkedFailKind)
+		}
+	}
+}
+
+func TestImageTxtCollection(t *testing.T) {
+	type testEntry struct {
+		Name           string
+		Ref            string
+		IncludeSources []string
+		ExcludeSources []string
+		ExpectedImages []string
+		ShouldFail     bool
+		FailKind       failKind
+	}
+	tt := []testEntry{
+		{
+			Name: "http ref basic",
+			Ref:  fmt.Sprintf("%s/images-http.txt", testServer.URL),
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format pull all",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format include sources A",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			IncludeSources: []string{
+				"core", "rke",
+			},
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+			},
+		},
+		{
+			Name: "http ref sources format include sources B",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			IncludeSources: []string{
+				"nginx", "rancher", "cert-manager",
+			},
+			ExpectedImages: []string{
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format exclude sources A",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExcludeSources: []string{
+				"cert-manager",
+			},
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+			},
+		},
+		{
+			Name: "http ref sources format exclude sources B",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExcludeSources: []string{
+				"core",
+			},
+			ExpectedImages: []string{
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "local file ref",
+			Ref:  "./testdata/images-file.txt",
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+	}
+
+	checkErrorNew := checkError(failKindNew)
+	checkErrorContents := checkError(failKindContents)
+
+	for _, curTest := range tt {
+		t.Run(curTest.Name, func(innerT *testing.T) {
+			curImageTxt, err := New(curTest.Ref,
+				WithIncludeSources(curTest.IncludeSources...),
+				WithExcludeSources(curTest.ExcludeSources...),
+			)
+			checkErrorNew(innerT, err, curTest.ShouldFail, curTest.FailKind)
+
+			ociContents, err := curImageTxt.Contents()
+			checkErrorContents(innerT, err, curTest.ShouldFail, curTest.FailKind)
+
+			if err := checkImages(ociContents, curTest.ExpectedImages); err != nil {
+				innerT.Fatal(err)
+			}
+		})
+	}
+}
+
+func checkImages(content map[name.Reference]artifacts.OCI, refs []string) error {
+	contentCopy := make(map[name.Reference]artifacts.OCI, len(content))
+	for k, v := range content {
+		contentCopy[k] = v
+	}
+	for _, ref := range refs {
+		nameRef, err := name.ParseReference(ref)
+		if err != nil {
+			return fmt.Errorf("ref %s: %w", ref, ErrInvalidRef)
+		}
+		target, ok := content[nameRef]
+		if !ok {
+			return fmt.Errorf("ref %s: %w", ref, ErrRefNotFound)
+		}
+		if _, ok := target.(*image.Image); !ok {
+			return fmt.Errorf("got underlying type %T: %w", target, ErrRefNotImage)
+		}
+		delete(contentCopy, nameRef)
+	}
+
+	if len(contentCopy) != 0 {
+		return ErrExtraRefsFound
+	}
+
+	return nil
+}

pkg/collection/imagetxt/testdata/http/images-http.txt

@@ -0,0 +1,5 @@
+busybox
+nginx:1.19
+rancher/hyperkube:v1.21.7-rancher1
+docker.io/rancher/klipper-lb:v0.3.4
+quay.io/jetstack/cert-manager-controller:v1.6.1

pkg/collection/imagetxt/testdata/http/images-src-http.txt

@@ -0,0 +1,5 @@
+busybox core
+nginx:1.19 core,nginx
+rancher/hyperkube:v1.21.7-rancher1 rancher,rke
+docker.io/rancher/klipper-lb:v0.3.4 rancher,k3s
+quay.io/jetstack/cert-manager-controller:v1.6.1 cert-manager

pkg/collection/imagetxt/testdata/images-file.txt

@@ -0,0 +1,5 @@
+busybox
+nginx:1.19
+rancher/hyperkube:v1.21.7-rancher1
+docker.io/rancher/klipper-lb:v0.3.4
+quay.io/jetstack/cert-manager-controller:v1.6.1

pkg/content/content.go

@@ -8,6 +8,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
+	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha2"
 )
 
 func Load(data []byte) (schema.ObjectKind, error) {
@@ -16,8 +17,9 @@ func Load(data []byte) (schema.ObjectKind, error) {
 		return nil, err
 	}
 
-	if tm.GroupVersionKind().GroupVersion() != v1alpha1.ContentGroupVersion && tm.GroupVersionKind().GroupVersion() != v1alpha1.CollectionGroupVersion {
-		return nil, fmt.Errorf("unrecognized content/collection type: %s", tm.GroupVersionKind().String())
+	gv := tm.GroupVersionKind().GroupVersion()
+	if gv != v1alpha1.ContentGroupVersion && gv != v1alpha1.CollectionGroupVersion && gv != v1alpha2.ContentGroupVersion && gv != v1alpha2.CollectionGroupVersion {
+		return nil, fmt.Errorf("unrecognized API type: %s", tm.GroupVersionKind().String())
 	}
 
 	return tm, nil

pkg/content/image/image.go

@@ -8,9 +8,9 @@ import (
 	"github.com/rancherfederal/hauler/pkg/artifact"
 )
 
-var _ artifact.OCI = (*image)(nil)
+var _ artifact.OCI = (*Image)(nil)
 
-func (i *image) MediaType() string {
+func (i *Image) MediaType() string {
 	mt, err := i.Image.MediaType()
 	if err != nil {
 		return ""
@@ -18,15 +18,15 @@ func (i *image) MediaType() string {
 	return string(mt)
 }
 
-func (i *image) RawConfig() ([]byte, error) {
+func (i *Image) RawConfig() ([]byte, error) {
 	return i.RawConfigFile()
 }
 
-type image struct {
+type Image struct {
 	gv1.Image
 }
 
-func NewImage(ref string) (*image, error) {
+func NewImage(ref string) (*Image, error) {
 	r, err := name.ParseReference(ref)
 	if err != nil {
 		return nil, err
@@ -37,7 +37,7 @@ func NewImage(ref string) (*image, error) {
 		return nil, err
 	}
 
-	return &image{
+	return &Image{
 		Image: img,
 	}, nil
 }