Wrap errors

[0.2] Backport auth, insecure, plain HTTP flags

.github/workflows/ci.yaml

@@ -29,3 +29,4 @@ jobs:
           args: release --rm-dist
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}

.goreleaser.yaml

@@ -3,6 +3,10 @@ before:
   hooks:
     - go mod tidy
     - go mod download
+
+env:
+  - vpkg=github.com/rancherfederal/hauler/pkg/version
+
 builds:
   - main: cmd/hauler/main.go
     goos:
@@ -12,6 +16,8 @@ builds:
     goarch:
       - amd64
       - arm64
+    ldflags:
+      - -s -w -X {{ .Env.vpkg }}.GitVersion={{ .Version }} -X {{ .Env.vpkg }}.commit={{ .ShortCommit }} -X {{ .Env.vpkg }}.buildDate={{ .Date }}
     env:
       - CGO_ENABLED=0
 

README.md

@@ -1,11 +1,28 @@
 # Hauler: Airgap Assistant
 
-__⚠️ WARNING: This is an experimental, work in progress project.  _Everything_ is subject to change, and it is actively in development, so let us know what you think!__
+> ⚠️ This project is still in active development and _not_ GA.  While a lot of the core features are ready, we're still adding a _ton_, and we may make breaking api and feature changes version to version.
 
-`hauler` is a command line tool for that aims to simplify the painpoints that exist around airgapped Kubernetes deployments.
-It remains as unopinionated as possible, and does _not_ attempt to enforce a specific cluster type or application deployment model.
-Instead, it focuses solely on simplifying the primary airgap pain points:
-* artifact collection
-* artifact distribution
+`hauler` simplifies the airgap experience without forcing you to adopt a specific workflow for your infrastructure or application.  
 
-`hauler` achieves this by leaning heavily on the [oci spec](https://github.com/opencontainers), and the vast ecosystem of tooling available for fetching and distributing oci content.
+To accomplish this, it focuses strictly on two of the biggest airgap pain points:
+
+* content collection
+* content distribution
+
+As OCI registries have become ubiquitous nowadays for storing and distributing containers.  Their success and widespread adoption has led many projects to expand beyond containers.
+
+`hauler` capitalizes on this, and leverages the [`oci`](https://github.com/opencontainers) spec to be a simple, zero dependency tool to collect, transport, and distribute your artifacts.
+
+## Getting started
+
+See the [quickstart](docs/walkthrough.md#Quickstart) for a quick way to get started with some of `haulers` capabilities.
+
+For a guided example of all of `haulers` capabilities, check out the [guided example](docs/walkthrough.md#guided-examples).
+
+## Acknowledgements
+
+`hauler` wouldn't be possible without the open source community, but there are a few dependent projects that stand out:
+
+* [go-containerregistry](https://github.com/google/go-containerregistry)
+* [oras](https://github.com/oras-project/oras)
+* [cosign](https://github.com/sigstore/cosign)

ROADMAP.md

@@ -1,10 +1,30 @@
 # Hauler Roadmap
 
-## v0.0.x
+## \> v0.2.0
 
-- Install single-node k3s cluster into an Ubuntu machine using the tarball installation method
+- Leverage `referrers` api to robustly link content/collection
+- Support signing for all `artifact.OCI` contents
+- Support encryption for `artifact.OCI` layers
+- Support incremental updates to stores (some implementation of layer diffing)
+- Safely embed container runtime for user created `collections` creation and transformation
+- Better defaults/configuration/security around for long-lived embedded registry
+- Better support multi-platform content
+- Better leverage `oras` (`>=0.5.0`) for content relocation
+- Store git repos as CAS in OCI format
 
-## v0.1.0
+## v0.2.0 - MVP 2
+
+- Re-focus on cli and framework for oci content fetching and delivery
+- Focus on initial key contents
+  - Files (local/remote)
+  - Charts (local/remote)
+  - Images
+- Establish framework for `content` and `collections`
+- Define initial `content` types (`file`, `chart`, `image`)
+- Define initial `collection` types (`thickchart`, `k3s`)
+- Define framework for manipulating OCI content (`artifact.OCI`, `artifact.Collection`)
+
+## v0.1.0 - MVP 1
 
 - Install single-node k3s cluster
   - Support tarball and rpm installation methods
@@ -25,18 +45,6 @@
   - NOTE: "generic" option - most other use cases can be satisfied by a specially crafted file
     server directory
 
+## v0.0.x
 
-## Potential future features
-
-- Helm charts
-  - Pull charts, migrate chart artifacts
-  - Analyze required container images, add to dependency list
-- Yum repo
-  - Provide package list, collect all dependencies
-  - Deploy fully configured yum repo into file server
-- Deploy Minio for S3 API
-  - MVP: backed by HA storage solution (e.g. AWS S3, Azure Blob Storage)
-  - Stable: backed by local storage, including backups
-- Split archives into chunks of chosen size
-  - Enables easier transfer via physical media
-  - Allows smaller network transfers, losing less progress on failed upload (or working around timeouts)
+- Install single-node k3s cluster into an Ubuntu machine using the tarball installation method

cmd/hauler/cli/cli.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -28,7 +29,9 @@ func New() *cobra.Command {
 		Use:   "hauler",
 		Short: "",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			log.FromContext(cmd.Context()).SetLevel(ro.logLevel)
+			l := log.FromContext(cmd.Context())
+			l.SetLevel(ro.logLevel)
+			l.Debugf("running cli command [%s]", cmd.CommandPath())
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -37,50 +40,46 @@ func New() *cobra.Command {
 	}
 
 	pf := cmd.PersistentFlags()
-	pf.StringVarP(&ro.logLevel, "log-level", "l", "info", "")
-	pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to $XDG_CACHE_DIR/hauler)")
+	pf.StringVarP(&ro.logLevel, "log-level", "l", "info", `Verbosity of logs ("debug", info", "warn", "error")`)
+	pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to $XDG_CACHE_HOME/hauler)")
 	pf.StringVarP(&ro.storeDir, "store", "s", "", "Location to create store at (defaults to $PWD/store)")
 
 	// Add subcommands
 	addDownload(cmd)
 	addStore(cmd)
+	addVersion(cmd)
 
 	return cmd
 }
 
 func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
-	lgr := log.FromContext(ctx)
+	l := log.FromContext(ctx)
 	dir := o.storeDir
 
 	if dir == "" {
-		lgr.Debugf("no store path specified, defaulting to $PWD/store")
-		pwd, err := os.Getwd()
-		if err != nil {
-			return nil, err
-		}
-
-		dir = filepath.Join(pwd, defaultStoreLocation)
+		l.Debugf("no store path specified, defaulting to $PWD/store")
+		dir = defaultStoreLocation
 	}
 
 	abs, err := filepath.Abs(dir)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("resolve store absolute path: %w", err)
 	}
 
-	lgr.Debugf("using store at %s", abs)
+	l.Debugf("using store at %s", abs)
 	if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) {
 		err := os.Mkdir(abs, os.ModePerm)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("create store directory %s: %w", abs, err)
 		}
 	} else if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("stat %s: %w", abs, err)
 	}
 
 	// TODO: Do we want this to be configurable?
 	c, err := o.getCache(ctx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("get cache: %w", err)
 	}
 
 	s := store.NewStore(ctx, abs, store.WithCache(c))
@@ -91,18 +90,17 @@ func (o *rootOpts) getCache(ctx context.Context) (cache.Cache, error) {
 	dir := o.cacheDir
 
 	if dir == "" {
-		// Default to $XDG_CACHE_DIR
-		cachedir, err := os.UserCacheDir()
+		// Default to $XDG_CACHE_HOME/hauler
+		userCacheDir, err := os.UserCacheDir()
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("get default cache directory: %w", err)
 		}
+		dir = filepath.Join(userCacheDir, "hauler")
+	}
 
-		abs, _ := filepath.Abs(filepath.Join(cachedir, "hauler"))
-		if err := os.MkdirAll(abs, os.ModePerm); err != nil {
-			return nil, err
-		}
-
-		dir = abs
+	abs, _ := filepath.Abs(dir)
+	if err := os.MkdirAll(abs, os.ModePerm); err != nil {
+		return nil, fmt.Errorf("create cache directory %s: %w", abs, err)
 	}
 
 	c := cache.NewFilesystem(dir)

cmd/hauler/cli/download/download.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"path"
 
 	"github.com/containerd/containerd/remotes/docker"
@@ -18,35 +19,99 @@ import (
 
 	"github.com/rancherfederal/hauler/pkg/artifact/types"
 	"github.com/rancherfederal/hauler/pkg/log"
+	"github.com/rancherfederal/hauler/pkg/version"
 )
 
 type Opts struct {
 	DestinationDir string
-	OutputFile     string
+
+	Username  string
+	Password  string
+	Insecure  bool
+	PlainHTTP bool
 }
 
 func (o *Opts) AddArgs(cmd *cobra.Command) {
 	f := cmd.Flags()
 
-	f.StringVar(&o.DestinationDir, "dir", "", "Directory to save contents to (defaults to current directory)")
-	f.StringVarP(&o.OutputFile, "output", "o", "", "(Optional) Override name of file to save.")
+	f.StringVarP(&o.DestinationDir, "output", "o", "", "Directory to save contents to (defaults to current directory)")
+	f.StringVarP(&o.Username, "username", "u", "", "Username when copying to an authenticated remote registry")
+	f.StringVarP(&o.Password, "password", "p", "", "Password when copying to an authenticated remote registry")
+	f.BoolVar(&o.Insecure, "insecure", false, "Toggle allowing insecure connections when copying to a remote registry")
+	f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry")
 }
 
 func Cmd(ctx context.Context, o *Opts, reference string) error {
-	lgr := log.FromContext(ctx)
-	lgr.Debugf("running command `hauler download`")
+	l := log.FromContext(ctx)
 
 	cs := content.NewFileStore(o.DestinationDir)
 	defer cs.Close()
 
-	ref, err := name.ParseReference(reference)
+	// build + configure oras client
+	var refOpts []name.Option
+	remoteOpts := []remote.Option{
+		remote.WithAuthFromKeychain(authn.DefaultKeychain),
+	}
+
+	if o.PlainHTTP {
+		refOpts = append(refOpts, name.Insecure)
+	}
+
+	if o.Username != "" || o.Password != "" {
+		basicAuth := &authn.Basic{
+			Username: o.Username,
+			Password: o.Password,
+		}
+		remoteOpts = append(remoteOpts, remote.WithAuth(basicAuth))
+	}
+
+	if o.Insecure {
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig.InsecureSkipVerify = true
+
+		remoteOpts = append(remoteOpts, remote.WithTransport(transport))
+	}
+
+	// build + configure containerd client
+	var registryOpts []docker.RegistryOpt
+
+	if o.PlainHTTP {
+		registryOpts = append(registryOpts, docker.WithPlainHTTP(docker.MatchAllHosts))
+	}
+
+	if o.Username != "" || o.Password != "" {
+		creds := func(string) (string, string, error) {
+			return o.Username, o.Password, nil
+		}
+		authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(creds))
+		registryOpts = append(registryOpts, docker.WithAuthorizer(authorizer))
+	}
+
+	if o.Insecure {
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig.InsecureSkipVerify = true
+
+		httpClient := &http.Client{
+			Transport: transport,
+		}
+		registryOpts = append(registryOpts, docker.WithClient(httpClient))
+	}
+
+	resolverOpts := docker.ResolverOptions{
+		Hosts:   docker.ConfigureDefaultRegistries(registryOpts...),
+		Headers: http.Header{},
+	}
+	resolverOpts.Headers.Set("User-Agent", "hauler/"+version.GitVersion)
+
+	resolver := docker.NewResolver(resolverOpts)
+
+	// begin dowloading target
+	ref, err := name.ParseReference(reference, refOpts...)
 	if err != nil {
 		return err
 	}
 
-	// resolver := docker.NewResolver(docker.ResolverOptions{})
-
-	desc, err := remote.Get(ref)
+	desc, err := remote.Get(ref, remoteOpts...)
 	if err != nil {
 		return err
 	}
@@ -64,48 +129,56 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 	// TODO: These need to be factored out into each of the contents own logic
 	switch manifest.Config.MediaType {
 	case types.DockerConfigJSON, types.OCIManifestSchema1:
-		lgr.Infof("identified [image] (%s) content", manifest.Config.MediaType)
-		img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+		l.Debugf("identified [image] (%s) content", manifest.Config.MediaType)
+		img, err := remote.Image(ref, remoteOpts...)
 		if err != nil {
 			return err
 		}
 
-		outputFile := o.OutputFile
-		if outputFile == "" {
-			outputFile = fmt.Sprintf("%s:%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier())
-		}
+		outputFile := fmt.Sprintf("%s_%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier())
 
 		if err := tarball.WriteToFile(outputFile, ref, img); err != nil {
 			return err
 		}
 
-		lgr.Infof("downloaded [%s] to [%s]", ref.Name(), outputFile)
+		l.Infof("downloaded image [%s] to [%s]", ref.Name(), outputFile)
 
-	case types.FileMediaType:
-		lgr.Infof("identified [file] (%s) content", manifest.Config.MediaType)
+	case types.FileConfigMediaType:
+		l.Debugf("identified [file] (%s) content", manifest.Config.MediaType)
 
 		fs := content.NewFileStore(o.DestinationDir)
 
-		resolver := docker.NewResolver(docker.ResolverOptions{})
-		mdesc, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
+		// TODO - additional accepted media types
+		_, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
 		if err != nil {
 			return err
 		}
 
-		lgr.Infof("downloaded [%d] files with digest [%s]", len(descs), mdesc)
+		ldescs := len(descs)
+		for i, desc := range descs {
+			// NOTE: This is safe without a map key check b/c we're not allowing unnamed content from oras.Pull
+			l.Infof("downloaded (%d/%d) files to [%s]", i+1, ldescs, desc.Annotations[ocispec.AnnotationTitle])
+		}
 
 	case types.ChartLayerMediaType, types.ChartConfigMediaType:
-		lgr.Infof("identified [chart] (%s) content", manifest.Config.MediaType)
+		l.Debugf("identified [chart] (%s) content", manifest.Config.MediaType)
 
 		fs := content.NewFileStore(o.DestinationDir)
 
-		resolver := docker.NewResolver(docker.ResolverOptions{})
-		mdesc, _, err := oras.Pull(ctx, resolver, ref.Name(), fs)
+		// TODO - additional accepted media types
+		_, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
 		if err != nil {
 			return err
 		}
 
-		lgr.Infof("downloaded chart [%s] with digest [%s]", "donkey", mdesc.Digest.String())
+		cn := path.Base(ref.Name())
+		for _, d := range descs {
+			if n, ok := d.Annotations[ocispec.AnnotationTitle]; ok {
+				cn = n
+			}
+		}
+
+		l.Infof("downloaded chart [%s] to [%s]", ref.String(), cn)
 
 	default:
 		return fmt.Errorf("unrecognized content type: %s", manifest.Config.MediaType)
@@ -113,17 +186,3 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 
 	return nil
 }
-
-func getManifest(ctx context.Context, ref string) (*remote.Descriptor, error) {
-	r, err := name.ParseReference(ref)
-	if err != nil {
-		return nil, fmt.Errorf("parsing reference %q: %v", ref, err)
-	}
-
-	desc, err := remote.Get(r, remote.WithContext(ctx))
-	if err != nil {
-		return nil, err
-	}
-
-	return desc, nil
-}

cmd/hauler/cli/store.go

@@ -8,8 +8,9 @@ import (
 
 func addStore(parent *cobra.Command) {
 	cmd := &cobra.Command{
-		Use:   "store",
-		Short: "Interact with hauler's embedded content store",
+		Use:     "store",
+		Aliases: []string{"s"},
+		Short:   "Interact with hauler's embedded content store",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return cmd.Help()
 		},
@@ -149,9 +150,10 @@ func addStoreList() *cobra.Command {
 	o := &store.ListOpts{}
 
 	cmd := &cobra.Command{
-		Use:   "list",
-		Short: "List all content references in a store",
-		Args:  cobra.ExactArgs(0),
+		Use:     "list",
+		Short:   "List all content references in a store",
+		Args:    cobra.ExactArgs(0),
+		Aliases: []string{"ls"},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
@@ -263,7 +265,7 @@ func addStoreAddChart() *cobra.Command {
 		Short: "Add a chart to the content store",
 		Example: `
 # add a chart
-hauler store add longhorn --repo "https://charts.longhorn.io"
+hauler store add chart longhorn --repo "https://charts.longhorn.io"
 
 # add a specific version of a chart
 hauler store add chart rancher --repo "https://releases.rancher.com/server-charts/latest" --version "2.6.2"

cmd/hauler/cli/store/add.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"github.com/google/go-containerregistry/pkg/name"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
@@ -25,9 +26,6 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) {
 }
 
 func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference string) error {
-	lgr := log.FromContext(ctx)
-	lgr.Debugf("running cli command `hauler store add`")
-
 	s.Open()
 	defer s.Close()
 
@@ -40,12 +38,12 @@ func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference s
 }
 
 func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error {
-	lgr := log.FromContext(ctx)
+	l := log.FromContext(ctx)
 
 	if fi.Name == "" {
 		base := filepath.Base(fi.Ref)
 		fi.Name = filepath.Base(fi.Ref)
-		lgr.Warnf("no name specified for file reference [%s], using base filepath: [%s]", fi.Ref, base)
+		l.Warnf("no name specified for file reference [%s], using base filepath: [%s]", fi.Ref, base)
 	}
 
 	oci, err := file.NewFile(fi.Ref, fi.Name)
@@ -63,7 +61,7 @@ func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error {
 		return err
 	}
 
-	lgr.Infof("added file [%s] to store at [%s] with manifest digest [%s]", fi.Ref, ref.Name(), desc.Digest.String())
+	l.Infof("file [%s] added at: [%s]", ref.Name(), desc.Annotations[ocispec.AnnotationTitle])
 	return nil
 }
 
@@ -77,9 +75,6 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
 }
 
 func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference string) error {
-	lgr := log.FromContext(ctx)
-	lgr.Debugf("running cli command `hauler store add image`")
-
 	s.Open()
 	defer s.Close()
 
@@ -91,7 +86,7 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference
 }
 
 func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error {
-	lgr := log.FromContext(ctx)
+	l := log.FromContext(ctx)
 
 	oci, err := image.NewImage(i.Ref)
 	if err != nil {
@@ -108,7 +103,7 @@ func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error {
 		return err
 	}
 
-	lgr.Infof("added image [%s] to store at [%s] with manifest digest [%s]", i.Ref, ref.Context().RepositoryStr(), desc.Digest.String())
+	l.Infof("image [%s] added at: [%s]", ref.Name(), desc.Annotations[ocispec.AnnotationTitle])
 	return nil
 }
 
@@ -136,9 +131,6 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
 }
 
 func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName string) error {
-	lgr := log.FromContext(ctx)
-	lgr.Debugf("running cli command `hauler store add chart`")
-
 	s.Open()
 	defer s.Close()
 
@@ -152,7 +144,7 @@ func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName
 }
 
 func storeChart(ctx context.Context, s *store.Store, ch v1alpha1.Chart) error {
-	lgr := log.FromContext(ctx)
+	l := log.FromContext(ctx)
 
 	oci, err := chart.NewChart(ch.Name, ch.RepoURL, ch.Version)
 	if err != nil {
@@ -174,6 +166,6 @@ func storeChart(ctx context.Context, s *store.Store, ch v1alpha1.Chart) error {
 		return err
 	}
 
-	lgr.Infof("added chart [%s] to store at [%s:%s] with manifest digest [%s]", ch.Name, ref.Context().RepositoryStr(), ref.Identifier(), desc.Digest.String())
+	l.Infof("chart [%s] added at: [%s]", ref.Name(), desc.Annotations[ocispec.AnnotationTitle])
 	return nil
 }

cmd/hauler/cli/store/copy.go

@@ -21,8 +21,7 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) {
 }
 
 func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, registry string) error {
-	lgr := log.FromContext(ctx)
-	lgr.Debugf("running cli command `hauler store copy`")
+	l := log.FromContext(ctx)
 
 	s.Open()
 	defer s.Close()
@@ -48,7 +47,7 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, registry string)
 			return err
 		}
 
-		lgr.Infof("relocating [%s] -> [%s]", ref.Name(), rref.Name())
+		l.Infof("copying [%s] -> [%s]", ref.Name(), rref.Name())
 		if err := remote.Write(rref, o); err != nil {
 			return err
 		}

cmd/hauler/cli/store/extract.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/rancherfederal/hauler/cmd/hauler/cli/download"
 	"github.com/rancherfederal/hauler/pkg/layout"
-	"github.com/rancherfederal/hauler/pkg/log"
 	"github.com/rancherfederal/hauler/pkg/store"
 )
 
@@ -22,9 +21,6 @@ func (o *ExtractOpts) AddArgs(cmd *cobra.Command) {
 }
 
 func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, reference string) error {
-	l := log.FromContext(ctx)
-	l.Debugf("running command `hauler store extract`")
-
 	s.Open()
 	defer s.Close()
 

cmd/hauler/cli/store/list.go

@@ -9,7 +9,6 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/spf13/cobra"
 
-	"github.com/rancherfederal/hauler/pkg/log"
 	"github.com/rancherfederal/hauler/pkg/store"
 )
 
@@ -23,9 +22,6 @@ func (o *ListOpts) AddFlags(cmd *cobra.Command) {
 }
 
 func ListCmd(ctx context.Context, o *ListOpts, s *store.Store) error {
-	lgr := log.FromContext(ctx)
-	lgr.Debugf("running cli command `hauler store list`")
-
 	s.Open()
 	defer s.Close()
 
@@ -34,18 +30,17 @@ func ListCmd(ctx context.Context, o *ListOpts, s *store.Store) error {
 		return err
 	}
 
-	// TODO: Just use a tabler library
-	tw := tabwriter.NewWriter(os.Stdout, 8, 12, 4, '\t', 0)
+	tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
 	defer tw.Flush()
 
-	fmt.Fprintf(tw, "#\tReference\tIdentifier\n")
-	for i, r := range refs {
+	fmt.Fprintf(tw, "Reference\tTag/Digest\n")
+	for _, r := range refs {
 		ref, err := name.ParseReference(r, name.WithDefaultRegistry(""))
 		if err != nil {
 			return err
 		}
 
-		fmt.Fprintf(tw, "%d\t%s\t%s\n", i, ref.Context().String(), ref.Identifier())
+		fmt.Fprintf(tw, "%s\t%s\n", ref.Context().String(), ref.Identifier())
 	}
 
 	return nil

cmd/hauler/cli/store/load.go

@@ -9,26 +9,33 @@ import (
 	"github.com/rancherfederal/hauler/pkg/log"
 )
 
-type LoadOpts struct{}
+type LoadOpts struct {
+	OutputDir string
+}
 
 func (o *LoadOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
-	_ = f
+
+	f.StringVarP(&o.OutputDir, "output", "o", "", "Directory to unload archived contents to (defaults to $PWD/haul)")
 }
 
 // LoadCmd
 // TODO: Just use mholt/archiver for now, even though we don't need most of it
 func LoadCmd(ctx context.Context, o *LoadOpts, dir string, archiveRefs ...string) error {
 	l := log.FromContext(ctx)
-	l.Debugf("running command `hauler store load`")
 
 	// TODO: Support more formats?
 	a := archiver.NewTarZstd()
 	a.OverwriteExisting = true
 
+	odir := dir
+	if o.OutputDir != "" {
+		odir = o.OutputDir
+	}
+
 	for _, archiveRef := range archiveRefs {
-		l.Infof("Loading content from %s to %s", archiveRef, dir)
-		err := a.Unarchive(archiveRef, dir)
+		l.Infof("loading content from [%s] to [%s]", archiveRef, odir)
+		err := a.Unarchive(archiveRef, odir)
 		if err != nil {
 			return err
 		}

cmd/hauler/cli/store/save.go

@@ -25,7 +25,6 @@ func (o *SaveOpts) AddArgs(cmd *cobra.Command) {
 // TODO: Just use mholt/archiver for now, even though we don't need most of it
 func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) error {
 	l := log.FromContext(ctx)
-	l.Debugf("running command `hauler store save`")
 
 	// TODO: Support more formats?
 	a := archiver.NewTarZstd()
@@ -36,7 +35,6 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er
 		return err
 	}
 
-	l.Infof("Saving data dir (%s) as compressed archive to %s", dir, absOutputfile)
 	cwd, err := os.Getwd()
 	if err != nil {
 		return err
@@ -51,5 +49,6 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er
 		return err
 	}
 
+	l.Infof("saved haul [%s] -> [%s]", dir, absOutputfile)
 	return nil
 }

cmd/hauler/cli/store/serve.go

@@ -10,13 +10,13 @@ import (
 	"github.com/distribution/distribution/v3/registry"
 	"github.com/spf13/cobra"
 
-	"github.com/rancherfederal/hauler/pkg/log"
 	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type ServeOpts struct {
 	Port       int
 	ConfigFile string
+	Daemon     bool
 
 	storedir string
 }
@@ -26,13 +26,11 @@ func (o *ServeOpts) AddFlags(cmd *cobra.Command) {
 
 	f.IntVarP(&o.Port, "port", "p", 5000, "Port to listen on")
 	f.StringVarP(&o.ConfigFile, "config", "c", "", "Path to a config file, will override all other configs")
+	f.BoolVarP(&o.Daemon, "daemon", "d", false, "Toggle serving as a daemon")
 }
 
 // ServeCmd does
 func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error {
-	l := log.FromContext(ctx)
-	l.Debugf("running command `hauler store serve`")
-
 	cfg := o.defaultConfig(s)
 	if o.ConfigFile != "" {
 		ucfg, err := loadConfig(o.ConfigFile)
@@ -47,7 +45,6 @@ func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error {
 		return err
 	}
 
-	l.Infof("Starting registry listening on :%d", o.Port)
 	if err = r.ListenAndServe(); err != nil {
 		return err
 	}

cmd/hauler/cli/store/sync.go

@@ -11,7 +11,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
-	"github.com/rancherfederal/hauler/pkg/collection/chart"
+	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha2"
+	tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
+	"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
 	"github.com/rancherfederal/hauler/pkg/collection/k3s"
 	"github.com/rancherfederal/hauler/pkg/content"
 	"github.com/rancherfederal/hauler/pkg/log"
@@ -30,7 +32,6 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
 
 func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 	l := log.FromContext(ctx)
-	l.Debugf("running cli command `hauler store sync`")
 
 	// Start from an empty store (contents are cached elsewhere)
 	l.Debugf("flushing any existing content in store: %s", s.DataDir)
@@ -71,79 +72,121 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 
 			l.Infof("syncing [%s] to [%s]", obj.GroupVersionKind().String(), s.DataDir)
 
-			// TODO: Should type switch instead...
-			switch obj.GroupVersionKind().Kind {
-			case v1alpha1.FilesContentKind:
-				var cfg v1alpha1.Files
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
+			gvk := obj.GroupVersionKind()
 
-				for _, f := range cfg.Spec.Files {
-					err := storeFile(ctx, s, f)
+			switch {
+			// content.hauler.cattle.io/v1alpha1
+			case gvk.GroupVersion() == v1alpha1.ContentGroupVersion:
+				l.Warnf(
+					"API version %s is deprecated in v0.3; ok to use in v0.2, use %s instead in v0.3",
+					gvk.GroupVersion().String(),
+					v1alpha2.ContentGroupVersion.String(),
+				)
+				switch gvk.Kind {
+				// content.hauler.cattle.io/v1alpha1 Files
+				case v1alpha1.FilesContentKind:
+					var cfg v1alpha1.Files
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					for _, f := range cfg.Spec.Files {
+						err := storeFile(ctx, s, f)
+						if err != nil {
+							return err
+						}
+					}
+				// content.hauler.cattle.io/v1alpha1 Images
+				case v1alpha1.ImagesContentKind:
+					var cfg v1alpha1.Images
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					for _, i := range cfg.Spec.Images {
+						err := storeImage(ctx, s, i)
+						if err != nil {
+							return err
+						}
+					}
+				// content.hauler.cattle.io/v1alpha1 Charts
+				case v1alpha1.ChartsContentKind:
+					var cfg v1alpha1.Charts
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					for _, ch := range cfg.Spec.Charts {
+						err := storeChart(ctx, s, ch)
+						if err != nil {
+							return err
+						}
+					}
+				// collection.hauler.cattle.io/v1alpha1 unknown
+				default:
+					return fmt.Errorf("unsupported Kind %s for %s", obj.GroupVersionKind().Kind, obj.GroupVersionKind().GroupVersion().String())
+				}
+			// collection.hauler.cattle.io/v1alpha1
+			case gvk.GroupVersion() == v1alpha1.CollectionGroupVersion:
+				l.Warnf(
+					"API version %s is deprecated in v0.3; ok to use in v0.2, use %s instead in v0.3",
+					gvk.GroupVersion().String(),
+					v1alpha2.CollectionGroupVersion.String(),
+				)
+				switch gvk.Kind {
+				// collection.hauler.cattle.io/v1alpha1 K3s
+				case v1alpha1.K3sCollectionKind:
+					var cfg v1alpha1.K3s
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
+						return err
+					}
+					k, err := k3s.NewK3s(cfg.Spec.Version)
 					if err != nil {
 						return err
 					}
-				}
-
-			case v1alpha1.ImagesContentKind:
-				var cfg v1alpha1.Images
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				for _, i := range cfg.Spec.Images {
-					err := storeImage(ctx, s, i)
-					if err != nil {
+					if _, err := s.AddCollection(ctx, k); err != nil {
 						return err
 					}
-				}
-
-			case v1alpha1.ChartsContentKind:
-				var cfg v1alpha1.Charts
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				for _, ch := range cfg.Spec.Charts {
-					err := storeChart(ctx, s, ch)
-					if err != nil {
+				// collection.hauler.cattle.io/v1alpha1 ThickCharts
+				case v1alpha1.ChartsCollectionKind:
+					var cfg v1alpha1.ThickCharts
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
 						return err
 					}
-				}
-
-			case v1alpha1.K3sCollectionKind:
-				var cfg v1alpha1.K3s
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				k, err := k3s.NewK3s(cfg.Spec.Version)
-				if err != nil {
-					return err
-				}
-
-				if _, err := s.AddCollection(ctx, k); err != nil {
-					return err
-				}
-
-			case v1alpha1.ChartsCollectionKind:
-				var cfg v1alpha1.ThickCharts
-				if err := yaml.Unmarshal(doc, &cfg); err != nil {
-					return err
-				}
-
-				for _, cfg := range cfg.Spec.Charts {
-					tc, err := chart.NewChart(cfg.Name, cfg.RepoURL, cfg.Version)
-					if err != nil {
+					for _, cfg := range cfg.Spec.Charts {
+						tc, err := tchart.NewChart(cfg.Name, cfg.RepoURL, cfg.Version)
+						if err != nil {
+							return err
+						}
+						if _, err := s.AddCollection(ctx, tc); err != nil {
+							return err
+						}
+					}
+				// collection.hauler.cattle.io/v1alpha1 ImageTxts
+				case v1alpha1.ImageTxtsContentKind:
+					var cfg v1alpha1.ImageTxts
+					if err := yaml.Unmarshal(doc, &cfg); err != nil {
 						return err
 					}
 
-					if _, err := s.AddCollection(ctx, tc); err != nil {
-						return err
-					}
-				}
+					for _, cfgIt := range cfg.Spec.ImageTxts {
+						it, err := imagetxt.New(cfgIt.Ref,
+							imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
+							imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
+						)
+						if err != nil {
+							return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
+						}
 
+						if _, err := s.AddCollection(ctx, it); err != nil {
+							return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
+						}
+					}
+				// collection.hauler.cattle.io/v1alpha1 unknown
+				default:
+					return fmt.Errorf("unsupported Kind %s for %s", gvk.Kind, gvk.GroupVersion().String())
+				}
+			// content.hauler.cattle.io/v1alpha2 + collection.hauler.cattle.io/v1alpha2
+			case gvk.GroupVersion() == v1alpha2.ContentGroupVersion || gvk.GroupVersion() == v1alpha2.CollectionGroupVersion:
+				return fmt.Errorf("API group + version %s not yet supported", gvk.GroupVersion().String())
+			// unknown
 			default:
 				return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
 			}

cmd/hauler/cli/version.go

@@ -0,0 +1,37 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/rancherfederal/hauler/pkg/version"
+)
+
+func addVersion(parent *cobra.Command) {
+	var json bool
+
+	cmd := &cobra.Command{
+		Use:     "version",
+		Short:   "Print current hauler version",
+		Long:    "Print current hauler version",
+		Aliases: []string{"v"},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			v := version.GetVersionInfo()
+			response := v.String()
+			if json {
+				data, err := v.JSONString()
+				if err != nil {
+					return err
+				}
+				response = data
+			}
+			fmt.Print(response)
+			return nil
+		},
+	}
+
+	cmd.Flags().BoolVar(&json, "json", false, "toggle output in JSON")
+
+	parent.AddCommand(cmd)
+}

docs/walkthrough.md

@@ -0,0 +1,177 @@
+# Walkthrough
+
+## Installation
+
+The latest version of `hauler` is available as statically compiled binaries for most combinations of operating systems and architectures on the GitHub [releases](https://github.com/rancherfederal/hauler/releases) page.
+
+## Quickstart
+
+The tl;dr for how to use `hauler` to fetch, transport, and distribute `content`:
+
+```bash
+# fetch some content
+hauler store add file "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+hauler store add chart longhorn --repo "https://charts.longhorn.io"
+hauler store add image "rancher/cowsay"
+
+# transport the content
+hauler store save
+
+# <-airgap the haul.tar.zst file generated->
+
+# load the content
+hauler store load
+
+# serve the content
+hauler store serve
+```
+
+While the example above fits into a quickstart, it falls short of demonstrating all the capabilities `hauler` has to offer, including taking advantage of its fully declarative nature.  Keep reading the [Guided Examples](#Guided-Examples) below for a more thorough walkthrough of `haulers` full capabilities.
+
+## Guided Examples
+
+Since `hauler`'s primary objective is to simplify the content collection/distribution airgap process, a lot of the design revolves around the typical airgap workflow:
+
+```bash
+fetch -> save - | <airgap> | -> validate/load -> distribute
+```
+
+This is accomplished as follows:
+
+```bash
+# fetch content
+hauler store add ...
+
+# compress and archive content
+hauler store save
+
+# <airgap>
+
+# validate/load content
+hauler store load ...
+
+# distribute content
+hauler store serve
+```
+
+At this point you're probably wondering: what is `content`? In `hauler` land, there are a few important terms given to important resources:
+
+* `artifact`: anything that can be represented as an [`oci artifact`](https://github.com/opencontainers/artifacts)
+* `content`: built in "primitive" types of `artifacts` that `hauler` understands
+
+### Built in content
+
+As of today, `hauler` understands three types of `content`, one with a strong legacy of community support and consensus ([`image-spec`]()), one with a finalized spec and experimental support ([`chart-spec`]()), and one generic type created just for `hauler`.  These `content` types are outlined below:
+
+__`files`__:
+
+Generic content that can be represented as a file, either sourced locally or remotely.
+
+```bash
+# local file
+hauler store add file path/to/local/file.txt
+
+# remote file
+hauler store add file https://get.k3s.io
+```
+
+__`images`__:
+
+Any OCI compatible image can be fetched remotely.
+
+```bash
+# "shorthand"  image references
+hauler store add image rancher/k3s:v1.22.2-k3s1
+
+# fully qualified image references
+hauler store add image ghcr.io/fluxcd/flux-cli@sha256:02aa820c3a9c57d67208afcfc4bce9661658c17d15940aea369da259d2b976dd
+```
+
+__`charts`__:
+
+Helm charts represented as OCI content.
+
+```bash
+# add a helm chart (defaults to latest version)
+hauler store add chart loki --repo "https://grafana.github.io/helm-charts"
+
+# add a specific version of a helm chart
+hauler store add chart loki --repo "https://grafana.github.io/helm-charts" --version 2.8.1
+
+# install directly from the oci content
+HELM_EXPERIMENTAL_OCI=1 helm install loki oci://localhost:3000/library/loki --version 2.8.1
+```
+
+> Note: `hauler` supports the currently experimental format of helm as OCI content, but can also be represented as the usual tarball if necessary
+
+### Content API
+
+While imperatively adding `content` to `hauler` is a simple way to get started, the recommended long term approach is to use the provided api that each `content` has, in conjunction with the `sync` command.
+
+```bash
+# create a haul from declaratively defined content
+hauler store sync -f testdata/contents.yaml
+```
+
+> For a commented view of the `contents` api, take a look at the `testdata` folder in the root of the project.
+
+The API for each type of built-in `content` allows you to easily and declaratively define all the `content` that exist within a `haul`, and ensures a more gitops compatible workflow for managing the lifecycle of your `hauls`.
+
+### Collections
+
+Earlier we referred to `content` as "primitives".  While the quotes justify the loose definition of that term, we call it that because they can be used to build groups of `content`, which we call `collections`.
+
+`collections` are groups of 1 or more `contents` that collectively represent something desirable.  Just like `content`, there are a handful that are built in to `hauler`.
+
+Since `collections` usually contain more purposefully crafted `contents`, we restrict their use to the declarative commands (`sync`):
+
+```bash
+# sync a collection
+hauler store sync -f my-collection.yaml
+
+# sync sets of content/collection
+hauler store sync -f collection.yaml -f content.yaml
+```
+
+__`thickcharts`__:
+
+Thick Charts represent the combination of `charts` and `images`.  When storing a thick chart, the chart _and_ the charts dependent images will be fetched and stored by `hauler`.  
+
+```yaml
+# thick-chart.yaml
+apiVersion: collection.hauler.cattle.io/v1alpha1
+kind: ThickCharts
+metadata:
+  name: loki
+spec:
+  charts:
+    - name: loki
+      repoURL: https://grafana.github.io/helm-charts
+```
+
+When syncing the collection above, `hauler` will identify the images the chart depends on and store those too
+
+> The method for identifying images is constantly changing, as of today, the chart is rendered and a configurable set of container defining json path's are processed.  The most common paths are recognized by hauler, but this can be configured for the more niche CRDs out there.
+
+__`k3s`__:
+
+Combining `files` and `images`, full clusters can also be captured by `hauler` for further simplifying the already simple nature of `k3s`.
+
+```yaml
+# k3s.yaml
+---
+apiVersion: collection.hauler.cattle.io/v1alpha1
+kind: K3s
+metadata:
+  name: k3s
+spec:
+  version: stable
+```
+
+Using the collection above, the dependent files (`k3s` executable and `https://get.k3s.io` script) will be fetched, as well as all the dependent images.
+
+> We know not everyone uses the get.k3s.io script to provision k3s, in the future this may change, but until then you're welcome to mix and match the `collection` with any of your own additional `content` 
+
+#### User defined `collections`
+
+Although `content` and `collections` can only be used when they are baked in to `hauler`, the goal is to allow these to be securely user-defined, allowing you to define your own desirable `collection` types, and leave the heavy lifting to `hauler`.  Check out our [roadmap](../ROADMAP.md) and [milestones]() for more info on that.

go.mod

@@ -43,10 +43,10 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.7.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/cli v20.10.7+incompatible // indirect
+	github.com/docker/cli v20.10.9+incompatible // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/docker v20.10.7+incompatible // indirect
-	github.com/docker/docker-credential-helpers v0.6.3 // indirect
+	github.com/docker/docker v20.10.9+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
@@ -55,7 +55,7 @@ require (
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
-	github.com/fatih/color v1.7.0 // indirect
+	github.com/fatih/color v1.9.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
@@ -63,6 +63,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -72,7 +73,7 @@ require (
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
@@ -84,20 +85,20 @@ require (
 	github.com/jmoiron/sqlx v1.3.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.11 // indirect
-	github.com/klauspost/compress v1.13.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/pgzip v1.2.4 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.10.0 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mattn/go-colorable v0.0.9 // indirect
-	github.com/mattn/go-isatty v0.0.4 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/mitchellh/copystructure v1.1.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
-	github.com/mitchellh/reflectwalk v1.0.1 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
@@ -115,10 +116,12 @@ require (
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
 	github.com/russross/blackfriday v1.5.2 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/ulikunitz/xz v0.5.7 // indirect
@@ -131,17 +134,17 @@ require (
 	github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 // indirect
 	github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
-	golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 // indirect
-	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
-	google.golang.org/grpc v1.38.0 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492 // indirect
+	google.golang.org/grpc v1.39.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -195,6 +195,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
@@ -319,6 +320,7 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
 github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
+github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -333,18 +335,21 @@ github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3/go.mo
 github.com/distribution/distribution/v3 v3.0.0-20210926092439-1563384b69df h1:zafDqOsnugdrReF9Pe0wybnfFtEIaegSyHNIvnwKPVk=
 github.com/distribution/distribution/v3 v3.0.0-20210926092439-1563384b69df/go.mod h1:ZDZib/BOniVWcXcsy0voU8gR00znhe5VJm47d3H2Y5g=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/cli v20.10.7+incompatible h1:pv/3NqibQKphWZiAskMzdz8w0PRbtTaEB+f6NwdU7Is=
 github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.9+incompatible h1:OJ7YkwQA+k2Oi51lmCojpjiygKpi76P7bg91b2eJxYU=
+github.com/docker/cli v20.10.9+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ=
 github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ=
+github.com/docker/docker v20.10.9+incompatible h1:JlsVnETOjM2RLQa0Cc1XCIspUdXW3Zenq9P54uXBm6k=
+github.com/docker/docker v20.10.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
+github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
+github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
@@ -377,6 +382,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
@@ -388,8 +394,9 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
-github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -478,8 +485,9 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
 github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
-github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobuffalo/logger v1.0.3 h1:YaXOTHNPCvkqqA7w05A4v0k2tCdpr+sgFlgINbQ6gqc=
@@ -599,8 +607,9 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
@@ -709,8 +718,9 @@ github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
-github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs=
 github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
 github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -757,16 +767,23 @@ github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2
 github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
-github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
+github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-oci8 v0.1.1/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
@@ -782,8 +799,9 @@ github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
-github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4=
 github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -797,8 +815,9 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f h1:2+myh5ml7lgEU/51gbeLHfKGNfgEQQIWrlbdaOsidbQ=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
 github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
@@ -955,6 +974,8 @@ github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08/go.mod h1:9qZd/S8DqW
 github.com/rancher/wrangler v0.8.4 h1:aGMqLQHtn9yB8vw584TO8x/tvZMiaY7Yyi37Fp22UFU=
 github.com/rancher/wrangler v0.8.4/go.mod h1:dKEaHNB4izxmPUtpq1Hvr3z3Oh+9k5pCZyFO9sUhlaY=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -974,8 +995,9 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -999,8 +1021,9 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
+github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
@@ -1174,8 +1197,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1273,8 +1297,9 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210913180222-943fd674d43e h1:+b/22bPvDYt4NPDcy4xAGCmON713ONAWFeY3Z7I3tR8=
+golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1313,6 +1338,7 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1344,6 +1370,7 @@ golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1397,12 +1424,14 @@ golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211013075003-97ac67df715c h1:taxlMj0D/1sOAuv/CbSD+MMDof2vbyPTqz5FNYKpXt8=
+golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1412,8 +1441,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1488,6 +1518,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1578,8 +1609,9 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492 h1:7yQQsvnwjfEahbNNEKcBHv3mR+HnB1ctGY/z1JXzx8M=
+google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1605,8 +1637,9 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI=
+google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1619,8 +1652,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

pkg/apis/hauler.cattle.io/v1alpha1/imagetxt.go

@@ -0,0 +1,30 @@
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ImageTxtsContentKind = "ImageTxts"
+)
+
+type ImageTxts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ImageTxtsSpec `json:"spec,omitempty"`
+}
+
+type ImageTxtsSpec struct {
+	ImageTxts []ImageTxt `json:"imageTxts,omitempty"`
+}
+
+type ImageTxt struct {
+	Ref     string          `json:"ref,omitempty"`
+	Sources ImageTxtSources `json:"sources,omitempty"`
+}
+
+type ImageTxtSources struct {
+	Include []string `json:"include,omitempty"`
+	Exclude []string `json:"exclude,omitempty"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/chart.go

@@ -0,0 +1,47 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ChartsContentKind    = "Charts"
+	ChartsCollectionKind = "ThickCharts"
+)
+
+type Charts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ChartSpec `json:"spec,omitempty"`
+}
+
+type ChartSpec struct {
+	Charts []Chart `json:"charts,omitempty"`
+}
+
+type Chart struct {
+	Name    string `json:"name"`
+	RepoURL string `json:"repoURL"`
+	Version string `json:"version"`
+}
+
+type ThickCharts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ChartSpec `json:"spec,omitempty"`
+}
+
+type ThickChartSpec struct {
+	Charts []ThickChart `json:"charts,omitempty"`
+}
+
+type ThickChart struct {
+	Chart       `json:",inline,omitempty"`
+	ExtraImages []ChartImage `json:"extraImages,omitempty"`
+}
+
+type ChartImage struct {
+	Reference string `json:"ref"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/driver.go

@@ -0,0 +1,21 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	DriverContentKind = "Driver"
+)
+
+type Driver struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec DriverSpec `json:"spec"`
+}
+
+type DriverSpec struct {
+	Type    string `json:"type"`
+	Version string `json:"version"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/file.go

@@ -0,0 +1,27 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const FilesContentKind = "Files"
+
+type Files struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec FileSpec `json:"spec,omitempty"`
+}
+
+type FileSpec struct {
+	Files []File `json:"files,omitempty"`
+}
+
+type File struct {
+	// Path is the path to the file contents, can be a local or remote path
+	Path string `json:"path"`
+
+	// Name is an optional field specifying the name of the file. When specified,
+	// it will override any dynamic name discovery from Path
+	Name string `json:"name,omitempty"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/groupversion_info.go

@@ -0,0 +1,19 @@
+package v1alpha2
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+const (
+	Version         = "v1alpha2"
+	ContentGroup    = "content.hauler.cattle.io"
+	CollectionGroup = "collection.hauler.cattle.io"
+)
+
+var (
+	ContentGroupVersion = schema.GroupVersion{Group: ContentGroup, Version: Version}
+	SchemeBuilder       = &scheme.Builder{GroupVersion: ContentGroupVersion}
+
+	CollectionGroupVersion = schema.GroupVersion{Group: CollectionGroup, Version: Version}
+)

pkg/apis/hauler.cattle.io/v1alpha2/image.go

@@ -0,0 +1,22 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const ImagesContentKind = "Images"
+
+type Images struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ImageSpec `json:"spec,omitempty"`
+}
+
+type ImageSpec struct {
+	Images []Image `json:"images,omitempty"`
+}
+
+type Image struct {
+	Name string `json:"name"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/imagetxt.go

@@ -0,0 +1,30 @@
+package v1alpha2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ImageTxtsContentKind = "ImageTxts"
+)
+
+type ImageTxts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ImageTxtsSpec `json:"spec,omitempty"`
+}
+
+type ImageTxtsSpec struct {
+	ImageTxts []ImageTxt `json:"imageTxts,omitempty"`
+}
+
+type ImageTxt struct {
+	Name    string          `json:"name,omitempty"`
+	Sources ImageTxtSources `json:"sources,omitempty"`
+}
+
+type ImageTxtSources struct {
+	Include []string `json:"include,omitempty"`
+	Exclude []string `json:"exclude,omitempty"`
+}

pkg/apis/hauler.cattle.io/v1alpha2/k3s.go

@@ -0,0 +1,17 @@
+package v1alpha2
+
+import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+const K3sCollectionKind = "K3s"
+
+type K3s struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec K3sSpec `json:"spec,omitempty"`
+}
+
+type K3sSpec struct {
+	Version string `json:"version"`
+	Arch    string `json:"arch"`
+}

pkg/artifact/local/layer.go

@@ -9,8 +9,6 @@ import (
 	"github.com/rancherfederal/hauler/pkg/artifact/types"
 )
 
-type Opener func() (io.ReadCloser, error)
-
 func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 	var err error
 

pkg/artifact/local/opener.go

@@ -0,0 +1,25 @@
+package local
+
+import (
+	"io"
+	"net/http"
+	"os"
+)
+
+type Opener func() (io.ReadCloser, error)
+
+func LocalOpener(path string) Opener {
+	return func() (io.ReadCloser, error) {
+		return os.Open(path)
+	}
+}
+
+func RemoteOpener(url string) Opener {
+	return func() (io.ReadCloser, error) {
+		resp, err := http.Get(url)
+		if err != nil {
+			return nil, err
+		}
+		return resp.Body, nil
+	}
+}

pkg/artifact/types/types.go

@@ -3,14 +3,10 @@ package types
 const (
 	OCIManifestSchema1    = "application/vnd.oci.image.manifest.v1+json"
 	DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json"
-	DockerConfigJSON      = "application/vnd.docker.container.image.v1+json"
-	UnknownManifest       = "application/vnd.hauler.cattle.io.unknown.v1+json"
 
-	UnknownLayer       = "application/vnd.content.hauler.unknown.layer"
-	FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1"
-	FileMediaType      = "application/vnd.content.hauler.file.config.v1+json"
+	DockerConfigJSON = "application/vnd.docker.container.image.v1+json"
 
-	// ConfigMediaType is the reserved media type for the Helm chart manifest config
+	// ChartConfigMediaType is the reserved media type for the Helm chart manifest config
 	ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
 
 	// ChartLayerMediaType is the reserved media type for Helm chart package content
@@ -19,6 +15,21 @@ const (
 	// ProvLayerMediaType is the reserved media type for Helm chart provenance files
 	ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov"
 
+	// FileLayerMediaType is the reserved media type for File content layers
+	FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1"
+
+	// FileConfigMediaType is the reserved media type for File config
+	FileConfigMediaType = "application/vnd.content.hauler.file.config.v1+json"
+
+	// WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers
+	WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm"
+
+	// WasmConfigMediaType is the reserved media type for WASM configs
+	WasmConfigMediaType = "application/vnd.wasm.config.v1+json"
+
+	UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json"
+	UnknownLayer    = "application/vnd.content.hauler.unknown.layer"
+
 	OCIVendorPrefix    = "vnd.oci"
 	DockerVendorPrefix = "vnd.docker"
 	HaulerVendorPrefix = "vnd.hauler"

pkg/collection/chart/chart.go

@@ -12,7 +12,10 @@ var _ artifact.Collection = (*tchart)(nil)
 
 // tchart is a thick chart that includes all the dependent images as well as the chart itself
 type tchart struct {
-	chart *chart.Chart
+	name    string
+	repo    string
+	version string
+	chart   *chart.Chart
 
 	computed bool
 	contents map[gname.Reference]artifact.OCI
@@ -25,6 +28,9 @@ func NewChart(name, repo, version string) (artifact.Collection, error) {
 	}
 
 	return &tchart{
+		name:     name,
+		repo:     repo,
+		version:  version,
 		chart:    o,
 		contents: make(map[gname.Reference]artifact.OCI),
 	}, nil
@@ -46,10 +52,34 @@ func (c *tchart) compute() error {
 		return err
 	}
 
+	if err := c.chartContents(); err != nil {
+		return err
+	}
+
 	c.computed = true
 	return nil
 }
 
+func (c *tchart) chartContents() error {
+	oci, err := chart.NewChart(c.name, c.repo, c.version)
+	if err != nil {
+		return err
+	}
+
+	tag := c.version
+	if tag == "" {
+		tag = gname.DefaultTag
+	}
+
+	ref, err := gname.ParseReference(c.name, gname.WithDefaultRegistry(""), gname.WithDefaultTag(tag))
+	if err != nil {
+		return err
+	}
+
+	c.contents[ref] = oci
+	return nil
+}
+
 func (c *tchart) dependentImages() error {
 	ch, err := c.chart.Load()
 	if err != nil {

pkg/collection/imagetxt/imagetxt.go

@@ -0,0 +1,240 @@
+package imagetxt
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/rancherfederal/hauler/pkg/artifact"
+	"github.com/rancherfederal/hauler/pkg/artifact/local"
+	"github.com/rancherfederal/hauler/pkg/content/image"
+	"github.com/rancherfederal/hauler/pkg/log"
+
+	"github.com/google/go-containerregistry/pkg/name"
+)
+
+type ImageTxt struct {
+	Ref            string
+	IncludeSources map[string]bool
+	ExcludeSources map[string]bool
+
+	lock     *sync.Mutex
+	getter   local.Opener
+	computed bool
+	contents map[name.Reference]artifact.OCI
+}
+
+var _ artifact.Collection = (*ImageTxt)(nil)
+
+type Option interface {
+	Apply(*ImageTxt) error
+}
+
+type withIncludeSources []string
+
+func (o withIncludeSources) Apply(it *ImageTxt) error {
+	if it.IncludeSources == nil {
+		it.IncludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.IncludeSources[s] = true
+	}
+	return nil
+}
+
+func WithIncludeSources(include ...string) Option {
+	return withIncludeSources(include)
+}
+
+type withExcludeSources []string
+
+func (o withExcludeSources) Apply(it *ImageTxt) error {
+	if it.ExcludeSources == nil {
+		it.ExcludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.ExcludeSources[s] = true
+	}
+	return nil
+}
+
+func WithExcludeSources(exclude ...string) Option {
+	return withExcludeSources(exclude)
+}
+
+func New(ref string, opts ...Option) (*ImageTxt, error) {
+	it := &ImageTxt{
+		Ref: ref,
+
+		lock: &sync.Mutex{},
+	}
+
+	if strings.HasPrefix(ref, "http") || strings.HasPrefix(ref, "https") {
+		it.getter = local.RemoteOpener(ref)
+	} else {
+		it.getter = local.LocalOpener(ref)
+	}
+
+	for i, o := range opts {
+		if err := o.Apply(it); err != nil {
+			return nil, fmt.Errorf("invalid option %d: %v", i, err)
+		}
+	}
+
+	return it, nil
+}
+
+func (it *ImageTxt) Contents() (map[name.Reference]artifact.OCI, error) {
+	it.lock.Lock()
+	defer it.lock.Unlock()
+	if !it.computed {
+		if err := it.compute(); err != nil {
+			return nil, fmt.Errorf("compute OCI layout: %v", err)
+		}
+		it.computed = true
+	}
+	return it.contents, nil
+}
+
+func (it *ImageTxt) compute() error {
+	// TODO - pass in logger from context
+	l := log.NewLogger(os.Stdout)
+
+	it.contents = make(map[name.Reference]artifact.OCI)
+
+	r, err := it.getter()
+	if err != nil {
+		return fmt.Errorf("fetch image.txt ref %s: %v", it.Ref, err)
+	}
+	defer r.Close()
+
+	buf := &bytes.Buffer{}
+	if _, err := io.Copy(buf, r); err != nil {
+		return fmt.Errorf("read image.txt ref %s: %v", it.Ref, err)
+	}
+
+	entries, err := splitImagesTxt(buf)
+	if err != nil {
+		return fmt.Errorf("parse image.txt ref %s: %v", it.Ref, err)
+	}
+
+	foundSources := make(map[string]bool)
+	for _, e := range entries {
+		for s := range e.Sources {
+			foundSources[s] = true
+		}
+	}
+
+	var pullAll bool
+	targetSources := make(map[string]bool)
+
+	if len(foundSources) == 0 || (len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0) {
+		// pull all found images
+		pullAll = true
+
+		if len(foundSources) == 0 {
+			l.Infof("image txt file appears to have no sources; pulling all found images")
+			if len(it.IncludeSources) != 0 || len(it.ExcludeSources) != 0 {
+				l.Warnf("ImageTxt provided include or exclude sources; ignoring")
+			}
+		} else if len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0 {
+			l.Infof("image-sources txt file not filtered; pulling all found images")
+		}
+	} else {
+		// determine sources to pull
+		if len(it.IncludeSources) != 0 && len(it.ExcludeSources) != 0 {
+			l.Warnf("ImageTxt provided include and exclude sources; using only include sources")
+		}
+
+		if len(it.IncludeSources) != 0 {
+			targetSources = it.IncludeSources
+		} else {
+			for s := range foundSources {
+				targetSources[s] = true
+			}
+			for s := range it.ExcludeSources {
+				delete(targetSources, s)
+			}
+		}
+		var targetSourcesArr []string
+		for s := range targetSources {
+			targetSourcesArr = append(targetSourcesArr, s)
+		}
+		l.Infof("pulling images covering sources %s", strings.Join(targetSourcesArr, ", "))
+	}
+
+	for _, e := range entries {
+		var matchesSourceFilter bool
+		if pullAll {
+			l.Infof("pulling image %s", e.Reference)
+		} else {
+			for s := range e.Sources {
+				if targetSources[s] {
+					matchesSourceFilter = true
+					l.Infof("pulling image %s (matched source %s)", e.Reference, s)
+					break
+				}
+			}
+		}
+
+		if pullAll || matchesSourceFilter {
+			curImage, err := image.NewImage(e.Reference.String())
+			if err != nil {
+				return fmt.Errorf("pull image %s: %v", e.Reference, err)
+			}
+			it.contents[e.Reference] = curImage
+		}
+	}
+
+	return nil
+}
+
+type imageTxtEntry struct {
+	Reference name.Reference
+	Sources   map[string]bool
+}
+
+func splitImagesTxt(r io.Reader) ([]imageTxtEntry, error) {
+	var entries []imageTxtEntry
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		curEntry := imageTxtEntry{
+			Sources: make(map[string]bool),
+		}
+
+		lineContent := scanner.Text()
+		if lineContent == "" || strings.HasPrefix(lineContent, "#") {
+			// skip past empty and commented lines
+			continue
+		}
+		splitContent := strings.Split(lineContent, " ")
+		if len(splitContent) > 2 {
+			return nil, fmt.Errorf(
+				"invalid image.txt format: must contain only an image reference and sources separated by space; invalid line: %q",
+				lineContent)
+		}
+
+		curRef, err := name.ParseReference(splitContent[0])
+		if err != nil {
+			return nil, fmt.Errorf("invalid reference %s: %v", splitContent[0], err)
+		}
+		curEntry.Reference = curRef
+
+		if len(splitContent) == 2 {
+			for _, source := range strings.Split(splitContent[1], ",") {
+				curEntry.Sources[source] = true
+			}
+		}
+
+		entries = append(entries, curEntry)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("scan contents: %v", err)
+	}
+
+	return entries, nil
+}

pkg/collection/imagetxt/imagetxt_test.go

@@ -0,0 +1,216 @@
+package imagetxt
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/name"
+
+	artifacts "github.com/rancherfederal/hauler/pkg/artifact"
+	"github.com/rancherfederal/hauler/pkg/content/image"
+)
+
+var (
+	ErrInvalidRef     = errors.New("invalid reference")
+	ErrRefNotFound    = errors.New("ref not found")
+	ErrRefNotImage    = errors.New("ref is not image")
+	ErrExtraRefsFound = errors.New("extra refs found in contents")
+)
+
+var (
+	testServer *httptest.Server
+)
+
+func TestMain(m *testing.M) {
+	setup()
+	code := m.Run()
+	teardown()
+	os.Exit(code)
+}
+
+func setup() {
+	dir := http.Dir("./testdata/http/")
+	h := http.FileServer(dir)
+	testServer = httptest.NewServer(h)
+}
+
+func teardown() {
+	if testServer != nil {
+		testServer.Close()
+	}
+}
+
+type failKind string
+
+const (
+	failKindNew      = failKind("New")
+	failKindContents = failKind("Contents")
+)
+
+func checkError(checkedFailKind failKind) func(*testing.T, error, bool, failKind) {
+	return func(cet *testing.T, err error, testShouldFail bool, testFailKind failKind) {
+		if err != nil {
+			// if error should not have happened at all OR error should have happened
+			// at a different point, test failed
+			if !testShouldFail || testFailKind != checkedFailKind {
+				cet.Fatalf("unexpected error at %s: %v", checkedFailKind, err)
+			}
+			// test should fail at this point, test passed
+			return
+		}
+		// if no error occurred but error should have happened at this point, test
+		// failed
+		if testShouldFail && testFailKind == checkedFailKind {
+			cet.Fatalf("unexpected nil error at %s", checkedFailKind)
+		}
+	}
+}
+
+func TestImageTxtCollection(t *testing.T) {
+	type testEntry struct {
+		Name           string
+		Ref            string
+		IncludeSources []string
+		ExcludeSources []string
+		ExpectedImages []string
+		ShouldFail     bool
+		FailKind       failKind
+	}
+	tt := []testEntry{
+		{
+			Name: "http ref basic",
+			Ref:  fmt.Sprintf("%s/images-http.txt", testServer.URL),
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format pull all",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format include sources A",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			IncludeSources: []string{
+				"core", "rke",
+			},
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+			},
+		},
+		{
+			Name: "http ref sources format include sources B",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			IncludeSources: []string{
+				"nginx", "rancher", "cert-manager",
+			},
+			ExpectedImages: []string{
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format exclude sources A",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExcludeSources: []string{
+				"cert-manager",
+			},
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+			},
+		},
+		{
+			Name: "http ref sources format exclude sources B",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExcludeSources: []string{
+				"core",
+			},
+			ExpectedImages: []string{
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "local file ref",
+			Ref:  "./testdata/images-file.txt",
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+	}
+
+	checkErrorNew := checkError(failKindNew)
+	checkErrorContents := checkError(failKindContents)
+
+	for _, curTest := range tt {
+		t.Run(curTest.Name, func(innerT *testing.T) {
+			curImageTxt, err := New(curTest.Ref,
+				WithIncludeSources(curTest.IncludeSources...),
+				WithExcludeSources(curTest.ExcludeSources...),
+			)
+			checkErrorNew(innerT, err, curTest.ShouldFail, curTest.FailKind)
+
+			ociContents, err := curImageTxt.Contents()
+			checkErrorContents(innerT, err, curTest.ShouldFail, curTest.FailKind)
+
+			if err := checkImages(ociContents, curTest.ExpectedImages); err != nil {
+				innerT.Fatal(err)
+			}
+		})
+	}
+}
+
+func checkImages(content map[name.Reference]artifacts.OCI, refs []string) error {
+	contentCopy := make(map[name.Reference]artifacts.OCI, len(content))
+	for k, v := range content {
+		contentCopy[k] = v
+	}
+	for _, ref := range refs {
+		nameRef, err := name.ParseReference(ref)
+		if err != nil {
+			return fmt.Errorf("ref %s: %w", ref, ErrInvalidRef)
+		}
+		target, ok := content[nameRef]
+		if !ok {
+			return fmt.Errorf("ref %s: %w", ref, ErrRefNotFound)
+		}
+		if _, ok := target.(*image.Image); !ok {
+			return fmt.Errorf("got underlying type %T: %w", target, ErrRefNotImage)
+		}
+		delete(contentCopy, nameRef)
+	}
+
+	if len(contentCopy) != 0 {
+		return ErrExtraRefsFound
+	}
+
+	return nil
+}

pkg/collection/imagetxt/testdata/http/images-http.txt

@@ -0,0 +1,5 @@
+busybox
+nginx:1.19
+rancher/hyperkube:v1.21.7-rancher1
+docker.io/rancher/klipper-lb:v0.3.4
+quay.io/jetstack/cert-manager-controller:v1.6.1

pkg/collection/imagetxt/testdata/http/images-src-http.txt

@@ -0,0 +1,5 @@
+busybox core
+nginx:1.19 core,nginx
+rancher/hyperkube:v1.21.7-rancher1 rancher,rke
+docker.io/rancher/klipper-lb:v0.3.4 rancher,k3s
+quay.io/jetstack/cert-manager-controller:v1.6.1 cert-manager

pkg/collection/imagetxt/testdata/images-file.txt

@@ -0,0 +1,5 @@
+busybox
+nginx:1.19
+rancher/hyperkube:v1.21.7-rancher1
+docker.io/rancher/klipper-lb:v0.3.4
+quay.io/jetstack/cert-manager-controller:v1.6.1

pkg/content/content.go

@@ -8,6 +8,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
+	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha2"
 )
 
 func Load(data []byte) (schema.ObjectKind, error) {
@@ -16,8 +17,9 @@ func Load(data []byte) (schema.ObjectKind, error) {
 		return nil, err
 	}
 
-	if tm.GroupVersionKind().GroupVersion() != v1alpha1.ContentGroupVersion && tm.GroupVersionKind().GroupVersion() != v1alpha1.CollectionGroupVersion {
-		return nil, fmt.Errorf("unrecognized content/collection type: %s", tm.GroupVersionKind().String())
+	gv := tm.GroupVersionKind().GroupVersion()
+	if gv != v1alpha1.ContentGroupVersion && gv != v1alpha1.CollectionGroupVersion && gv != v1alpha2.ContentGroupVersion && gv != v1alpha2.CollectionGroupVersion {
+		return nil, fmt.Errorf("unrecognized API type: %s", tm.GroupVersionKind().String())
 	}
 
 	return tm, nil

pkg/content/file/config.go

@@ -30,7 +30,7 @@ func (c config) Descriptor() (gv1.Descriptor, error) {
 	}
 
 	return gv1.Descriptor{
-		MediaType:   types.FileMediaType,
+		MediaType:   types.FileConfigMediaType,
 		Size:        c.size,
 		Digest:      c.hash,
 		URLs:        c.URLs,
@@ -47,7 +47,7 @@ func (c config) Digest() (gv1.Hash, error) {
 }
 
 func (c config) MediaType() (gtypes.MediaType, error) {
-	return types.FileMediaType, nil
+	return types.FileConfigMediaType, nil
 }
 
 func (c config) Size() (int64, error) {

pkg/content/image/image.go

@@ -8,9 +8,9 @@ import (
 	"github.com/rancherfederal/hauler/pkg/artifact"
 )
 
-var _ artifact.OCI = (*image)(nil)
+var _ artifact.OCI = (*Image)(nil)
 
-func (i *image) MediaType() string {
+func (i *Image) MediaType() string {
 	mt, err := i.Image.MediaType()
 	if err != nil {
 		return ""
@@ -18,15 +18,15 @@ func (i *image) MediaType() string {
 	return string(mt)
 }
 
-func (i *image) RawConfig() ([]byte, error) {
+func (i *Image) RawConfig() ([]byte, error) {
 	return i.RawConfigFile()
 }
 
-type image struct {
+type Image struct {
 	gv1.Image
 }
 
-func NewImage(ref string) (*image, error) {
+func NewImage(ref string) (*Image, error) {
 	r, err := name.ParseReference(ref)
 	if err != nil {
 		return nil, err
@@ -37,7 +37,7 @@ func NewImage(ref string) (*image, error) {
 		return nil, err
 	}
 
-	return &image{
+	return &Image{
 		Image: img,
 	}, nil
 }

pkg/layout/artifact.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/google/go-containerregistry/pkg/name"
 	gv1 "github.com/google/go-containerregistry/pkg/v1"
@@ -84,6 +85,7 @@ func (l Path) WriteOci(o artifact.OCI, reference name.Reference) (ocispec.Descri
 		Digest:    digest.FromBytes(manifest),
 		Annotations: map[string]string{
 			ocispec.AnnotationRefName: reference.Name(),
+			ocispec.AnnotationTitle:   deregistry(reference).Name(),
 		},
 	}
 
@@ -134,3 +136,11 @@ func (l Path) appendDescriptor(desc ocispec.Descriptor) error {
 
 	return l.AppendDescriptor(gdesc)
 }
+
+// deregistry removes the registry content from a name.Reference
+func deregistry(ref name.Reference) name.Reference {
+	// No error checking b/c at this point we're already assumed to have a valid enough reference
+	dereg := strings.TrimLeft(strings.ReplaceAll(ref.Name(), ref.Context().RegistryStr(), ""), "/")
+	deref, _ := name.ParseReference(dereg, name.WithDefaultRegistry(""))
+	return deref
+}

pkg/layout/store.go

@@ -19,6 +19,12 @@ import (
 	"github.com/rancherfederal/hauler/pkg/artifact/types"
 )
 
+// interface guards
+var (
+	_ content.Provider = (*OCIStore)(nil)
+	_ content.Ingester = (*OCIStore)(nil)
+)
+
 // OCIStore represents a content compatible store adhering by the oci-layout spec
 type OCIStore struct {
 	content.Store
@@ -60,7 +66,6 @@ func Copy(ctx context.Context, s *OCIStore, registry string) error {
 		resolver := docker.NewResolver(docker.ResolverOptions{})
 		_, err = oras.Push(ctx, resolver, rref.Name(), s, m.Layers,
 			oras.WithConfig(m.Config), oras.WithNameValidation(nil), oras.WithManifest(mdesc))
-
 		if err != nil {
 			return err
 		}

pkg/log/log.go

@@ -29,7 +29,8 @@ type Fields map[string]string
 
 // NewLogger returns a new Logger
 func NewLogger(out io.Writer) Logger {
-	l := log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+	output := zerolog.ConsoleWriter{Out: os.Stdout}
+	l := log.Output(output)
 	return &logger{
 		zl: l.With().Timestamp().Logger(),
 	}

pkg/store/add.go

@@ -12,7 +12,6 @@ import (
 	"github.com/rancherfederal/hauler/pkg/artifact"
 	"github.com/rancherfederal/hauler/pkg/cache"
 	"github.com/rancherfederal/hauler/pkg/layout"
-	"github.com/rancherfederal/hauler/pkg/log"
 )
 
 // AddArtifact will add an artifact.OCI to the store
@@ -21,8 +20,6 @@ import (
 //  strict types to define generic content, but provides a processing pipeline suitable for extensibility.  In the
 //  future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout.
 func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference name.Reference) (ocispec.Descriptor, error) {
-	lgr := log.FromContext(ctx)
-
 	if err := s.precheck(); err != nil {
 		return ocispec.Descriptor{}, err
 	}
@@ -37,12 +34,16 @@ func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference nam
 		oci = cached
 	}
 
-	lgr.Debugf("staging %s", reference.Name())
-	if err := stg.add(ctx, oci, reference); err != nil {
+	pdesc, err := stg.add(ctx, oci, reference)
+	if err != nil {
 		return ocispec.Descriptor{}, err
 	}
 
-	return stg.commit(ctx, s)
+	if err := stg.commit(ctx, s); err != nil {
+		return ocispec.Descriptor{}, nil
+	}
+
+	return pdesc, nil
 }
 
 // Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting everything in the underlying store directory
@@ -68,9 +69,6 @@ func (s *Store) Flush(ctx context.Context) error {
 
 // AddCollection .
 func (s *Store) AddCollection(ctx context.Context, coll artifact.Collection) ([]ocispec.Descriptor, error) {
-	lgr := log.FromContext(ctx)
-	_ = lgr
-
 	if err := s.precheck(); err != nil {
 		return nil, err
 	}
@@ -105,25 +103,25 @@ type oci struct {
 	root   string
 }
 
-func (o *oci) add(ctx context.Context, oci artifact.OCI, reference name.Reference) error {
+func (o *oci) add(ctx context.Context, oci artifact.OCI, reference name.Reference) (ocispec.Descriptor, error) {
 	mdesc, err := o.layout.WriteOci(oci, reference)
-	if err != nil {
-		return err
-	}
-	_ = mdesc
-	return nil
-}
-
-func (o *oci) commit(ctx context.Context, s *Store) (ocispec.Descriptor, error) {
-	ts, err := layout.NewOCIStore(o.root)
 	if err != nil {
 		return ocispec.Descriptor{}, err
 	}
+	return mdesc, err
+}
 
-	err = layout.Copy(ctx, ts, s.Registry())
-
+func (o *oci) commit(ctx context.Context, s *Store) error {
 	defer o.close()
-	return ocispec.Descriptor{}, err
+	ts, err := layout.NewOCIStore(o.root)
+	if err != nil {
+		return err
+	}
+
+	if err = layout.Copy(ctx, ts, s.Registry()); err != nil {
+		return err
+	}
+	return err
 }
 
 func (o *oci) close() error {

pkg/store/store.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"regexp"
-	"strconv"
 	"time"
 
 	"github.com/distribution/distribution/v3/configuration"
@@ -93,7 +92,9 @@ func (s *Store) List(ctx context.Context) ([]string, error) {
 	for {
 		chunk := make([]string, 20) // randomly chosen number...
 		nf, err := reg.Repositories(ctx, chunk, last)
-		last = strconv.Itoa(nf)
+		if nf > 0 {
+			last = chunk[nf-1]
+		}
 
 		for _, e := range chunk {
 			if e == "" {
@@ -119,10 +120,9 @@ func (s *Store) List(ctx context.Context) ([]string, error) {
 		}
 
 		tsvc := repo.Tags(ctx)
-
 		ts, err := tsvc.All(ctx)
 		if err != nil {
-			return nil, err
+			continue
 		}
 
 		for _, t := range ts {

pkg/version/version.go

@@ -0,0 +1,61 @@
+package version
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"runtime"
+	"strings"
+	"text/tabwriter"
+)
+
+var (
+	GitVersion = "devel"
+	commit     = "unknown"
+	buildDate  = "unknown"
+)
+
+type Info struct {
+	GitVersion string
+	GitCommit  string
+	BuildDate  string
+
+	GoVersion string
+	Compiler  string
+	Platform  string
+}
+
+func GetVersionInfo() Info {
+	return Info{
+		GitVersion: GitVersion,
+		GitCommit:  commit,
+		BuildDate:  buildDate,
+
+		GoVersion: runtime.Version(),
+		Compiler:  runtime.Compiler,
+		Platform:  path.Join(runtime.GOOS, runtime.GOARCH),
+	}
+}
+
+func (i Info) String() string {
+	b := strings.Builder{}
+	w := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0)
+
+	fmt.Fprintf(w, "GitVersion:\t%s\n", i.GitVersion)
+	fmt.Fprintf(w, "GitCommit:\t%s\n", i.GitCommit)
+	fmt.Fprintf(w, "BuildDate:\t%s\n", i.BuildDate)
+	fmt.Fprintf(w, "GoVersion:\t%s\n", i.GoVersion)
+	fmt.Fprintf(w, "Compiler:\t%s\n", i.Compiler)
+	fmt.Fprintf(w, "Platform:\t%s\n", i.Platform)
+
+	w.Flush()
+	return b.String()
+}
+
+func (i Info) JSONString() (string, error) {
+	b, err := json.MarshalIndent(i, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

scripts/save-docker-image-tar.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-IMAGE_NAME="$1"
-SAVE_DIR="$2"
-
-if [ -z "${IMAGE_NAME}" ]; then
-  echo "[Usage] ./save-docker-image.sh <image_name>"
-  exit 1
-fi
-
-if [ -z "$2" ]; then
-  SAVE_DIR="."
-
-fi
-
-echo "Creating ${IMAGE_NAME} backup..."
-#docker save ${IMAGE_NAME} | gzip --stdout > ${SAVE_DIR}/${IMAGE_NAME}.tgz
-docker save ${IMAGE_NAME} > ${IMAGE_NAME}.tar

scripts/save-docker-image.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-IMAGE_NAME="$1"
-SAVE_DIR="$2"
-
-if [ -z "${IMAGE_NAME}" ]; then
-  echo "[Usage] ./save-docker-image.sh <image_name>"
-  exit 1
-fi
-
-if [ -z "$2" ]; then
-  SAVE_DIR="."
-
-fi
-
-echo "Creating ${IMAGE_NAME} backup..."
-docker save ${IMAGE_NAME} | gzip --stdout > ${SAVE_DIR}/${IMAGE_NAME}.tgz

scripts/save-k3s-artifacts.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-K3S_RELEASE="v1.18.8-rc1%2Bk3s1"
-
-SAVE_DIR="$1"
-
-if [ -z "$1" ]; then
-  SAVE_DIR="."
-fi
-
-# k3s - arm64
-wget -P ${SAVE_DIR} https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s-arm64
-
-# k3s - amd64
-wget -P ${SAVE_DIR} https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s
-
-# images - amd64
-wget -P ${SAVE_DIR} https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s-airgap-images-amd64.tar
-
-# images - arm64
-wget -P ${SAVE_DIR} https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s-airgap-images-arm64.tar
-
-# images.txt
-wget -P ${SAVE_DIR} https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s-images.txt
-

testdata/contents.yaml

@@ -4,7 +4,7 @@ metadata:
   name: myfile
 spec:
   files:
-    # hauler can save/redistribute files on disk
+    # hauler can save/redistribute files on disk (be careful! paths are relative)
     - ref: testdata/contents.yaml
 
     # TODO: when directories are specified, they will be archived and stored as a file
@@ -26,11 +26,17 @@ metadata:
   name: myimage
 spec:
   images:
-    # images can be referenced by their tag
-    - ref: rancher/k3s:v1.22.2-k3s2
+    # images can be referenced shorthanded without a tag
+    - ref: hello-world
+
+    # or namespaced with a tag
+    - ref: rancher/cowsay:latest
 
     # or by their digest:
-    - ref: registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f
+#    - ref: registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f
+
+    # or fully qualified from any OCI compliant registry registry
+    - ref: ghcr.io/fluxcd/flux-cli:v0.22.0
 
 ---
 apiVersion: content.hauler.cattle.io/v1alpha1