Merge pull request #110 from joshrwolf/override-files

pre 0.3 general bug fixes

.github/workflows/release.yaml

@@ -9,19 +9,17 @@ on:
 jobs:
   goreleaser:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      -
-        name: Set up Go
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.17.x
-      -
-        name: Run GoReleaser
+      - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:
           distribution: goreleaser

.github/workflows/unittest.yaml

@@ -0,0 +1,39 @@
+name: Unit Test
+on:
+  push:
+    paths-ignore:
+      - "**.md"
+      - ".github/**"
+      - "!.github/workflows/unittest.yaml"
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - ".github/**"
+      - "!.github/workflows/unitcoverage.yaml"
+  workflow_dispatch: {}
+jobs:
+  test:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.x
+      - name: Run Unit Tests
+        run: |
+          go test -race -covermode=atomic -coverprofile=coverage.out ./pkg/... ./internal/... ./cmd/...
+      - name: On Failure, Launch Debug Session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 5
+      - name: Upload Results To Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          files: ./coverage.out
+          verbose: true # optional (default = false)

.gitignore

@@ -20,11 +20,11 @@ airgap-scp.sh
 
 # test artifacts
 *.tar*
+*.out
 
 # generated
 dist/
-./bundle/
 tmp/
 bin/
-pkg.yaml
-haul/
+/store/
+/registry/

.goreleaser.yaml

@@ -22,7 +22,7 @@ builds:
       - CGO_ENABLED=0
 
 universal_binaries:
-  - replace: true
+  - replace: false
 
 changelog:
   skip: false

README.md

@@ -1,4 +1,4 @@
-# Hauler: Airgap Assistant
+# Hauler: Airgap Swiss Army Knife
 
 > ⚠️ This project is still in active development and _not_ GA.  While a lot of the core features are ready, we're still adding a _ton_, and we may make breaking api and feature changes version to version.
 

cmd/hauler/cli/cli.go

@@ -1,26 +1,15 @@
 package cli
 
 import (
-	"context"
-	"errors"
-	"os"
-	"path/filepath"
-
 	"github.com/spf13/cobra"
 
-	"github.com/rancherfederal/hauler/pkg/cache"
 	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type rootOpts struct {
 	logLevel string
-	cacheDir string
-	storeDir string
 }
 
-const defaultStoreLocation = "haul"
-
 var ro = &rootOpts{}
 
 func New() *cobra.Command {
@@ -40,74 +29,12 @@ func New() *cobra.Command {
 
 	pf := cmd.PersistentFlags()
 	pf.StringVarP(&ro.logLevel, "log-level", "l", "info", "")
-	pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to $XDG_CACHE_HOME/hauler)")
-	pf.StringVarP(&ro.storeDir, "store", "s", "", "Location to create store at (defaults to $PWD/store)")
 
 	// Add subcommands
 	addDownload(cmd)
 	addStore(cmd)
+	addServe(cmd)
 	addVersion(cmd)
 
 	return cmd
 }
-
-func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
-	l := log.FromContext(ctx)
-	dir := o.storeDir
-
-	if dir == "" {
-		l.Debugf("no store path specified, defaulting to $PWD/store")
-		pwd, err := os.Getwd()
-		if err != nil {
-			return nil, err
-		}
-
-		dir = filepath.Join(pwd, defaultStoreLocation)
-	}
-
-	abs, err := filepath.Abs(dir)
-	if err != nil {
-		return nil, err
-	}
-
-	l.Debugf("using store at %s", abs)
-	if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) {
-		err := os.Mkdir(abs, os.ModePerm)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
-
-	// TODO: Do we want this to be configurable?
-	c, err := o.getCache(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	s := store.NewStore(ctx, abs, store.WithCache(c))
-	return s, nil
-}
-
-func (o *rootOpts) getCache(ctx context.Context) (cache.Cache, error) {
-	dir := o.cacheDir
-
-	if dir == "" {
-		// Default to $XDG_CACHE_HOME
-		cachedir, err := os.UserCacheDir()
-		if err != nil {
-			return nil, err
-		}
-
-		abs, _ := filepath.Abs(filepath.Join(cachedir, "hauler"))
-		if err := os.MkdirAll(abs, os.ModePerm); err != nil {
-			return nil, err
-		}
-
-		dir = abs
-	}
-
-	c := cache.NewFilesystem(dir)
-	return c, nil
-}

cmd/hauler/cli/download.go

@@ -21,13 +21,13 @@ Note that the content type determines it's format on disk.  Hauler's built in co
 	- Chart: as a .tar.gz named after the chart (ex: loki:2.0.2 --> loki-2.0.2.tar.gz)`,
 		Example: `
 # Download a file
-hauler dl my-file.yaml:latest
+hauler dl localhost:5000/my-file.yaml:latest
 
 # Download an image
-hauler dl rancher/k3s:v1.22.2-k3s2
+hauler dl localhost:5000/rancher/k3s:v1.22.2-k3s2
 
 # Download a chart
-hauler dl longhorn:1.2.0`,
+hauler dl localhost:5000/hauler/longhorn:1.2.0`,
 		Aliases: []string{"dl"},
 		Args:    cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, arg []string) error {

cmd/hauler/cli/download/download.go

@@ -3,47 +3,60 @@ package download
 import (
 	"context"
 	"encoding/json"
-	"fmt"
-	"path"
 
-	"github.com/containerd/containerd/remotes/docker"
 	"github.com/google/go-containerregistry/pkg/authn"
-	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
-	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 	"oras.land/oras-go/pkg/content"
 	"oras.land/oras-go/pkg/oras"
 
-	"github.com/rancherfederal/hauler/pkg/artifact/types"
+	"github.com/rancherfederal/ocil/pkg/consts"
+
+	"github.com/rancherfederal/hauler/internal/mapper"
 	"github.com/rancherfederal/hauler/pkg/log"
+	"github.com/rancherfederal/hauler/pkg/reference"
 )
 
 type Opts struct {
 	DestinationDir string
-	OutputFile     string
+
+	Username  string
+	Password  string
+	Insecure  bool
+	PlainHTTP bool
 }
 
 func (o *Opts) AddArgs(cmd *cobra.Command) {
 	f := cmd.Flags()
 
-	f.StringVar(&o.DestinationDir, "dir", "", "Directory to save contents to (defaults to current directory)")
-	f.StringVarP(&o.OutputFile, "output", "o", "", "(Optional) Override name of file to save.")
+	f.StringVarP(&o.DestinationDir, "output", "o", "", "Directory to save contents to (defaults to current directory)")
+	f.StringVarP(&o.Username, "username", "u", "", "Username when copying to an authenticated remote registry")
+	f.StringVarP(&o.Password, "password", "p", "", "Password when copying to an authenticated remote registry")
+	f.BoolVar(&o.Insecure, "insecure", false, "Toggle allowing insecure connections when copying to a remote registry")
+	f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry")
 }
 
-func Cmd(ctx context.Context, o *Opts, reference string) error {
+func Cmd(ctx context.Context, o *Opts, ref string) error {
 	l := log.FromContext(ctx)
 
-	cs := content.NewFileStore(o.DestinationDir)
-	defer cs.Close()
-
-	ref, err := name.ParseReference(reference)
+	ropts := content.RegistryOptions{
+		Username:  o.Username,
+		Password:  o.Password,
+		Insecure:  o.Insecure,
+		PlainHTTP: o.PlainHTTP,
+	}
+	rs, err := content.NewRegistry(ropts)
 	if err != nil {
 		return err
 	}
 
-	desc, err := remote.Get(ref)
+	r, err := reference.Parse(ref)
+	if err != nil {
+		return err
+	}
+
+	desc, err := remote.Get(r, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithContext(ctx))
 	if err != nil {
 		return err
 	}
@@ -58,66 +71,17 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 		return err
 	}
 
-	// TODO: These need to be factored out into each of the contents own logic
-	switch manifest.Config.MediaType {
-	case types.DockerConfigJSON, types.OCIManifestSchema1:
-		l.Debugf("identified [image] (%s) content", manifest.Config.MediaType)
-		img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
-		if err != nil {
-			return err
-		}
-
-		outputFile := o.OutputFile
-		if outputFile == "" {
-			outputFile = fmt.Sprintf("%s:%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier())
-		}
-
-		if err := tarball.WriteToFile(outputFile, ref, img); err != nil {
-			return err
-		}
-
-		l.Infof("downloaded image [%s] to [%s]", ref.Name(), outputFile)
-
-	case types.FileConfigMediaType:
-		l.Debugf("identified [file] (%s) content", manifest.Config.MediaType)
-
-		fs := content.NewFileStore(o.DestinationDir)
-
-		resolver := docker.NewResolver(docker.ResolverOptions{})
-		_, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
-		if err != nil {
-			return err
-		}
-
-		ldescs := len(descs)
-		for i, desc := range descs {
-			// NOTE: This is safe without a map key check b/c we're not allowing unnamed content from oras.Pull
-			l.Infof("downloaded (%d/%d) files to [%s]", i+1, ldescs, desc.Annotations[ocispec.AnnotationTitle])
-		}
-
-	case types.ChartLayerMediaType, types.ChartConfigMediaType:
-		l.Debugf("identified [chart] (%s) content", manifest.Config.MediaType)
-
-		fs := content.NewFileStore(o.DestinationDir)
-
-		resolver := docker.NewResolver(docker.ResolverOptions{})
-		_, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
-		if err != nil {
-			return err
-		}
-
-		cn := path.Base(ref.Name())
-		for _, d := range descs {
-			if n, ok := d.Annotations[ocispec.AnnotationTitle]; ok {
-				cn = n
-			}
-		}
-
-		l.Infof("downloaded chart [%s] to [%s]", ref.String(), cn)
-
-	default:
-		return fmt.Errorf("unrecognized content type: %s", manifest.Config.MediaType)
+	mapperStore, err := mapper.FromManifest(manifest, o.DestinationDir)
+	if err != nil {
+		return err
 	}
 
+	pushedDesc, err := oras.Copy(ctx, rs, r.Name(), mapperStore, "",
+		oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2))
+	if err != nil {
+		return err
+	}
+
+	l.Infof("downloaded [%s] with digest [%s]", pushedDesc.MediaType, pushedDesc.Digest.String())
 	return nil
 }

cmd/hauler/cli/download/download_test.go

@@ -1,38 +0,0 @@
-package download
-
-import (
-	"context"
-	"testing"
-)
-
-func TestCmd(t *testing.T) {
-	ctx := context.Background()
-
-	type args struct {
-		ctx       context.Context
-		o         *Opts
-		reference string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{
-			name: "should work",
-			args: args{
-				ctx:       ctx,
-				o:         &Opts{DestinationDir: ""},
-				reference: "localhost:3000/hauler/file.txt:latest",
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := Cmd(tt.args.ctx, tt.args.o, tt.args.reference); (err != nil) != tt.wantErr {
-				t.Errorf("Cmd() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}

cmd/hauler/cli/serve.go

@@ -0,0 +1,55 @@
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/rancherfederal/hauler/cmd/hauler/cli/serve"
+)
+
+func addServe(parent *cobra.Command) {
+	cmd := &cobra.Command{
+		Use:   "serve",
+		Short: "Run one or more of hauler's embedded servers types",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cmd.Help()
+		},
+	}
+
+	cmd.AddCommand(
+		addServeFiles(),
+		addServeRegistry(),
+	)
+
+	parent.AddCommand(cmd)
+}
+
+func addServeFiles() *cobra.Command {
+	o := &serve.FilesOpts{}
+	cmd := &cobra.Command{
+		Use:   "files",
+		Short: "Start a fileserver",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			return serve.FilesCmd(ctx, o)
+		},
+	}
+	o.AddFlags(cmd)
+
+	return cmd
+}
+
+func addServeRegistry() *cobra.Command {
+	o := &serve.RegistryOpts{}
+
+	cmd := &cobra.Command{
+		Use:   "registry",
+		Short: "Start a registry",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			return serve.RegistryCmd(ctx, o)
+		},
+	}
+	o.AddFlags(cmd)
+
+	return cmd
+}

cmd/hauler/cli/serve/files.go

@@ -0,0 +1,37 @@
+package serve
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"github.com/rancherfederal/hauler/internal/server"
+)
+
+type FilesOpts struct {
+	Root string
+	Port int
+}
+
+func (o *FilesOpts) AddFlags(cmd *cobra.Command) {
+	f := cmd.Flags()
+	f.StringVarP(&o.Root, "root", "r", ".", "Path to root of the directory to serve")
+	f.IntVarP(&o.Port, "port", "p", 8080, "Port to listen on")
+}
+
+func FilesCmd(ctx context.Context, o *FilesOpts) error {
+	cfg := server.FileConfig{
+		Root: o.Root,
+		Port: o.Port,
+	}
+
+	s, err := server.NewFile(ctx, cfg)
+	if err != nil {
+		return err
+	}
+
+	if err := s.ListenAndServe(); err != nil {
+		return err
+	}
+	return nil
+}

cmd/hauler/cli/serve/registry.go

@@ -0,0 +1,80 @@
+package serve
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+
+	"github.com/distribution/distribution/v3/configuration"
+	dcontext "github.com/distribution/distribution/v3/context"
+	"github.com/distribution/distribution/v3/version"
+	"github.com/spf13/cobra"
+
+	"github.com/rancherfederal/hauler/internal/server"
+)
+
+type RegistryOpts struct {
+	Root       string
+	Port       int
+	ConfigFile string
+}
+
+func (o *RegistryOpts) AddFlags(cmd *cobra.Command) {
+	f := cmd.Flags()
+	f.StringVarP(&o.Root, "root", "r", ".", "Path to root of the directory to serve")
+	f.IntVarP(&o.Port, "port", "p", 5000, "Port to listen on")
+	f.StringVarP(&o.ConfigFile, "config", "c", "", "Path to a config file, will override all other configs")
+}
+
+func RegistryCmd(ctx context.Context, o *RegistryOpts) error {
+	ctx = dcontext.WithVersion(ctx, version.Version)
+
+	cfg := o.defaultConfig()
+	if o.ConfigFile != "" {
+		ucfg, err := loadConfig(o.ConfigFile)
+		if err != nil {
+			return err
+		}
+		cfg = ucfg
+	}
+
+	s, err := server.NewRegistry(ctx, cfg)
+	if err != nil {
+		return err
+	}
+
+	if err := s.ListenAndServe(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func loadConfig(filename string) (*configuration.Configuration, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	return configuration.Parse(f)
+}
+
+func (o *RegistryOpts) defaultConfig() *configuration.Configuration {
+	cfg := &configuration.Configuration{
+		Version: "0.1",
+		Storage: configuration.Storage{
+			"cache":      configuration.Parameters{"blobdescriptor": "inmemory"},
+			"filesystem": configuration.Parameters{"rootdirectory": o.Root},
+
+			// TODO: Ensure this is toggleable via cli arg if necessary
+			// "maintenance": configuration.Parameters{"readonly.enabled": false},
+		},
+	}
+	cfg.Log.Level = "info"
+	cfg.HTTP.Addr = fmt.Sprintf(":%d", o.Port)
+	cfg.HTTP.Headers = http.Header{
+		"X-Content-Type-Options": []string{"nosniff"},
+	}
+
+	return cfg
+}

cmd/hauler/cli/store.go

@@ -2,10 +2,13 @@ package cli
 
 import (
 	"github.com/spf13/cobra"
+	"helm.sh/helm/v3/pkg/action"
 
 	"github.com/rancherfederal/hauler/cmd/hauler/cli/store"
 )
 
+var rootStoreOpts = &store.RootOpts{}
+
 func addStore(parent *cobra.Command) {
 	cmd := &cobra.Command{
 		Use:     "store",
@@ -15,6 +18,7 @@ func addStore(parent *cobra.Command) {
 			return cmd.Help()
 		},
 	}
+	rootStoreOpts.AddArgs(cmd)
 
 	cmd.AddCommand(
 		addStoreSync(),
@@ -22,7 +26,7 @@ func addStore(parent *cobra.Command) {
 		addStoreLoad(),
 		addStoreSave(),
 		addStoreServe(),
-		addStoreList(),
+		addStoreInfo(),
 		addStoreCopy(),
 
 		// TODO: Remove this in favor of sync?
@@ -33,7 +37,7 @@ func addStore(parent *cobra.Command) {
 }
 
 func addStoreExtract() *cobra.Command {
-	o := &store.ExtractOpts{}
+	o := &store.ExtractOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:     "extract",
@@ -43,7 +47,7 @@ func addStoreExtract() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
@@ -57,7 +61,7 @@ func addStoreExtract() *cobra.Command {
 }
 
 func addStoreSync() *cobra.Command {
-	o := &store.SyncOpts{}
+	o := &store.SyncOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:   "sync",
@@ -65,7 +69,7 @@ func addStoreSync() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
@@ -79,7 +83,7 @@ func addStoreSync() *cobra.Command {
 }
 
 func addStoreLoad() *cobra.Command {
-	o := &store.LoadOpts{}
+	o := &store.LoadOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:   "load",
@@ -88,12 +92,13 @@ func addStoreLoad() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
+			_ = s
 
-			return store.LoadCmd(ctx, o, s.DataDir, args...)
+			return store.LoadCmd(ctx, o, args...)
 		},
 	}
 	o.AddFlags(cmd)
@@ -102,7 +107,7 @@ func addStoreLoad() *cobra.Command {
 }
 
 func addStoreServe() *cobra.Command {
-	o := &store.ServeOpts{}
+	o := &store.ServeOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:   "serve",
@@ -110,7 +115,7 @@ func addStoreServe() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
@@ -124,7 +129,7 @@ func addStoreServe() *cobra.Command {
 }
 
 func addStoreSave() *cobra.Command {
-	o := &store.SaveOpts{}
+	o := &store.SaveOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:   "save",
@@ -133,12 +138,13 @@ func addStoreSave() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
+			_ = s
 
-			return store.SaveCmd(ctx, o, o.FileName, s.DataDir)
+			return store.SaveCmd(ctx, o, o.FileName)
 		},
 	}
 	o.AddArgs(cmd)
@@ -146,23 +152,23 @@ func addStoreSave() *cobra.Command {
 	return cmd
 }
 
-func addStoreList() *cobra.Command {
-	o := &store.ListOpts{}
+func addStoreInfo() *cobra.Command {
+	o := &store.InfoOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
-		Use:     "list",
-		Short:   "List all content references in a store",
+		Use:     "info",
+		Short:   "Print out information about the store",
 		Args:    cobra.ExactArgs(0),
-		Aliases: []string{"ls"},
+		Aliases: []string{"i", "list", "ls"},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
 
-			return store.ListCmd(ctx, o, s)
+			return store.InfoCmd(ctx, o, s)
 		},
 	}
 	o.AddFlags(cmd)
@@ -171,7 +177,7 @@ func addStoreList() *cobra.Command {
 }
 
 func addStoreCopy() *cobra.Command {
-	o := &store.CopyOpts{}
+	o := &store.CopyOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:   "copy",
@@ -180,7 +186,7 @@ func addStoreCopy() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
@@ -212,7 +218,7 @@ func addStoreAdd() *cobra.Command {
 }
 
 func addStoreAddFile() *cobra.Command {
-	o := &store.AddFileOpts{}
+	o := &store.AddFileOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:   "file",
@@ -221,7 +227,7 @@ func addStoreAddFile() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
@@ -235,7 +241,7 @@ func addStoreAddFile() *cobra.Command {
 }
 
 func addStoreAddImage() *cobra.Command {
-	o := &store.AddImageOpts{}
+	o := &store.AddImageOpts{RootOpts: rootStoreOpts}
 
 	cmd := &cobra.Command{
 		Use:   "image",
@@ -244,7 +250,7 @@ func addStoreAddImage() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}
@@ -258,13 +264,22 @@ func addStoreAddImage() *cobra.Command {
 }
 
 func addStoreAddChart() *cobra.Command {
-	o := &store.AddChartOpts{}
+	o := &store.AddChartOpts{
+		RootOpts:  rootStoreOpts,
+		ChartOpts: &action.ChartPathOptions{},
+	}
 
 	cmd := &cobra.Command{
 		Use:   "chart",
-		Short: "Add a chart to the content store",
+		Short: "Add a local or remote chart to the content store",
 		Example: `
-# add a chart
+# add a local chart
+hauler store add chart path/to/chart/directory
+
+# add a local compressed chart
+hauler store add chart path/to/chart.tar.gz
+
+# add a remote chart
 hauler store add chart longhorn --repo "https://charts.longhorn.io"
 
 # add a specific version of a chart
@@ -274,7 +289,7 @@ hauler store add chart rancher --repo "https://releases.rancher.com/server-chart
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			s, err := ro.getStore(ctx)
+			s, err := o.Store(ctx)
 			if err != nil {
 				return err
 			}

cmd/hauler/cli/store/add.go

@@ -2,21 +2,25 @@ package store
 
 import (
 	"context"
-	"path/filepath"
 
 	"github.com/google/go-containerregistry/pkg/name"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
 	"github.com/spf13/cobra"
+	"helm.sh/helm/v3/pkg/action"
+
+	"github.com/rancherfederal/ocil/pkg/artifacts/file"
+	"github.com/rancherfederal/ocil/pkg/artifacts/image"
+
+	"github.com/rancherfederal/ocil/pkg/store"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
 	"github.com/rancherfederal/hauler/pkg/content/chart"
-	"github.com/rancherfederal/hauler/pkg/content/file"
-	"github.com/rancherfederal/hauler/pkg/content/image"
 	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
+	"github.com/rancherfederal/hauler/pkg/reference"
 )
 
 type AddFileOpts struct {
+	*RootOpts
 	Name string
 }
 
@@ -25,47 +29,38 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) {
 	f.StringVarP(&o.Name, "name", "n", "", "(Optional) Name to assign to file in store")
 }
 
-func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference string) error {
-	s.Open()
-	defer s.Close()
-
+func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Layout, reference string) error {
 	cfg := v1alpha1.File{
-		Ref:  reference,
-		Name: o.Name,
+		Path: reference,
 	}
 
 	return storeFile(ctx, s, cfg)
 }
 
-func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error {
+func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
 	l := log.FromContext(ctx)
 
-	if fi.Name == "" {
-		base := filepath.Base(fi.Ref)
-		fi.Name = filepath.Base(fi.Ref)
-		l.Warnf("no name specified for file reference [%s], using base filepath: [%s]", fi.Ref, base)
+	copts := getter.ClientOptions{
+		NameOverride: fi.Name,
 	}
 
-	oci, err := file.NewFile(fi.Ref, fi.Name)
+	f := file.NewFile(fi.Path, file.WithClient(getter.NewClient(copts)))
+	ref, err := reference.NewTagged(f.Name(fi.Path), reference.DefaultTag)
 	if err != nil {
 		return err
 	}
 
-	ref, err := name.ParseReference(fi.Name, name.WithDefaultRegistry(""))
+	desc, err := s.AddOCI(ctx, f, ref.Name())
 	if err != nil {
 		return err
 	}
 
-	desc, err := s.AddArtifact(ctx, oci, ref)
-	if err != nil {
-		return err
-	}
-
-	l.Infof("file [%s] added at: [%s]", ref.Name(), desc.Annotations[ocispec.AnnotationTitle])
+	l.Infof("added 'file' to store at [%s], with digest [%s]", ref.Name(), desc.Digest.String())
 	return nil
 }
 
 type AddImageOpts struct {
+	*RootOpts
 	Name string
 }
 
@@ -74,98 +69,93 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
 	_ = f
 }
 
-func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference string) error {
-	s.Open()
-	defer s.Close()
-
+func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error {
 	cfg := v1alpha1.Image{
-		Ref: reference,
+		Name: reference,
 	}
 
 	return storeImage(ctx, s, cfg)
 }
 
-func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error {
+func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error {
 	l := log.FromContext(ctx)
 
-	oci, err := image.NewImage(i.Ref)
+	img, err := image.NewImage(i.Name)
 	if err != nil {
 		return err
 	}
 
-	ref, err := name.ParseReference(i.Ref)
+	r, err := name.ParseReference(i.Name)
 	if err != nil {
 		return err
 	}
 
-	desc, err := s.AddArtifact(ctx, oci, ref)
+	desc, err := s.AddOCI(ctx, img, r.Name())
 	if err != nil {
 		return err
 	}
 
-	l.Infof("image [%s] added at: [%s]", ref.Name(), desc.Annotations[ocispec.AnnotationTitle])
+	l.Infof("added 'image' to store at [%s], with digest [%s]", r.Name(), desc.Digest.String())
 	return nil
 }
 
 type AddChartOpts struct {
-	Version string
-	RepoURL string
+	*RootOpts
 
-	// TODO: Support helm auth
-	Username              string
-	Password              string
-	PassCredentialsAll    bool
-	CertFile              string
-	KeyFile               string
-	CaFile                string
-	InsecureSkipTLSverify bool
-	RepositoryConfig      string
-	RepositoryCache       string
+	ChartOpts *action.ChartPathOptions
 }
 
 func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
 
-	f.StringVarP(&o.RepoURL, "repo", "r", "", "Chart repository URL")
-	f.StringVar(&o.Version, "version", "", "(Optional) Version of the chart to download, defaults to latest if not specified")
+	f.StringVar(&o.ChartOpts.RepoURL, "repo", "", "chart repository url where to locate the requested chart")
+	f.StringVar(&o.ChartOpts.Version, "version", "", "specify a version constraint for the chart version to use. This constraint can be a specific tag (e.g. 1.1.1) or it may reference a valid range (e.g. ^2.0.0). If this is not specified, the latest version is used")
+	f.BoolVar(&o.ChartOpts.Verify, "verify", false, "verify the package before using it")
+	f.StringVar(&o.ChartOpts.Username, "username", "", "chart repository username where to locate the requested chart")
+	f.StringVar(&o.ChartOpts.Password, "password", "", "chart repository password where to locate the requested chart")
+	f.StringVar(&o.ChartOpts.CertFile, "cert-file", "", "identify HTTPS client using this SSL certificate file")
+	f.StringVar(&o.ChartOpts.KeyFile, "key-file", "", "identify HTTPS client using this SSL key file")
+	f.BoolVar(&o.ChartOpts.InsecureSkipTLSverify, "insecure-skip-tls-verify", false, "skip tls certificate checks for the chart download")
+	f.StringVar(&o.ChartOpts.CaFile, "ca-file", "", "verify certificates of HTTPS-enabled servers using this CA bundle")
 }
 
-func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName string) error {
-	s.Open()
-	defer s.Close()
-
+func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Layout, chartName string) error {
+	// TODO: Reduce duplicates between api chart and upstream helm opts
 	cfg := v1alpha1.Chart{
 		Name:    chartName,
-		RepoURL: o.RepoURL,
-		Version: o.Version,
+		RepoURL: o.ChartOpts.RepoURL,
+		Version: o.ChartOpts.Version,
 	}
 
-	return storeChart(ctx, s, cfg)
+	return storeChart(ctx, s, cfg, o.ChartOpts)
 }
 
-func storeChart(ctx context.Context, s *store.Store, ch v1alpha1.Chart) error {
+func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart, opts *action.ChartPathOptions) error {
 	l := log.FromContext(ctx)
 
-	oci, err := chart.NewChart(ch.Name, ch.RepoURL, ch.Version)
+	// TODO: This shouldn't be necessary
+	opts.RepoURL = cfg.RepoURL
+	opts.Version = cfg.Version
+
+	chrt, err := chart.NewChart(cfg.Name, opts)
 	if err != nil {
 		return err
 	}
 
-	tag := ch.Version
-	if tag == "" {
-		tag = name.DefaultTag
-	}
-
-	ref, err := name.ParseReference(ch.Name, name.WithDefaultRegistry(""), name.WithDefaultTag(tag))
+	c, err := chrt.Load()
 	if err != nil {
 		return err
 	}
 
-	desc, err := s.AddArtifact(ctx, oci, ref)
+	ref, err := reference.NewTagged(c.Name(), c.Metadata.Version)
+	if err != nil {
+		return err
+	}
+	desc, err := s.AddOCI(ctx, chrt, ref.Name())
 	if err != nil {
 		return err
 	}
 
-	l.Infof("chart [%s] added at: [%s]", ref.Name(), desc.Annotations[ocispec.AnnotationTitle])
+	l.Infof("added 'chart' to store at [%s], with digest [%s]", ref.Name(), desc.Digest.String())
 	return nil
 }

cmd/hauler/cli/store/copy.go

@@ -2,56 +2,85 @@ package store
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
-	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
+	"oras.land/oras-go/pkg/content"
+
+	"github.com/rancherfederal/ocil/pkg/store"
 
 	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
+	"github.com/rancherfederal/hauler/pkg/reference"
 )
 
-type CopyOpts struct{}
+type CopyOpts struct {
+	*RootOpts
+
+	Username  string
+	Password  string
+	Insecure  bool
+	PlainHTTP bool
+}
 
 func (o *CopyOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
-	_ = f
 
-	// TODO: Regex matching
+	f.StringVarP(&o.Username, "username", "u", "", "Username when copying to an authenticated remote registry")
+	f.StringVarP(&o.Password, "password", "p", "", "Password when copying to an authenticated remote registry")
+	f.BoolVar(&o.Insecure, "insecure", false, "Toggle allowing insecure connections when copying to a remote registry")
+	f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry")
 }
 
-func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, registry string) error {
+func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error {
 	l := log.FromContext(ctx)
 
-	s.Open()
-	defer s.Close()
+	var descs []ocispec.Descriptor
+	components := strings.SplitN(targetRef, "://", 2)
+	switch components[0] {
+	case "dir":
+		l.Debugf("identified directory target reference")
+		fs := content.NewFile(components[1])
+		defer fs.Close()
 
-	refs, err := s.List(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, r := range refs {
-		ref, err := name.ParseReference(r, name.WithDefaultRegistry(s.Registry()))
-		if err != nil {
-			return err
-		}
-
-		o, err := remote.Image(ref)
-		if err != nil {
-			return err
-		}
-
-		rref, err := name.ParseReference(r, name.WithDefaultRegistry(registry))
-		if err != nil {
-			return err
-		}
-
-		l.Infof("copying [%s] -> [%s]", ref.Name(), rref.Name())
-		if err := remote.Write(rref, o); err != nil {
-			return err
-		}
+		ds, err := s.CopyAll(ctx, fs, nil)
+		if err != nil {
+			return err
+		}
+		descs = ds
+
+	case "registry":
+		l.Debugf("identified registry target reference")
+		ropts := content.RegistryOptions{
+			Username:  o.Username,
+			Password:  o.Password,
+			Insecure:  o.Insecure,
+			PlainHTTP: o.PlainHTTP,
+		}
+		r, err := content.NewRegistry(ropts)
+		if err != nil {
+			return err
+		}
+
+		mapperFn := func(ref string) (string, error) {
+			r, err := reference.Relocate(ref, components[1])
+			if err != nil {
+				return "", err
+			}
+			return r.Name(), nil
+		}
+
+		ds, err := s.CopyAll(ctx, r, mapperFn)
+		if err != nil {
+			return err
+		}
+		descs = ds
+
+	default:
+		return fmt.Errorf("detecting protocol from [%s]", targetRef)
 	}
 
+	l.Infof("Copied [%d] artifacts to [%s]", len(descs), components[1])
 	return nil
 }

cmd/hauler/cli/store/extract.go

@@ -2,36 +2,76 @@ package store
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 
-	"github.com/rancherfederal/hauler/cmd/hauler/cli/download"
-	"github.com/rancherfederal/hauler/pkg/layout"
-	"github.com/rancherfederal/hauler/pkg/store"
+	"github.com/rancherfederal/ocil/pkg/store"
+
+	"github.com/rancherfederal/hauler/internal/mapper"
+	"github.com/rancherfederal/hauler/pkg/log"
+	"github.com/rancherfederal/hauler/pkg/reference"
 )
 
 type ExtractOpts struct {
+	*RootOpts
 	DestinationDir string
 }
 
 func (o *ExtractOpts) AddArgs(cmd *cobra.Command) {
 	f := cmd.Flags()
 
-	f.StringVar(&o.DestinationDir, "dir", "", "Directory to save contents to (defaults to current directory)")
+	f.StringVarP(&o.DestinationDir, "output", "o", "", "Directory to save contents to (defaults to current directory)")
 }
 
-func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, reference string) error {
-	s.Open()
-	defer s.Close()
+func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string) error {
+	l := log.FromContext(ctx)
 
-	eref, err := layout.RelocateReference(reference, s.Registry())
+	r, err := reference.Parse(ref)
 	if err != nil {
 		return err
 	}
 
-	gopts := &download.Opts{
-		DestinationDir: o.DestinationDir,
+	found := false
+	if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
+		if reference != r.Name() {
+			return nil
+		}
+		found = true
+
+		rc, err := s.Fetch(ctx, desc)
+		if err != nil {
+			return err
+		}
+		defer rc.Close()
+
+		var m ocispec.Manifest
+		if err := json.NewDecoder(rc).Decode(&m); err != nil {
+			return err
+		}
+
+		mapperStore, err := mapper.FromManifest(m, o.DestinationDir)
+		if err != nil {
+			return err
+		}
+
+		pushedDesc, err := s.Copy(ctx, r.Name(), mapperStore, "")
+		if err != nil {
+			return err
+		}
+
+		l.Infof("extracted [%s] from store with digest [%s]", pushedDesc.MediaType, pushedDesc.Digest.String())
+
+		return nil
+	}); err != nil {
+		return err
 	}
 
-	return download.Cmd(ctx, gopts, eref.Name())
+	if !found {
+		return fmt.Errorf("reference [%s] not found in store (hint: use `hauler store info` to list store contents)", ref)
+	}
+
+	return nil
 }

cmd/hauler/cli/store/flags.go

@@ -0,0 +1,84 @@
+package store
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/rancherfederal/ocil/pkg/layer"
+	"github.com/rancherfederal/ocil/pkg/store"
+	"github.com/spf13/cobra"
+
+	"github.com/rancherfederal/hauler/pkg/log"
+)
+
+const (
+	DefaultStoreName = "store"
+	DefaultCacheDir  = "hauler"
+)
+
+type RootOpts struct {
+	StoreDir string
+	CacheDir string
+}
+
+func (o *RootOpts) AddArgs(cmd *cobra.Command) {
+	pf := cmd.PersistentFlags()
+	pf.StringVar(&o.CacheDir, "cache", "", "Location of where to store cache data (defaults to $XDG_CACHE_DIR/hauler)")
+	pf.StringVarP(&o.StoreDir, "store", "s", DefaultStoreName, "Location to create store at")
+}
+
+func (o *RootOpts) Store(ctx context.Context) (*store.Layout, error) {
+	l := log.FromContext(ctx)
+	dir := o.StoreDir
+
+	abs, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	l.Debugf("using store at %s", abs)
+	if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) {
+		err := os.Mkdir(abs, os.ModePerm)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	// TODO: Do we want this to be configurable?
+	c, err := o.Cache(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	s, err := store.NewLayout(abs, store.WithCache(c))
+	if err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func (o *RootOpts) Cache(ctx context.Context) (layer.Cache, error) {
+	dir := o.CacheDir
+
+	if dir == "" {
+		// Default to $XDG_CACHE_HOME
+		cachedir, err := os.UserCacheDir()
+		if err != nil {
+			return nil, err
+		}
+
+		abs, _ := filepath.Abs(filepath.Join(cachedir, DefaultCacheDir))
+		if err := os.MkdirAll(abs, os.ModePerm); err != nil {
+			return nil, err
+		}
+
+		dir = abs
+	}
+
+	c := layer.NewFilesystemCache(dir)
+	return c, nil
+}

cmd/hauler/cli/store/info.go

@@ -0,0 +1,148 @@
+package store
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"text/tabwriter"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/spf13/cobra"
+
+	"github.com/rancherfederal/ocil/pkg/consts"
+
+	"github.com/rancherfederal/ocil/pkg/store"
+
+	"github.com/rancherfederal/hauler/pkg/reference"
+)
+
+type InfoOpts struct {
+	*RootOpts
+
+	OutputFormat string
+	SizeUnit     string
+}
+
+func (o *InfoOpts) AddFlags(cmd *cobra.Command) {
+	f := cmd.Flags()
+
+	f.StringVarP(&o.OutputFormat, "output", "o", "table", "Output format (table, json)")
+
+	// TODO: Regex/globbing
+}
+
+func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
+	var items []item
+	if err := s.Walk(func(ref string, desc ocispec.Descriptor) error {
+		if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
+			return nil
+		}
+
+		rc, err := s.Fetch(ctx, desc)
+		if err != nil {
+			return err
+		}
+		defer rc.Close()
+
+		var m ocispec.Manifest
+		if err := json.NewDecoder(rc).Decode(&m); err != nil {
+			return err
+		}
+
+		i := newItem(s, desc, m)
+		items = append(items, i)
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	var msg string
+	switch o.OutputFormat {
+	case "json":
+		msg = buildJson(items...)
+
+	default:
+		msg = buildTable(items...)
+	}
+	fmt.Println(msg)
+	return nil
+}
+
+func buildTable(items ...item) string {
+	b := strings.Builder{}
+	tw := tabwriter.NewWriter(&b, 1, 1, 3, ' ', 0)
+
+	fmt.Fprintf(tw, "Reference\tType\t# Layers\tSize\n")
+	fmt.Fprintf(tw, "---------\t----\t--------\t----\n")
+
+	for _, i := range items {
+		fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n",
+			i.Reference, i.Type, i.Layers, i.Size,
+		)
+	}
+	tw.Flush()
+	return b.String()
+}
+
+func buildJson(item ...item) string {
+	data, err := json.MarshalIndent(item, "", "  ")
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+type item struct {
+	Reference string
+	Type      string
+	Layers    int
+	Size      string
+}
+
+func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item {
+	var size int64 = 0
+	for _, l := range m.Layers {
+		size = +l.Size
+	}
+
+	// Generate a human-readable content type
+	var ctype string
+	switch m.Config.MediaType {
+	case consts.DockerConfigJSON:
+		ctype = "image"
+	case consts.ChartConfigMediaType:
+		ctype = "chart"
+	case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType:
+		ctype = "file"
+	default:
+		ctype = "unknown"
+	}
+
+	ref, err := reference.Parse(desc.Annotations[ocispec.AnnotationRefName])
+	if err != nil {
+		return item{}
+	}
+
+	return item{
+		Reference: ref.Name(),
+		Type:      ctype,
+		Layers:    len(m.Layers),
+		Size:      byteCountSI(size),
+	}
+}
+
+func byteCountSI(b int64) string {
+	const unit = 1000
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB",
+		float64(b)/float64(div), "kMGTPE"[exp])
+}

cmd/hauler/cli/store/list.go

@@ -1,47 +0,0 @@
-package store
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"text/tabwriter"
-
-	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/spf13/cobra"
-
-	"github.com/rancherfederal/hauler/pkg/store"
-)
-
-type ListOpts struct{}
-
-func (o *ListOpts) AddFlags(cmd *cobra.Command) {
-	f := cmd.Flags()
-	_ = f
-
-	// TODO: Regex matching
-}
-
-func ListCmd(ctx context.Context, o *ListOpts, s *store.Store) error {
-	s.Open()
-	defer s.Close()
-
-	refs, err := s.List(ctx)
-	if err != nil {
-		return err
-	}
-
-	tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
-	defer tw.Flush()
-
-	fmt.Fprintf(tw, "Reference\tTag/Digest\n")
-	for _, r := range refs {
-		ref, err := name.ParseReference(r, name.WithDefaultRegistry(""))
-		if err != nil {
-			return err
-		}
-
-		fmt.Fprintf(tw, "%s\t%s\n", ref.Context().String(), ref.Identifier())
-	}
-
-	return nil
-}

cmd/hauler/cli/store/load.go

@@ -2,40 +2,33 @@ package store
 
 import (
 	"context"
+	"os"
 
 	"github.com/mholt/archiver/v3"
+	"github.com/rancherfederal/ocil/pkg/content"
+	"github.com/rancherfederal/ocil/pkg/store"
 	"github.com/spf13/cobra"
 
 	"github.com/rancherfederal/hauler/pkg/log"
 )
 
 type LoadOpts struct {
-	OutputDir string
+	*RootOpts
 }
 
 func (o *LoadOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
-
-	f.StringVarP(&o.OutputDir, "output", "o", "", "Directory to unload archived contents to (defaults to $PWD/haul)")
+	_ = f
 }
 
 // LoadCmd
 // TODO: Just use mholt/archiver for now, even though we don't need most of it
-func LoadCmd(ctx context.Context, o *LoadOpts, dir string, archiveRefs ...string) error {
+func LoadCmd(ctx context.Context, o *LoadOpts, archiveRefs ...string) error {
 	l := log.FromContext(ctx)
 
-	// TODO: Support more formats?
-	a := archiver.NewTarZstd()
-	a.OverwriteExisting = true
-
-	odir := dir
-	if o.OutputDir != "" {
-		odir = o.OutputDir
-	}
-
 	for _, archiveRef := range archiveRefs {
-		l.Infof("loading content from [%s] to [%s]", archiveRef, odir)
-		err := a.Unarchive(archiveRef, odir)
+		l.Infof("loading content from [%s] to [%s]", archiveRef, o.StoreDir)
+		err := unarchiveLayoutTo(ctx, archiveRef, o.StoreDir)
 		if err != nil {
 			return err
 		}
@@ -43,3 +36,29 @@ func LoadCmd(ctx context.Context, o *LoadOpts, dir string, archiveRefs ...string
 
 	return nil
 }
+
+// unarchiveLayoutTo accepts an archived oci layout and extracts the contents to an existing oci layout, preserving the index
+func unarchiveLayoutTo(ctx context.Context, archivePath string, dest string) error {
+	tmpdir, err := os.MkdirTemp("", "hauler")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpdir)
+
+	if err := archiver.Unarchive(archivePath, tmpdir); err != nil {
+		return err
+	}
+
+	s, err := store.NewLayout(tmpdir)
+	if err != nil {
+		return err
+	}
+
+	ts, err := content.NewOCI(dest)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.CopyAll(ctx, ts, nil)
+	return err
+}

cmd/hauler/cli/store/save.go

@@ -12,6 +12,7 @@ import (
 )
 
 type SaveOpts struct {
+	*RootOpts
 	FileName string
 }
 
@@ -23,7 +24,7 @@ func (o *SaveOpts) AddArgs(cmd *cobra.Command) {
 
 // SaveCmd
 // TODO: Just use mholt/archiver for now, even though we don't need most of it
-func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) error {
+func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string) error {
 	l := log.FromContext(ctx)
 
 	// TODO: Support more formats?
@@ -40,7 +41,7 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er
 		return err
 	}
 	defer os.Chdir(cwd)
-	if err := os.Chdir(dir); err != nil {
+	if err := os.Chdir(o.StoreDir); err != nil {
 		return err
 	}
 
@@ -49,6 +50,6 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er
 		return err
 	}
 
-	l.Infof("saved haul [%s] -> [%s]", dir, absOutputfile)
+	l.Infof("saved store [%s] -> [%s]", o.StoreDir, absOutputfile)
 	return nil
 }

cmd/hauler/cli/store/serve.go

@@ -7,14 +7,23 @@ import (
 	"os"
 
 	"github.com/distribution/distribution/v3/configuration"
-	"github.com/distribution/distribution/v3/registry"
+	dcontext "github.com/distribution/distribution/v3/context"
+	_ "github.com/distribution/distribution/v3/registry/storage/driver/base"
+	_ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem"
+	_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
+	"github.com/distribution/distribution/v3/version"
 	"github.com/spf13/cobra"
 
-	"github.com/rancherfederal/hauler/pkg/store"
+	"github.com/rancherfederal/ocil/pkg/store"
+
+	"github.com/rancherfederal/hauler/internal/server"
 )
 
 type ServeOpts struct {
+	*RootOpts
+
 	Port       int
+	RootDir    string
 	ConfigFile string
 	Daemon     bool
 
@@ -25,13 +34,28 @@ func (o *ServeOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
 
 	f.IntVarP(&o.Port, "port", "p", 5000, "Port to listen on")
+	f.StringVar(&o.RootDir, "directory", "registry", "Directory to use for registry backend (defaults to '$PWD/registry')")
 	f.StringVarP(&o.ConfigFile, "config", "c", "", "Path to a config file, will override all other configs")
 	f.BoolVarP(&o.Daemon, "daemon", "d", false, "Toggle serving as a daemon")
 }
 
-// ServeCmd does
-func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error {
-	cfg := o.defaultConfig(s)
+// ServeCmd serves the embedded registry almost identically to how distribution/v3 does it
+func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Layout) error {
+	ctx = dcontext.WithVersion(ctx, version.Version)
+
+	tr := server.NewTempRegistry(ctx, o.RootDir)
+	if err := tr.Start(); err != nil {
+		return err
+	}
+
+	opts := &CopyOpts{}
+	if err := CopyCmd(ctx, opts, s, "registry://"+tr.Registry()); err != nil {
+		return err
+	}
+
+	tr.Close()
+
+	cfg := o.defaultConfig()
 	if o.ConfigFile != "" {
 		ucfg, err := loadConfig(o.ConfigFile)
 		if err != nil {
@@ -40,7 +64,7 @@ func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error {
 		cfg = ucfg
 	}
 
-	r, err := registry.NewRegistry(ctx, cfg)
+	r, err := server.NewRegistry(ctx, cfg)
 	if err != nil {
 		return err
 	}
@@ -48,7 +72,6 @@ func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error {
 	if err = r.ListenAndServe(); err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -61,15 +84,15 @@ func loadConfig(filename string) (*configuration.Configuration, error) {
 	return configuration.Parse(f)
 }
 
-func (o *ServeOpts) defaultConfig(s *store.Store) *configuration.Configuration {
+func (o *ServeOpts) defaultConfig() *configuration.Configuration {
 	cfg := &configuration.Configuration{
 		Version: "0.1",
 		Storage: configuration.Storage{
 			"cache":      configuration.Parameters{"blobdescriptor": "inmemory"},
-			"filesystem": configuration.Parameters{"rootdirectory": s.DataDir},
+			"filesystem": configuration.Parameters{"rootdirectory": o.RootDir},
 
 			// TODO: Ensure this is toggleable via cli arg if necessary
-			"maintenance": configuration.Parameters{"readonly.enabled": true},
+			// "maintenance": configuration.Parameters{"readonly.enabled": false},
 		},
 	}
 	cfg.Log.Level = "info"

cmd/hauler/cli/store/sync.go

@@ -8,17 +8,21 @@ import (
 	"os"
 
 	"github.com/spf13/cobra"
+	"helm.sh/helm/v3/pkg/action"
 	"k8s.io/apimachinery/pkg/util/yaml"
 
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
 	tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
+	"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
 	"github.com/rancherfederal/hauler/pkg/collection/k3s"
 	"github.com/rancherfederal/hauler/pkg/content"
 	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type SyncOpts struct {
+	*RootOpts
 	ContentFiles []string
 }
 
@@ -28,18 +32,15 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
 	f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
 }
 
-func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
+func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
 	l := log.FromContext(ctx)
 
 	// Start from an empty store (contents are cached elsewhere)
-	l.Debugf("flushing any existing content in store: %s", s.DataDir)
+	l.Debugf("flushing content store")
 	if err := s.Flush(ctx); err != nil {
 		return err
 	}
 
-	s.Open()
-	defer s.Close()
-
 	for _, filename := range o.ContentFiles {
 		l.Debugf("processing content file: '%s'", filename)
 		fi, err := os.Open(filename)
@@ -65,10 +66,11 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 		for _, doc := range docs {
 			obj, err := content.Load(doc)
 			if err != nil {
-				return err
+				l.Debugf("skipping sync of unknown content")
+				continue
 			}
 
-			l.Infof("syncing [%s] to [%s]", obj.GroupVersionKind().String(), s.DataDir)
+			l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())
 
 			// TODO: Should type switch instead...
 			switch obj.GroupVersionKind().Kind {
@@ -105,7 +107,8 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 				}
 
 				for _, ch := range cfg.Spec.Charts {
-					err := storeChart(ctx, s, ch)
+					// TODO: Provide a way to configure syncs
+					err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
 					if err != nil {
 						return err
 					}
@@ -122,7 +125,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 					return err
 				}
 
-				if _, err := s.AddCollection(ctx, k); err != nil {
+				if _, err := s.AddOCICollection(ctx, k); err != nil {
 					return err
 				}
 
@@ -133,16 +136,39 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 				}
 
 				for _, cfg := range cfg.Spec.Charts {
-					tc, err := tchart.NewChart(cfg.Name, cfg.RepoURL, cfg.Version)
+					tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
+						RepoURL: cfg.RepoURL,
+						Version: cfg.Version,
+					})
 					if err != nil {
 						return err
 					}
 
-					if _, err := s.AddCollection(ctx, tc); err != nil {
+					if _, err := s.AddOCICollection(ctx, tc); err != nil {
 						return err
 					}
 				}
 
+			case v1alpha1.ImageTxtsContentKind:
+				var cfg v1alpha1.ImageTxts
+				if err := yaml.Unmarshal(doc, &cfg); err != nil {
+					return err
+				}
+
+				for _, cfgIt := range cfg.Spec.ImageTxts {
+					it, err := imagetxt.New(cfgIt.Ref,
+						imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
+						imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
+					)
+					if err != nil {
+						return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
+					}
+
+					if _, err := s.AddOCICollection(ctx, it); err != nil {
+						return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
+					}
+				}
+
 			default:
 				return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
 			}

cmd/hauler/cli/version.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/rancherfederal/hauler/pkg/version"
+	"github.com/rancherfederal/hauler/internal/version"
 )
 
 func addVersion(parent *cobra.Command) {
@@ -13,8 +13,7 @@ func addVersion(parent *cobra.Command) {
 
 	cmd := &cobra.Command{
 		Use:     "version",
-		Short:   "Print current hauler version",
-		Long:    "Print current hauler version",
+		Short:   "Print the current version",
 		Aliases: []string{"v"},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v := version.GetVersionInfo()
@@ -30,7 +29,6 @@ func addVersion(parent *cobra.Command) {
 			return nil
 		},
 	}
-
 	cmd.Flags().BoolVar(&json, "json", false, "toggle output in JSON")
 
 	parent.AddCommand(cmd)

go.mod

@@ -3,67 +3,70 @@ module github.com/rancherfederal/hauler
 go 1.17
 
 require (
-	github.com/containerd/containerd v1.5.7
-	github.com/distribution/distribution/v3 v3.0.0-20210926092439-1563384b69df
-	github.com/google/go-containerregistry v0.6.0
-	github.com/mholt/archiver/v3 v3.5.0
-	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.0.1
-	github.com/rancher/wrangler v0.8.4
+	github.com/containerd/containerd v1.5.9
+	github.com/distribution/distribution/v3 v3.0.0-20211125133600-cc4627fc6e5f
+	github.com/docker/go-metrics v0.0.1
+	github.com/google/go-containerregistry v0.7.0
+	github.com/gorilla/handlers v1.5.1
+	github.com/gorilla/mux v1.8.0
+	github.com/mholt/archiver/v3 v3.5.1
+	github.com/opencontainers/image-spec v1.0.2
+	github.com/pkg/errors v0.9.1
+	github.com/rancherfederal/ocil v0.1.9
 	github.com/rs/zerolog v1.26.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cobra v1.2.1
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	helm.sh/helm/v3 v3.7.1
-	k8s.io/apimachinery v0.22.2
-	k8s.io/client-go v0.22.2
-	oras.land/oras-go v0.4.0
-	sigs.k8s.io/controller-runtime v0.10.3
+	github.com/spf13/cobra v1.3.0
+	helm.sh/helm/v3 v3.8.0
+	k8s.io/apimachinery v0.23.1
+	k8s.io/client-go v0.23.1
+	oras.land/oras-go v1.1.0
+)
+
+replace (
+	github.com/go-logr/logr v1.2.0 => github.com/go-logr/logr v0.4.0
+	k8s.io/klog/v2 v2.30.0 => k8s.io/klog/v2 v2.9.0
 )
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.2 // indirect
-	github.com/Masterminds/squirrel v1.5.0 // indirect
+	github.com/Masterminds/squirrel v1.5.2 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
-	github.com/andybalholm/brotli v1.0.0 // indirect
+	github.com/andybalholm/brotli v1.0.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
 	github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd // indirect
 	github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
 	github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.7.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/cli v20.10.9+incompatible // indirect
+	github.com/docker/cli v20.10.11+incompatible // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/docker v20.10.9+incompatible // indirect
+	github.com/docker/docker v20.10.12+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
-	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
-	github.com/dsnet/compress v0.0.1 // indirect
-	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
+	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
-	github.com/fatih/color v1.9.0 // indirect
+	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
-	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
-	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -73,28 +76,26 @@ require (
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.2.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/gorilla/handlers v1.5.1 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jmoiron/sqlx v1.3.1 // indirect
+	github.com/jmoiron/sqlx v1.3.4 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
-	github.com/klauspost/pgzip v1.2.4 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
-	github.com/lib/pq v1.10.0 // indirect
+	github.com/lib/pq v1.10.4 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.13 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
@@ -103,28 +104,27 @@ require (
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nwaples/rardecode v1.1.0 // indirect
+	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/gomega v1.15.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/pierrec/lz4/v4 v4.0.3 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/prometheus/common v0.30.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
 	github.com/russross/blackfriday v1.5.2 // indirect
-	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
-	github.com/ulikunitz/xz v0.5.7 // indirect
+	github.com/ulikunitz/xz v0.5.9 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -134,32 +134,34 @@ require (
 	github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 // indirect
 	github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
-	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
-	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
-	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
+	golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492 // indirect
-	google.golang.org/grpc v1.39.0 // indirect
+	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
+	google.golang.org/grpc v1.43.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.22.2 // indirect
-	k8s.io/apiextensions-apiserver v0.22.2 // indirect
-	k8s.io/apiserver v0.22.2 // indirect
-	k8s.io/cli-runtime v0.22.1 // indirect
-	k8s.io/component-base v0.22.2 // indirect
-	k8s.io/klog/v2 v2.9.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
-	k8s.io/kubectl v0.22.1 // indirect
-	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
-	sigs.k8s.io/kustomize/api v0.8.11 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.11.0 // indirect
+	k8s.io/api v0.23.1 // indirect
+	k8s.io/apiextensions-apiserver v0.23.1 // indirect
+	k8s.io/apiserver v0.23.1 // indirect
+	k8s.io/cli-runtime v0.23.1 // indirect
+	k8s.io/component-base v0.23.1 // indirect
+	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/kubectl v0.23.1 // indirect
+	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/kustomize/api v0.10.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

internal/mapper/filestore.go

@@ -0,0 +1,85 @@
+package mapper
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	ccontent "github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/remotes"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"oras.land/oras-go/pkg/content"
+)
+
+// NewMapperFileStore creates a new file store that uses mapper functions for each detected descriptor.
+// 		This extends content.File, and differs in that it allows much more functionality into how each descriptor is written.
+func NewMapperFileStore(root string, mapper map[string]Fn) *store {
+	fs := content.NewFile(root)
+	return &store{
+		File:   fs,
+		mapper: mapper,
+	}
+}
+
+func (s *store) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) {
+	var tag, hash string
+	parts := strings.SplitN(ref, "@", 2)
+	if len(parts) > 0 {
+		tag = parts[0]
+	}
+	if len(parts) > 1 {
+		hash = parts[1]
+	}
+	return &pusher{
+		store:  s.File,
+		tag:    tag,
+		ref:    hash,
+		mapper: s.mapper,
+	}, nil
+}
+
+type store struct {
+	*content.File
+	mapper map[string]Fn
+}
+
+func (s *pusher) Push(ctx context.Context, desc ocispec.Descriptor) (ccontent.Writer, error) {
+	// TODO: This is suuuuuper ugly... redo this when oras v2 is out
+	if _, ok := content.ResolveName(desc); ok {
+		p, err := s.store.Pusher(ctx, s.ref)
+		if err != nil {
+			return nil, err
+		}
+		return p.Push(ctx, desc)
+	}
+
+	// If no custom mapper found, fall back to content.File mapper
+	if _, ok := s.mapper[desc.MediaType]; !ok {
+		return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(desc.Digest)), nil
+	}
+
+	filename, err := s.mapper[desc.MediaType](desc)
+	if err != nil {
+		return nil, err
+	}
+
+	fullFileName := filepath.Join(s.store.ResolvePath(""), filename)
+	// TODO: Don't rewrite everytime, we can check the digest
+	f, err := os.OpenFile(fullFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return nil, errors.Wrap(err, "pushing file")
+	}
+
+	w := content.NewIoContentWriter(f, content.WithInputHash(desc.Digest), content.WithOutputHash(desc.Digest))
+	return w, nil
+}
+
+type pusher struct {
+	store  *content.File
+	tag    string
+	ref    string
+	mapper map[string]Fn
+}

internal/mapper/mappers.go

@@ -0,0 +1,83 @@
+package mapper
+
+import (
+	"fmt"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"oras.land/oras-go/pkg/target"
+
+	"github.com/rancherfederal/ocil/pkg/consts"
+)
+
+type Fn func(desc ocispec.Descriptor) (string, error)
+
+// FromManifest will return the appropriate content store given a reference and source type adequate for storing the results on disk
+func FromManifest(manifest ocispec.Manifest, root string) (target.Target, error) {
+	// TODO: Don't rely solely on config mediatype
+	switch manifest.Config.MediaType {
+	case consts.DockerConfigJSON, consts.OCIManifestSchema1:
+		s := NewMapperFileStore(root, Images())
+		defer s.Close()
+		return s, nil
+
+	case consts.ChartLayerMediaType, consts.ChartConfigMediaType:
+		s := NewMapperFileStore(root, Chart())
+		defer s.Close()
+		return s, nil
+
+	default:
+		s := NewMapperFileStore(root, nil)
+		defer s.Close()
+		return s, nil
+	}
+}
+
+func Images() map[string]Fn {
+	m := make(map[string]Fn)
+
+	manifestMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) {
+		return "manifest.json", nil
+	})
+
+	for _, l := range []string{consts.DockerManifestSchema2, consts.OCIManifestSchema1} {
+		m[l] = manifestMapperFn
+	}
+
+	layerMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) {
+		return fmt.Sprintf("%s.tar.gz", desc.Digest.String()), nil
+	})
+
+	for _, l := range []string{consts.OCILayer, consts.DockerLayer} {
+		m[l] = layerMapperFn
+	}
+
+	configMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) {
+		return "config.json", nil
+	})
+
+	for _, l := range []string{consts.DockerConfigJSON} {
+		m[l] = configMapperFn
+	}
+
+	return m
+}
+
+func Chart() map[string]Fn {
+	m := make(map[string]Fn)
+
+	chartMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) {
+		f := "chart.tar.gz"
+		if _, ok := desc.Annotations[ocispec.AnnotationTitle]; ok {
+			f = desc.Annotations[ocispec.AnnotationTitle]
+		}
+		return f, nil
+	})
+
+	provMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) {
+		return "prov.json", nil
+	})
+
+	m[consts.ChartLayerMediaType] = chartMapperFn
+	m[consts.ProvLayerMediaType] = provMapperFn
+	return m
+}

internal/server/file.go

@@ -0,0 +1,42 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+)
+
+type FileConfig struct {
+	Root string
+	Host string
+	Port int
+}
+
+// NewFile returns a fileserver
+// TODO: Better configs
+func NewFile(ctx context.Context, cfg FileConfig) (Server, error) {
+	r := mux.NewRouter()
+	r.Handle("/", handlers.LoggingHandler(os.Stdout, http.FileServer(http.Dir(cfg.Root))))
+
+	if cfg.Root == "" {
+		cfg.Root = "."
+	}
+
+	if cfg.Port == 0 {
+		cfg.Port = 8080
+	}
+
+	srv := &http.Server{
+		Handler:      r,
+		Addr:         fmt.Sprintf(":%d", cfg.Port),
+		WriteTimeout: 15 * time.Second,
+		ReadTimeout:  15 * time.Second,
+	}
+
+	return srv, nil
+}

internal/server/registry.go

@@ -0,0 +1,119 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"time"
+
+	"github.com/distribution/distribution/v3/configuration"
+	"github.com/distribution/distribution/v3/registry"
+	"github.com/distribution/distribution/v3/registry/handlers"
+	"github.com/docker/go-metrics"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func NewRegistry(ctx context.Context, cfg *configuration.Configuration) (*registry.Registry, error) {
+	r, err := registry.NewRegistry(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	if cfg.HTTP.Debug.Prometheus.Enabled {
+		path := cfg.HTTP.Debug.Prometheus.Path
+		if path == "" {
+			path = "/metrics"
+		}
+		http.Handle(path, metrics.Handler())
+	}
+
+	return r, nil
+}
+
+type tmpRegistryServer struct {
+	*httptest.Server
+}
+
+func NewTempRegistry(ctx context.Context, root string) *tmpRegistryServer {
+	cfg := &configuration.Configuration{
+		Version: "0.1",
+		Storage: configuration.Storage{
+			"cache":      configuration.Parameters{"blobdescriptor": "inmemory"},
+			"filesystem": configuration.Parameters{"rootdirectory": root},
+		},
+	}
+	cfg.Log.Level = "error"
+	cfg.HTTP.Headers = http.Header{
+		"X-Content-Type-Options": []string{"nosniff"},
+	}
+
+	l, err := logrus.ParseLevel("panic")
+	if err != nil {
+		l = logrus.ErrorLevel
+	}
+	logrus.SetLevel(l)
+
+	app := handlers.NewApp(ctx, cfg)
+	app.RegisterHealthChecks()
+	handler := alive("/", app)
+
+	s := httptest.NewUnstartedServer(handler)
+	return &tmpRegistryServer{
+		Server: s,
+	}
+}
+
+// Registry returns the URL of the server without the protocol, suitable for content references
+func (t *tmpRegistryServer) Registry() string {
+	return strings.Replace(t.Server.URL, "http://", "", 1)
+}
+
+func (t *tmpRegistryServer) Start() error {
+	t.Server.Start()
+
+	err := retry(5, 1*time.Second, func() (err error) {
+		resp, err := http.Get(t.Server.URL + "/v2")
+		if err != nil {
+			return err
+		}
+		resp.Body.Close()
+		if resp.StatusCode == http.StatusOK {
+			return nil
+		}
+		return errors.New("to start temporary registry")
+
+	})
+	return err
+}
+
+func (t *tmpRegistryServer) Stop() {
+	t.Server.Close()
+}
+
+func alive(path string, handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == path {
+			w.Header().Set("Cache-Control", "no-cache")
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		handler.ServeHTTP(w, r)
+	})
+}
+
+func retry(attempts int, sleep time.Duration, f func() error) (err error) {
+	for i := 0; i < attempts; i++ {
+		if i > 0 {
+			time.Sleep(sleep)
+			sleep *= 2
+		}
+		err = f()
+		if err == nil {
+			return nil
+		}
+	}
+	return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
+}

internal/server/server.go

@@ -0,0 +1,5 @@
+package server
+
+type Server interface {
+	ListenAndServe() error
+}

pkg/apis/hauler.cattle.io/v1alpha1/chart.go

@@ -21,24 +21,27 @@ type ChartSpec struct {
 }
 
 type Chart struct {
-	Name    string `json:"name"`
-	RepoURL string `json:"repoURL"`
-	Version string `json:"version"`
+	Name    string `json:"name,omitempty"`
+	RepoURL string `json:"repoURL,omitempty"`
+	Version string `json:"version,omitempty"`
 }
 
 type ThickCharts struct {
 	*metav1.TypeMeta  `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	Spec ChartSpec `json:"spec,omitempty"`
+	Spec ThickChartSpec `json:"spec,omitempty"`
 }
 
 type ThickChartSpec struct {
-	ThickCharts []ThickChart `json:"charts,omitempty"`
+	Charts []ThickChart `json:"charts,omitempty"`
 }
 
 type ThickChart struct {
-	Name    string `json:"name"`
-	RepoURL string `json:"repoURL"`
-	Version string `json:"version"`
+	Chart       `json:",inline,omitempty"`
+	ExtraImages []ChartImage `json:"extraImages,omitempty"`
+}
+
+type ChartImage struct {
+	Reference string `json:"ref"`
 }

pkg/apis/hauler.cattle.io/v1alpha1/file.go

@@ -18,6 +18,10 @@ type FileSpec struct {
 }
 
 type File struct {
-	Ref  string `json:"ref"`
+	// Path is the path to the file contents, can be a local or remote path
+	Path string `json:"path"`
+
+	// Name is an optional field specifying the name of the file when specified,
+	// 	it will override any dynamic name discovery from Path
 	Name string `json:"name,omitempty"`
 }

pkg/apis/hauler.cattle.io/v1alpha1/groupversion_info.go

@@ -2,7 +2,6 @@ package v1alpha1
 
 import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/controller-runtime/pkg/scheme"
 )
 
 const (
@@ -13,7 +12,7 @@ const (
 
 var (
 	ContentGroupVersion = schema.GroupVersion{Group: ContentGroup, Version: Version}
-	SchemeBuilder       = &scheme.Builder{GroupVersion: ContentGroupVersion}
+	// SchemeBuilder       = &scheme.Builder{GroupVersion: ContentGroupVersion}
 
 	CollectionGroupVersion = schema.GroupVersion{Group: CollectionGroup, Version: Version}
 )

pkg/apis/hauler.cattle.io/v1alpha1/image.go

@@ -18,5 +18,6 @@ type ImageSpec struct {
 }
 
 type Image struct {
-	Ref string `json:"ref"`
+	// Name is the full location for the image, can be referenced by tags or digests
+	Name string `json:"name"`
 }

pkg/apis/hauler.cattle.io/v1alpha1/imagetxt.go

@@ -0,0 +1,30 @@
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ImageTxtsContentKind = "ImageTxts"
+)
+
+type ImageTxts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ImageTxtsSpec `json:"spec,omitempty"`
+}
+
+type ImageTxtsSpec struct {
+	ImageTxts []ImageTxt `json:"imageTxts,omitempty"`
+}
+
+type ImageTxt struct {
+	Ref     string          `json:"ref,omitempty"`
+	Sources ImageTxtSources `json:"sources,omitempty"`
+}
+
+type ImageTxtSources struct {
+	Include []string `json:"include,omitempty"`
+	Exclude []string `json:"exclude,omitempty"`
+}

pkg/artifact/config.go

@@ -1,10 +0,0 @@
-package artifact
-
-import v1 "github.com/google/go-containerregistry/pkg/v1"
-
-type Config interface {
-	// Raw returns the config bytes
-	Raw() ([]byte, error)
-
-	Descriptor() (v1.Descriptor, error)
-}

pkg/artifact/local/layer.go

@@ -1,127 +0,0 @@
-package local
-
-import (
-	"io"
-
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-	gtypes "github.com/google/go-containerregistry/pkg/v1/types"
-
-	"github.com/rancherfederal/hauler/pkg/artifact/types"
-)
-
-type Opener func() (io.ReadCloser, error)
-
-func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
-	var err error
-
-	layer := &layer{
-		mediaType:   types.UnknownLayer,
-		annotations: make(map[string]string, 1),
-	}
-
-	layer.uncompressedOpener = opener
-	layer.compressedOpener = func() (io.ReadCloser, error) {
-		rc, err := opener()
-		if err != nil {
-			return nil, err
-		}
-		// TODO: actually compress this
-		return rc, nil
-	}
-
-	for _, opt := range opts {
-		opt(layer)
-	}
-
-	if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil {
-		return nil, err
-	}
-
-	if layer.diffID, _, err = compute(layer.compressedOpener); err != nil {
-		return nil, err
-	}
-
-	return layer, nil
-}
-
-func compute(opener Opener) (v1.Hash, int64, error) {
-	rc, err := opener()
-	if err != nil {
-		return v1.Hash{}, 0, err
-	}
-	defer rc.Close()
-	return v1.SHA256(rc)
-}
-
-type LayerOption func(*layer)
-
-func WithMediaType(mt string) LayerOption {
-	return func(l *layer) {
-		l.mediaType = mt
-	}
-}
-
-func WithAnnotations(annotations map[string]string) LayerOption {
-	return func(l *layer) {
-		if l.annotations == nil {
-			l.annotations = make(map[string]string)
-		}
-		l.annotations = annotations
-	}
-}
-
-type layer struct {
-	digest             v1.Hash
-	diffID             v1.Hash
-	size               int64
-	compressedOpener   Opener
-	uncompressedOpener Opener
-	mediaType          string
-	annotations        map[string]string
-	urls               []string
-}
-
-func (l layer) Descriptor() (*v1.Descriptor, error) {
-	digest, err := l.Digest()
-	if err != nil {
-		return nil, err
-	}
-	mt, err := l.MediaType()
-	if err != nil {
-		return nil, err
-	}
-	return &v1.Descriptor{
-		MediaType:   mt,
-		Size:        l.size,
-		Digest:      digest,
-		Annotations: l.annotations,
-		URLs:        l.urls,
-
-		// TODO: Allow platforms
-		Platform: nil,
-	}, nil
-}
-
-func (l layer) Digest() (v1.Hash, error) {
-	return l.digest, nil
-}
-
-func (l layer) DiffID() (v1.Hash, error) {
-	return l.diffID, nil
-}
-
-func (l layer) Compressed() (io.ReadCloser, error) {
-	return l.compressedOpener()
-}
-
-func (l layer) Uncompressed() (io.ReadCloser, error) {
-	return l.uncompressedOpener()
-}
-
-func (l layer) Size() (int64, error) {
-	return l.size, nil
-}
-
-func (l layer) MediaType() (gtypes.MediaType, error) {
-	return gtypes.MediaType(l.mediaType), nil
-}

pkg/artifact/oci.go

@@ -1,24 +0,0 @@
-package artifact
-
-import (
-	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/google/go-containerregistry/pkg/v1"
-)
-
-// OCI is the bare minimum we need to represent an artifact in an oci layout
-//  At a high level, it is not constrained by an Image's config, manifests, and layer ordinality
-//  This specific implementation fully encapsulates v1.Layer's within a more generic form
-type OCI interface {
-	MediaType() string
-
-	Manifest() (*v1.Manifest, error)
-
-	RawConfig() ([]byte, error)
-
-	Layers() ([]v1.Layer, error)
-}
-
-type Collection interface {
-	// Contents returns the list of contents in the collection
-	Contents() (map[name.Reference]OCI, error)
-}

pkg/artifact/types/types.go

@@ -1,37 +0,0 @@
-package types
-
-const (
-	OCIManifestSchema1    = "application/vnd.oci.image.manifest.v1+json"
-	DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json"
-
-	DockerConfigJSON = "application/vnd.docker.container.image.v1+json"
-
-	// ChartConfigMediaType is the reserved media type for the Helm chart manifest config
-	ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
-
-	// ChartLayerMediaType is the reserved media type for Helm chart package content
-	ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"
-
-	// ProvLayerMediaType is the reserved media type for Helm chart provenance files
-	ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov"
-
-	// FileLayerMediaType is the reserved media type for File content layers
-	FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1"
-
-	// FileConfigMediaType is the reserved media type for File config
-	FileConfigMediaType = "application/vnd.content.hauler.file.config.v1+json"
-
-	// WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers
-	WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm"
-
-	// WasmConfigMediaType is the reserved media type for WASM configs
-	WasmConfigMediaType = "application/vnd.wasm.config.v1+json"
-
-	UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json"
-	UnknownLayer    = "application/vnd.content.hauler.unknown.layer"
-
-	OCIVendorPrefix    = "vnd.oci"
-	DockerVendorPrefix = "vnd.docker"
-	HaulerVendorPrefix = "vnd.hauler"
-	OCIImageIndexFile  = "index.json"
-)

pkg/cache/cache.go

@@ -1,102 +0,0 @@
-package cache
-
-import (
-	"errors"
-	"io"
-
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/types"
-
-	"github.com/rancherfederal/hauler/pkg/artifact"
-)
-
-type Cache interface {
-	Put(v1.Layer) (v1.Layer, error)
-
-	Get(v1.Hash) (v1.Layer, error)
-}
-
-var ErrLayerNotFound = errors.New("layer not found")
-
-type oci struct {
-	artifact.OCI
-
-	c Cache
-}
-
-func Oci(o artifact.OCI, c Cache) artifact.OCI {
-	return &oci{
-		OCI: o,
-		c:   c,
-	}
-}
-
-func (o *oci) Layers() ([]v1.Layer, error) {
-	ls, err := o.OCI.Layers()
-	if err != nil {
-		return nil, err
-	}
-
-	var out []v1.Layer
-	for _, l := range ls {
-		out = append(out, &lazyLayer{inner: l, c: o.c})
-	}
-	return out, nil
-}
-
-type lazyLayer struct {
-	inner v1.Layer
-	c     Cache
-}
-
-func (l *lazyLayer) Compressed() (io.ReadCloser, error) {
-	digest, err := l.inner.Digest()
-	if err != nil {
-		return nil, err
-	}
-
-	layer, err := l.getOrPut(digest)
-	if err != nil {
-		return nil, err
-	}
-
-	return layer.Compressed()
-}
-
-func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) {
-	diffID, err := l.inner.DiffID()
-	if err != nil {
-		return nil, err
-	}
-
-	layer, err := l.getOrPut(diffID)
-	if err != nil {
-		return nil, err
-	}
-
-	return layer.Uncompressed()
-}
-
-func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) {
-	var layer v1.Layer
-	if cl, err := l.c.Get(h); err == nil {
-		layer = cl
-
-	} else if err == ErrLayerNotFound {
-		rl, err := l.c.Put(l.inner)
-		if err != nil {
-			return nil, err
-		}
-		layer = rl
-
-	} else {
-		return nil, err
-	}
-
-	return layer, nil
-}
-
-func (l *lazyLayer) Size() (int64, error)                { return l.inner.Size() }
-func (l *lazyLayer) DiffID() (v1.Hash, error)            { return l.inner.Digest() }
-func (l *lazyLayer) Digest() (v1.Hash, error)            { return l.inner.Digest() }
-func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() }

pkg/cache/doc.go

@@ -1,5 +0,0 @@
-package cache
-
-/*
-This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache
-*/

pkg/cache/filesystem.go

@@ -1,120 +0,0 @@
-package cache
-
-import (
-	"io"
-	"os"
-	"path/filepath"
-
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-
-	"github.com/rancherfederal/hauler/pkg/artifact/local"
-)
-
-type fs struct {
-	root string
-}
-
-func NewFilesystem(root string) Cache {
-	return &fs{root: root}
-}
-
-func (f *fs) Put(l v1.Layer) (v1.Layer, error) {
-	digest, err := l.Digest()
-	if err != nil {
-		return nil, err
-	}
-	diffID, err := l.DiffID()
-	if err != nil {
-		return nil, err
-	}
-	return &cachedLayer{
-		Layer:  l,
-		root:   f.root,
-		digest: digest,
-		diffID: diffID,
-	}, nil
-}
-
-func (f *fs) Get(h v1.Hash) (v1.Layer, error) {
-	opener := f.open(h)
-	l, err := local.LayerFromOpener(opener)
-	if os.IsNotExist(err) {
-		return nil, ErrLayerNotFound
-	}
-	return l, err
-}
-
-func (f *fs) open(h v1.Hash) local.Opener {
-	return func() (io.ReadCloser, error) {
-		return os.Open(layerpath(f.root, h))
-	}
-}
-
-type cachedLayer struct {
-	v1.Layer
-
-	root           string
-	digest, diffID v1.Hash
-}
-
-func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) {
-	lp := layerpath(l.root, h)
-	if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil {
-		return nil, err
-	}
-	return os.Create(lp)
-}
-
-func (l *cachedLayer) Compressed() (io.ReadCloser, error) {
-	f, err := l.create(l.digest)
-	if err != nil {
-		return nil, nil
-	}
-	rc, err := l.Layer.Compressed()
-	if err != nil {
-		return nil, err
-	}
-	return &readcloser{
-		t:      io.TeeReader(rc, f),
-		closes: []func() error{rc.Close, f.Close},
-	}, nil
-}
-
-func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) {
-	f, err := l.create(l.diffID)
-	if err != nil {
-		return nil, err
-	}
-	rc, err := l.Layer.Uncompressed()
-	if err != nil {
-		return nil, err
-	}
-	return &readcloser{
-		t:      io.TeeReader(rc, f),
-		closes: []func() error{rc.Close, f.Close},
-	}, nil
-}
-
-func layerpath(root string, h v1.Hash) string {
-	return filepath.Join(root, h.Algorithm, h.Hex)
-}
-
-type readcloser struct {
-	t      io.Reader
-	closes []func() error
-}
-
-func (rc *readcloser) Read(b []byte) (int, error) {
-	return rc.t.Read(b)
-}
-
-func (rc *readcloser) Close() error {
-	var err error
-	for _, c := range rc.closes {
-		lastErr := c()
-		if err == nil {
-			err = lastErr
-		}
-	}
-	return err
-}

pkg/collection/chart/chart.go

@@ -1,42 +1,40 @@
 package chart
 
 import (
-	gname "github.com/google/go-containerregistry/pkg/name"
+	"github.com/rancherfederal/ocil/pkg/artifacts"
+	"github.com/rancherfederal/ocil/pkg/artifacts/image"
+	"helm.sh/helm/v3/pkg/action"
 
-	"github.com/rancherfederal/hauler/pkg/artifact"
+	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
 	"github.com/rancherfederal/hauler/pkg/content/chart"
-	"github.com/rancherfederal/hauler/pkg/content/image"
+	"github.com/rancherfederal/hauler/pkg/reference"
 )
 
-var _ artifact.Collection = (*tchart)(nil)
+var _ artifacts.OCICollection = (*tchart)(nil)
 
 // tchart is a thick chart that includes all the dependent images as well as the chart itself
 type tchart struct {
-	name    string
-	repo    string
-	version string
-	chart   *chart.Chart
+	chart  *chart.Chart
+	config v1alpha1.ThickChart
 
 	computed bool
-	contents map[gname.Reference]artifact.OCI
+	contents map[string]artifacts.OCI
 }
 
-func NewChart(name, repo, version string) (artifact.Collection, error) {
-	o, err := chart.NewChart(name, repo, version)
+func NewThickChart(cfg v1alpha1.ThickChart, opts *action.ChartPathOptions) (artifacts.OCICollection, error) {
+	o, err := chart.NewChart(cfg.Chart.Name, opts)
 	if err != nil {
 		return nil, err
 	}
 
 	return &tchart{
-		name:     name,
-		repo:     repo,
-		version:  version,
 		chart:    o,
-		contents: make(map[gname.Reference]artifact.OCI),
+		config:   cfg,
+		contents: make(map[string]artifacts.OCI),
 	}, nil
 }
 
-func (c *tchart) Contents() (map[gname.Reference]artifact.OCI, error) {
+func (c *tchart) Contents() (map[string]artifacts.OCI, error) {
 	if err := c.compute(); err != nil {
 		return nil, err
 	}
@@ -51,32 +49,28 @@ func (c *tchart) compute() error {
 	if err := c.dependentImages(); err != nil {
 		return err
 	}
-
 	if err := c.chartContents(); err != nil {
 		return err
 	}
+	if err := c.extraImages(); err != nil {
+		return err
+	}
 
 	c.computed = true
 	return nil
 }
 
 func (c *tchart) chartContents() error {
-	oci, err := chart.NewChart(c.name, c.repo, c.version)
+	ch, err := c.chart.Load()
 	if err != nil {
 		return err
 	}
 
-	tag := c.version
-	if tag == "" {
-		tag = gname.DefaultTag
-	}
-
-	ref, err := gname.ParseReference(c.name, gname.WithDefaultRegistry(""), gname.WithDefaultTag(tag))
+	ref, err := reference.NewTagged(ch.Name(), ch.Metadata.Version)
 	if err != nil {
 		return err
 	}
-
-	c.contents[ref] = oci
+	c.contents[ref.Name()] = c.chart
 	return nil
 }
 
@@ -92,17 +86,22 @@ func (c *tchart) dependentImages() error {
 	}
 
 	for _, img := range imgs.Spec.Images {
-		ref, err := gname.ParseReference(img.Ref)
+		i, err := image.NewImage(img.Name)
 		if err != nil {
 			return err
 		}
-
-		i, err := image.NewImage(img.Ref)
-		if err != nil {
-			return err
-		}
-		c.contents[ref] = i
+		c.contents[img.Name] = i
+	}
+	return nil
+}
+
+func (c *tchart) extraImages() error {
+	for _, img := range c.config.ExtraImages {
+		i, err := image.NewImage(img.Reference)
+		if err != nil {
+			return err
+		}
+		c.contents[img.Reference] = i
 	}
-
 	return nil
 }

pkg/collection/chart/dependents.go

@@ -1,20 +1,18 @@
 package chart
 
 import (
+	"bufio"
 	"bytes"
-	"encoding/json"
 	"io"
 	"strings"
 
-	"github.com/rancher/wrangler/pkg/yaml"
 	"helm.sh/helm/v3/pkg/action"
 	helmchart "helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chartutil"
 	"helm.sh/helm/v3/pkg/kube/fake"
 	"helm.sh/helm/v3/pkg/storage"
 	"helm.sh/helm/v3/pkg/storage/driver"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/util/jsonpath"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
@@ -32,50 +30,37 @@ var defaultKnownImagePaths = []string{
 
 // ImagesInChart will render a chart and identify all dependent images from it
 func ImagesInChart(c *helmchart.Chart) (v1alpha1.Images, error) {
-	objs, err := template(c)
+	docs, err := template(c)
 	if err != nil {
 		return v1alpha1.Images{}, err
 	}
 
-	var imageRefs []string
-	for _, o := range objs {
-		d, err := o.(*unstructured.Unstructured).MarshalJSON()
+	var images []v1alpha1.Image
+	reader := yaml.NewYAMLReader(bufio.NewReader(strings.NewReader(docs)))
+	for {
+		raw, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
 		if err != nil {
-			// TODO: Should we actually capture these errors?
-			continue
+			return v1alpha1.Images{}, err
 		}
 
-		var obj interface{}
-		if err := json.Unmarshal(d, &obj); err != nil {
-			continue
-		}
-
-		j := jsonpath.New("")
-		j.AllowMissingKeys(true)
-
-		for _, p := range defaultKnownImagePaths {
-			r, err := parseJSONPath(obj, j, p)
-			if err != nil {
-				continue
-			}
-
-			imageRefs = append(imageRefs, r...)
+		found := find(raw, defaultKnownImagePaths...)
+		for _, f := range found {
+			images = append(images, v1alpha1.Image{Name: f})
 		}
 	}
 
 	ims := v1alpha1.Images{
 		Spec: v1alpha1.ImageSpec{
-			Images: []v1alpha1.Image{},
+			Images: images,
 		},
 	}
-
-	for _, ref := range imageRefs {
-		ims.Spec.Images = append(ims.Spec.Images, v1alpha1.Image{Ref: ref})
-	}
 	return ims, nil
 }
 
-func template(c *helmchart.Chart) ([]runtime.Object, error) {
+func template(c *helmchart.Chart) (string, error) {
 	s := storage.Init(driver.NewMemory())
 
 	templateCfg := &action.Configuration{
@@ -99,10 +84,33 @@ func template(c *helmchart.Chart) ([]runtime.Object, error) {
 
 	release, err := client.Run(c, vals)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	return yaml.ToObjects(bytes.NewBufferString(release.Manifest))
+	return release.Manifest, nil
+}
+
+func find(data []byte, paths ...string) []string {
+	var (
+		pathMatches []string
+		obj         interface{}
+	)
+
+	if err := yaml.Unmarshal(data, &obj); err != nil {
+		return nil
+	}
+	j := jsonpath.New("")
+	j.AllowMissingKeys(true)
+
+	for _, p := range paths {
+		r, err := parseJSONPath(obj, j, p)
+		if err != nil {
+			continue
+		}
+
+		pathMatches = append(pathMatches, r...)
+	}
+	return pathMatches
 }
 
 func parseJSONPath(data interface{}, parser *jsonpath.JSONPath, template string) ([]string, error) {

pkg/collection/imagetxt/imagetxt.go

@@ -0,0 +1,232 @@
+package imagetxt
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/rancherfederal/hauler/pkg/log"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	artifact "github.com/rancherfederal/ocil/pkg/artifacts"
+	"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
+	"github.com/rancherfederal/ocil/pkg/artifacts/image"
+)
+
+type ImageTxt struct {
+	Ref            string
+	IncludeSources map[string]bool
+	ExcludeSources map[string]bool
+
+	lock     *sync.Mutex
+	client   *getter.Client
+	computed bool
+	contents map[string]artifact.OCI
+}
+
+var _ artifact.OCICollection = (*ImageTxt)(nil)
+
+type Option interface {
+	Apply(*ImageTxt) error
+}
+
+type withIncludeSources []string
+
+func (o withIncludeSources) Apply(it *ImageTxt) error {
+	if it.IncludeSources == nil {
+		it.IncludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.IncludeSources[s] = true
+	}
+	return nil
+}
+
+func WithIncludeSources(include ...string) Option {
+	return withIncludeSources(include)
+}
+
+type withExcludeSources []string
+
+func (o withExcludeSources) Apply(it *ImageTxt) error {
+	if it.ExcludeSources == nil {
+		it.ExcludeSources = make(map[string]bool)
+	}
+	for _, s := range o {
+		it.ExcludeSources[s] = true
+	}
+	return nil
+}
+
+func WithExcludeSources(exclude ...string) Option {
+	return withExcludeSources(exclude)
+}
+
+func New(ref string, opts ...Option) (*ImageTxt, error) {
+	it := &ImageTxt{
+		Ref: ref,
+
+		client: getter.NewClient(getter.ClientOptions{}),
+		lock:   &sync.Mutex{},
+	}
+
+	for i, o := range opts {
+		if err := o.Apply(it); err != nil {
+			return nil, fmt.Errorf("invalid option %d: %v", i, err)
+		}
+	}
+
+	return it, nil
+}
+
+func (it *ImageTxt) Contents() (map[string]artifact.OCI, error) {
+	it.lock.Lock()
+	defer it.lock.Unlock()
+	if !it.computed {
+		if err := it.compute(); err != nil {
+			return nil, fmt.Errorf("compute OCI layout: %v", err)
+		}
+		it.computed = true
+	}
+	return it.contents, nil
+}
+
+func (it *ImageTxt) compute() error {
+	// TODO - pass in logger from context
+	l := log.NewLogger(os.Stdout)
+
+	it.contents = make(map[string]artifact.OCI)
+
+	ctx := context.TODO()
+
+	rc, err := it.client.ContentFrom(ctx, it.Ref)
+	if err != nil {
+		return fmt.Errorf("fetch image.txt ref %s: %w", it.Ref, err)
+	}
+	defer rc.Close()
+
+	entries, err := splitImagesTxt(rc)
+	if err != nil {
+		return fmt.Errorf("parse image.txt ref %s: %v", it.Ref, err)
+	}
+
+	foundSources := make(map[string]bool)
+	for _, e := range entries {
+		for s := range e.Sources {
+			foundSources[s] = true
+		}
+	}
+
+	var pullAll bool
+	targetSources := make(map[string]bool)
+
+	if len(foundSources) == 0 || (len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0) {
+		// pull all found images
+		pullAll = true
+
+		if len(foundSources) == 0 {
+			l.Infof("image txt file appears to have no sources; pulling all found images")
+			if len(it.IncludeSources) != 0 || len(it.ExcludeSources) != 0 {
+				l.Warnf("ImageTxt provided include or exclude sources; ignoring")
+			}
+		} else if len(it.IncludeSources) == 0 && len(it.ExcludeSources) == 0 {
+			l.Infof("image-sources txt file not filtered; pulling all found images")
+		}
+	} else {
+		// determine sources to pull
+		if len(it.IncludeSources) != 0 && len(it.ExcludeSources) != 0 {
+			l.Warnf("ImageTxt provided include and exclude sources; using only include sources")
+		}
+
+		if len(it.IncludeSources) != 0 {
+			targetSources = it.IncludeSources
+		} else {
+			for s := range foundSources {
+				targetSources[s] = true
+			}
+			for s := range it.ExcludeSources {
+				delete(targetSources, s)
+			}
+		}
+		var targetSourcesArr []string
+		for s := range targetSources {
+			targetSourcesArr = append(targetSourcesArr, s)
+		}
+		l.Infof("pulling images covering sources %s", strings.Join(targetSourcesArr, ", "))
+	}
+
+	for _, e := range entries {
+		var matchesSourceFilter bool
+		if pullAll {
+			l.Infof("pulling image %s", e.Reference)
+		} else {
+			for s := range e.Sources {
+				if targetSources[s] {
+					matchesSourceFilter = true
+					l.Infof("pulling image %s (matched source %s)", e.Reference, s)
+					break
+				}
+			}
+		}
+
+		if pullAll || matchesSourceFilter {
+			curImage, err := image.NewImage(e.Reference.String())
+			if err != nil {
+				return fmt.Errorf("pull image %s: %v", e.Reference, err)
+			}
+			it.contents[e.Reference.String()] = curImage
+		}
+	}
+
+	return nil
+}
+
+type imageTxtEntry struct {
+	Reference name.Reference
+	Sources   map[string]bool
+}
+
+func splitImagesTxt(r io.Reader) ([]imageTxtEntry, error) {
+	var entries []imageTxtEntry
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		curEntry := imageTxtEntry{
+			Sources: make(map[string]bool),
+		}
+
+		lineContent := scanner.Text()
+		if lineContent == "" || strings.HasPrefix(lineContent, "#") {
+			// skip past empty and commented lines
+			continue
+		}
+		splitContent := strings.Split(lineContent, " ")
+		if len(splitContent) > 2 {
+			return nil, fmt.Errorf(
+				"invalid image.txt format: must contain only an image reference and sources separated by space; invalid line: %q",
+				lineContent)
+		}
+
+		curRef, err := name.ParseReference(splitContent[0])
+		if err != nil {
+			return nil, fmt.Errorf("invalid reference %s: %v", splitContent[0], err)
+		}
+		curEntry.Reference = curRef
+
+		if len(splitContent) == 2 {
+			for _, source := range strings.Split(splitContent[1], ",") {
+				curEntry.Sources[source] = true
+			}
+		}
+
+		entries = append(entries, curEntry)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("scan contents: %v", err)
+	}
+
+	return entries, nil
+}

pkg/collection/imagetxt/imagetxt_test.go

@@ -0,0 +1,209 @@
+package imagetxt
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/rancherfederal/ocil/pkg/artifacts"
+	"github.com/rancherfederal/ocil/pkg/artifacts/image"
+)
+
+var (
+	ErrRefNotFound    = errors.New("ref not found")
+	ErrRefNotImage    = errors.New("ref is not image")
+	ErrExtraRefsFound = errors.New("extra refs found in contents")
+)
+
+var (
+	testServer *httptest.Server
+)
+
+func TestMain(m *testing.M) {
+	setup()
+	code := m.Run()
+	teardown()
+	os.Exit(code)
+}
+
+func setup() {
+	dir := http.Dir("./testdata/http/")
+	h := http.FileServer(dir)
+	testServer = httptest.NewServer(h)
+}
+
+func teardown() {
+	if testServer != nil {
+		testServer.Close()
+	}
+}
+
+type failKind string
+
+const (
+	failKindNew      = failKind("New")
+	failKindContents = failKind("Contents")
+)
+
+func checkError(checkedFailKind failKind) func(*testing.T, error, bool, failKind) {
+	return func(cet *testing.T, err error, testShouldFail bool, testFailKind failKind) {
+		if err != nil {
+			// if error should not have happened at all OR error should have happened
+			// at a different point, test failed
+			if !testShouldFail || testFailKind != checkedFailKind {
+				cet.Fatalf("unexpected error at %s: %v", checkedFailKind, err)
+			}
+			// test should fail at this point, test passed
+			return
+		}
+		// if no error occurred but error should have happened at this point, test
+		// failed
+		if testShouldFail && testFailKind == checkedFailKind {
+			cet.Fatalf("unexpected nil error at %s", checkedFailKind)
+		}
+	}
+}
+
+func TestImageTxtCollection(t *testing.T) {
+	type testEntry struct {
+		Name           string
+		Ref            string
+		IncludeSources []string
+		ExcludeSources []string
+		ExpectedImages []string
+		ShouldFail     bool
+		FailKind       failKind
+	}
+	tt := []testEntry{
+		{
+			Name: "http ref basic",
+			Ref:  fmt.Sprintf("%s/images-http.txt", testServer.URL),
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format pull all",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format include sources A",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			IncludeSources: []string{
+				"core", "rke",
+			},
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+			},
+		},
+		{
+			Name: "http ref sources format include sources B",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			IncludeSources: []string{
+				"nginx", "rancher", "cert-manager",
+			},
+			ExpectedImages: []string{
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "http ref sources format exclude sources A",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExcludeSources: []string{
+				"cert-manager",
+			},
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+			},
+		},
+		{
+			Name: "http ref sources format exclude sources B",
+			Ref:  fmt.Sprintf("%s/images-src-http.txt", testServer.URL),
+			ExcludeSources: []string{
+				"core",
+			},
+			ExpectedImages: []string{
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+		{
+			Name: "local file ref",
+			Ref:  "./testdata/images-file.txt",
+			ExpectedImages: []string{
+				"busybox",
+				"nginx:1.19",
+				"rancher/hyperkube:v1.21.7-rancher1",
+				"docker.io/rancher/klipper-lb:v0.3.4",
+				"quay.io/jetstack/cert-manager-controller:v1.6.1",
+			},
+		},
+	}
+
+	checkErrorNew := checkError(failKindNew)
+	checkErrorContents := checkError(failKindContents)
+
+	for _, curTest := range tt {
+		t.Run(curTest.Name, func(innerT *testing.T) {
+			curImageTxt, err := New(curTest.Ref,
+				WithIncludeSources(curTest.IncludeSources...),
+				WithExcludeSources(curTest.ExcludeSources...),
+			)
+			checkErrorNew(innerT, err, curTest.ShouldFail, curTest.FailKind)
+
+			ociContents, err := curImageTxt.Contents()
+			checkErrorContents(innerT, err, curTest.ShouldFail, curTest.FailKind)
+
+			if err := checkImages(ociContents, curTest.ExpectedImages); err != nil {
+				innerT.Fatal(err)
+			}
+		})
+	}
+}
+
+func checkImages(content map[string]artifacts.OCI, refs []string) error {
+	contentCopy := make(map[string]artifacts.OCI, len(content))
+	for k, v := range content {
+		contentCopy[k] = v
+	}
+	for _, ref := range refs {
+		target, ok := content[ref]
+		if !ok {
+			return fmt.Errorf("ref %s: %w", ref, ErrRefNotFound)
+		}
+		if _, ok := target.(*image.Image); !ok {
+			return fmt.Errorf("got underlying type %T: %w", target, ErrRefNotImage)
+		}
+		delete(contentCopy, ref)
+	}
+
+	if len(contentCopy) != 0 {
+		return ErrExtraRefsFound
+	}
+
+	return nil
+}

pkg/collection/imagetxt/testdata/http/images-http.txt

@@ -0,0 +1,5 @@
+busybox
+nginx:1.19
+rancher/hyperkube:v1.21.7-rancher1
+docker.io/rancher/klipper-lb:v0.3.4
+quay.io/jetstack/cert-manager-controller:v1.6.1

pkg/collection/imagetxt/testdata/http/images-src-http.txt

@@ -0,0 +1,5 @@
+busybox core
+nginx:1.19 core,nginx
+rancher/hyperkube:v1.21.7-rancher1 rancher,rke
+docker.io/rancher/klipper-lb:v0.3.4 rancher,k3s
+quay.io/jetstack/cert-manager-controller:v1.6.1 cert-manager

pkg/collection/imagetxt/testdata/images-file.txt

@@ -0,0 +1,5 @@
+busybox
+nginx:1.19
+rancher/hyperkube:v1.21.7-rancher1
+docker.io/rancher/klipper-lb:v0.3.4
+quay.io/jetstack/cert-manager-controller:v1.6.1

pkg/collection/k3s/k3s.go

@@ -10,14 +10,17 @@ import (
 	"path"
 	"strings"
 
-	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/rancherfederal/ocil/pkg/artifacts"
+	"github.com/rancherfederal/ocil/pkg/artifacts/image"
 
-	"github.com/rancherfederal/hauler/pkg/artifact"
-	"github.com/rancherfederal/hauler/pkg/content/file"
-	"github.com/rancherfederal/hauler/pkg/content/image"
+	"github.com/rancherfederal/ocil/pkg/artifacts/file"
+
+	"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
+
+	"github.com/rancherfederal/hauler/pkg/reference"
 )
 
-var _ artifact.Collection = (*k3s)(nil)
+var _ artifacts.OCICollection = (*k3s)(nil)
 
 const (
 	releaseUrl   = "https://github.com/k3s-io/k3s/releases/download"
@@ -37,18 +40,19 @@ type k3s struct {
 	arch    string
 
 	computed bool
-	contents map[name.Reference]artifact.OCI
+	contents map[string]artifacts.OCI
 	channels map[string]string
+	client   *getter.Client
 }
 
-func NewK3s(version string) (artifact.Collection, error) {
+func NewK3s(version string) (artifacts.OCICollection, error) {
 	return &k3s{
 		version:  version,
-		contents: make(map[name.Reference]artifact.OCI),
+		contents: make(map[string]artifacts.OCI),
 	}, nil
 }
 
-func (k *k3s) Contents() (map[name.Reference]artifact.OCI, error) {
+func (k *k3s) Contents() (map[string]artifacts.OCI, error) {
 	if err := k.compute(); err != nil {
 		return nil, err
 	}
@@ -94,31 +98,18 @@ func (k *k3s) executable() error {
 		return ErrExecutableNotfound
 	}
 
-	f, err := file.NewFile(fref, "k3s")
-	if err != nil {
-		return err
-	}
-
-	ref, err := name.ParseReference("hauler/k3s", name.WithDefaultTag(k.dnsCompliantVersion()), name.WithDefaultRegistry(""))
-	if err != nil {
-		return err
-	}
+	f := file.NewFile(fref)
 
+	ref := fmt.Sprintf("%s/k3s:%s", reference.DefaultNamespace, k.dnsCompliantVersion())
 	k.contents[ref] = f
 	return nil
 }
 
 func (k *k3s) bootstrap() error {
-	f, err := file.NewFile(bootstrapUrl, "get-k3s.io")
-	if err != nil {
-		return err
-	}
-
-	ref, err := name.ParseReference("hauler/get-k3s.io", name.WithDefaultRegistry(""), name.WithDefaultTag("latest"))
-	if err != nil {
-		return err
-	}
+	c := getter.NewClient(getter.ClientOptions{NameOverride: "k3s-init.sh"})
+	f := file.NewFile(bootstrapUrl, file.WithClient(c))
 
+	ref := fmt.Sprintf("%s/k3s-init.sh:%s", reference.DefaultNamespace, reference.DefaultTag)
 	k.contents[ref] = f
 	return nil
 }
@@ -135,16 +126,12 @@ func (k *k3s) images() error {
 	scanner := bufio.NewScanner(resp.Body)
 	for scanner.Scan() {
 		reference := scanner.Text()
-		ref, err := name.ParseReference(reference)
-		if err != nil {
-			return err
-		}
 		o, err := image.NewImage(reference)
 		if err != nil {
 			return err
 		}
 
-		k.contents[ref] = o
+		k.contents[reference] = o
 	}
 	return nil
 }

pkg/collection/k3s/k3s_test.go

@@ -1,71 +0,0 @@
-package k3s
-
-import (
-	"context"
-	"os"
-	"testing"
-
-	"github.com/rancherfederal/hauler/pkg/artifact"
-	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
-)
-
-// TODO: This is not at all a good test, we really just need to test the added collections functionality (like image scanning)
-func TestNewK3s(t *testing.T) {
-	ctx := context.Background()
-	l := log.NewLogger(os.Stdout)
-	ctx = l.WithContext(ctx)
-
-	tmpdir, err := os.MkdirTemp("", "hauler")
-	if err != nil {
-		t.Error(err)
-	}
-	defer os.Remove(tmpdir)
-
-	s := store.NewStore(ctx, tmpdir)
-	s.Open()
-	defer s.Close()
-
-	type args struct {
-		version string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    artifact.Collection
-		wantErr bool
-	}{
-		{
-			name: "should work",
-			args: args{
-				version: "v1.22.2+k3s2",
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewK3s(tt.args.version)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("NewK3s() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-
-			c, err := got.Contents()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			for r, o := range c {
-				if _, err := s.AddArtifact(ctx, o, r); err != nil {
-					t.Fatal(err)
-				}
-			}
-
-			// if !reflect.DeepEqual(got, tt.want) {
-			// 	t.Errorf("NewK3s() got = %v, want %v", got, tt.want)
-			// }
-		})
-	}
-}

pkg/content/chart/chart.go

@@ -1,9 +1,12 @@
 package chart
 
 import (
+	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -11,42 +14,55 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/partial"
 	gtypes "github.com/google/go-containerregistry/pkg/v1/types"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/rancherfederal/ocil/pkg/artifacts"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/cli"
 
-	"github.com/rancherfederal/hauler/pkg/artifact"
-	"github.com/rancherfederal/hauler/pkg/artifact/local"
-	"github.com/rancherfederal/hauler/pkg/artifact/types"
+	"github.com/rancherfederal/ocil/pkg/layer"
+
+	"github.com/rancherfederal/ocil/pkg/consts"
 )
 
-var _ artifact.OCI = (*Chart)(nil)
+var _ artifacts.OCI = (*Chart)(nil)
 
+// Chart implements the  OCI interface for Chart API objects. API spec values are
+// stored into the Repo, Name, and Version fields.
 type Chart struct {
-	path string
-
+	path        string
 	annotations map[string]string
 }
 
-func NewChart(name, repo, version string) (*Chart, error) {
+// NewChart is a helper method that returns NewLocalChart or NewRemoteChart depending on v1alpha1.Chart contents
+func NewChart(name string, opts *action.ChartPathOptions) (*Chart, error) {
 	cpo := action.ChartPathOptions{
-		RepoURL: repo,
-		Version: version,
+		RepoURL: opts.RepoURL,
+		Version: opts.Version,
+
+		CaFile:                opts.CaFile,
+		CertFile:              opts.CertFile,
+		KeyFile:               opts.KeyFile,
+		InsecureSkipTLSverify: opts.InsecureSkipTLSverify,
+		Keyring:               opts.Keyring,
+		Password:              opts.Password,
+		PassCredentialsAll:    opts.PassCredentialsAll,
+		Username:              opts.Username,
+		Verify:                opts.Verify,
 	}
 
-	cp, err := cpo.LocateChart(name, cli.New())
+	chartPath, err := cpo.LocateChart(name, cli.New())
 	if err != nil {
 		return nil, err
 	}
 
 	return &Chart{
-		path: cp,
-	}, nil
+		path: chartPath,
+	}, err
 }
 
 func (h *Chart) MediaType() string {
-	return types.OCIManifestSchema1
+	return consts.OCIManifestSchema1
 }
 
 func (h *Chart) Manifest() (*gv1.Manifest, error) {
@@ -94,23 +110,18 @@ func (h *Chart) configDescriptor() (gv1.Descriptor, error) {
 	}
 
 	return gv1.Descriptor{
-		MediaType: types.ChartConfigMediaType,
+		MediaType: consts.ChartConfigMediaType,
 		Size:      size,
 		Digest:    hash,
 	}, nil
 }
 
 func (h *Chart) Load() (*chart.Chart, error) {
-	rc, err := chartOpener(h.path)()
-	if err != nil {
-		return nil, err
-	}
-	defer rc.Close()
-	return loader.LoadArchive(rc)
+	return loader.Load(h.path)
 }
 
 func (h *Chart) Layers() ([]gv1.Layer, error) {
-	chartDataLayer, err := h.chartDataLayer()
+	chartDataLayer, err := h.chartData()
 	if err != nil {
 		return nil, err
 	}
@@ -125,17 +136,83 @@ func (h *Chart) RawChartData() ([]byte, error) {
 	return os.ReadFile(h.path)
 }
 
-func (h *Chart) chartDataLayer() (gv1.Layer, error) {
+// chartData loads the chart contents into memory and returns a NopCloser for the contents
+// 		Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB
+func (h *Chart) chartData() (gv1.Layer, error) {
+	info, err := os.Stat(h.path)
+	if err != nil {
+		return nil, err
+	}
+
+	var chartdata []byte
+	if info.IsDir() {
+		buf := &bytes.Buffer{}
+		gw := gzip.NewWriter(buf)
+		tw := tar.NewWriter(gw)
+
+		if err := filepath.WalkDir(h.path, func(path string, d fs.DirEntry, err error) error {
+			fi, err := d.Info()
+			if err != nil {
+				return err
+			}
+
+			header, err := tar.FileInfoHeader(fi, fi.Name())
+			if err != nil {
+				return err
+			}
+
+			rel, err := filepath.Rel(filepath.Dir(h.path), path)
+			if err != nil {
+				return err
+			}
+			header.Name = rel
+
+			if err := tw.WriteHeader(header); err != nil {
+				return err
+			}
+
+			if !d.IsDir() {
+				data, err := os.Open(path)
+				if err != nil {
+					return err
+				}
+				if _, err := io.Copy(tw, data); err != nil {
+					return err
+				}
+			}
+
+			return nil
+		}); err != nil {
+			return nil, err
+		}
+
+		if err := tw.Close(); err != nil {
+			return nil, err
+		}
+		if err := gw.Close(); err != nil {
+			return nil, err
+		}
+		chartdata = buf.Bytes()
+
+	} else {
+		data, err := os.ReadFile(h.path)
+		if err != nil {
+			return nil, err
+		}
+		chartdata = data
+	}
+
 	annotations := make(map[string]string)
 	annotations[ocispec.AnnotationTitle] = filepath.Base(h.path)
 
-	return local.LayerFromOpener(chartOpener(h.path),
-		local.WithMediaType(types.ChartLayerMediaType),
-		local.WithAnnotations(annotations))
-}
-
-func chartOpener(path string) local.Opener {
-	return func() (io.ReadCloser, error) {
-		return os.Open(path)
+	opener := func() layer.Opener {
+		return func() (io.ReadCloser, error) {
+			return io.NopCloser(bytes.NewBuffer(chartdata)), nil
+		}
 	}
+	chartDataLayer, err := layer.FromOpener(opener(),
+		layer.WithMediaType(consts.ChartLayerMediaType),
+		layer.WithAnnotations(annotations))
+
+	return chartDataLayer, err
 }

pkg/content/chart/chart_test.go

@@ -1,72 +1,117 @@
 package chart_test
 
 import (
-	"context"
 	"os"
-	"path"
+	"reflect"
 	"testing"
 
-	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/mholt/archiver/v3"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"helm.sh/helm/v3/pkg/action"
+
+	"github.com/rancherfederal/ocil/pkg/consts"
 
-	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
 	"github.com/rancherfederal/hauler/pkg/content/chart"
-	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
-func TestChart_Copy(t *testing.T) {
-	ctx := context.Background()
-	l := log.NewLogger(os.Stdout)
-	ctx = l.WithContext(ctx)
+var (
+	chartpath = "../../../testdata/podinfo-6.0.3.tgz"
+)
 
+func TestNewChart(t *testing.T) {
 	tmpdir, err := os.MkdirTemp("", "hauler")
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
-	defer os.Remove(tmpdir)
+	defer os.RemoveAll(tmpdir)
 
-	s := store.NewStore(ctx, tmpdir)
-	s.Open()
-	defer s.Close()
+	if err := archiver.Unarchive(chartpath, tmpdir); err != nil {
+		t.Fatal(err)
+	}
 
 	type args struct {
-		ctx      context.Context
-		registry string
+		name string
+		opts *action.ChartPathOptions
 	}
 	tests := []struct {
 		name    string
-		cfg     v1alpha1.Chart
 		args    args
+		want    v1.Descriptor
 		wantErr bool
 	}{
-		// TODO: This test isn't self-contained
 		{
-			name: "should work with unversioned chart",
-			cfg: v1alpha1.Chart{
-				Name:    "loki",
-				RepoURL: "https://grafana.github.io/helm-charts",
-			},
+			name: "should create from a chart archive",
 			args: args{
-				ctx:      ctx,
-				registry: s.Registry(),
+				name: chartpath,
+				opts: &action.ChartPathOptions{},
+			},
+			want: v1.Descriptor{
+				MediaType: consts.ChartLayerMediaType,
+				Size:      13524,
+				Digest: v1.Hash{
+					Algorithm: "sha256",
+					Hex:       "e30b95a08787de69ffdad3c232d65cfb131b5b50c6fd44295f48a078fceaa44e",
+				},
+				Annotations: map[string]string{
+					ocispec.AnnotationTitle: "podinfo-6.0.3.tgz",
+				},
+			},
+			wantErr: false,
+		},
+		// TODO: This isn't matching digests b/c of file timestamps not being respected
+		// {
+		// 	name:    "should create from a chart directory",
+		// 	args:    args{
+		// 		path: filepath.Join(tmpdir, "podinfo"),
+		// 	},
+		// 	want:    want,
+		// 	wantErr: false,
+		// },
+		{
+			// TODO: Use a mock helm server
+			name: "should fetch a remote chart",
+			args: args{
+				name: "ingress-nginx",
+				opts: &action.ChartPathOptions{RepoURL: "https://kubernetes.github.io/ingress-nginx", Version: "4.0.16"},
+			},
+			want: v1.Descriptor{
+				MediaType: consts.ChartLayerMediaType,
+				Size:      38591,
+				Digest: v1.Hash{
+					Algorithm: "sha256",
+					Hex:       "b0ea91f7febc6708ad9971871d2de6e8feb2072110c3add6dd7082d90753caa2",
+				},
+				Annotations: map[string]string{
+					ocispec.AnnotationTitle: "ingress-nginx-4.0.16.tgz",
+				},
 			},
 			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c, err := chart.NewChart(tt.cfg.Name, tt.cfg.RepoURL, tt.cfg.Version)
-			if err != nil {
-				t.Fatal(err)
-			}
-			ref, err := name.ParseReference(path.Join("hauler", tt.cfg.Name))
-			if err != nil {
-				t.Fatal(err)
+			got, err := chart.NewChart(tt.args.name, tt.args.opts)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("NewLocalChart() error = %v, wantErr %v", err, tt.wantErr)
+				return
 			}
 
-			if _, err := s.AddArtifact(ctx, c, ref); (err != nil) != tt.wantErr {
+			m, err := got.Manifest()
+			if err != nil {
 				t.Error(err)
 			}
+
+			// TODO: This changes when we support provenance files
+			if len(m.Layers) > 1 {
+				t.Errorf("Expected 1 layer for chart, got %d", len(m.Layers))
+			}
+			desc := m.Layers[0]
+
+			if !reflect.DeepEqual(desc, tt.want) {
+				t.Errorf("got: %v\nwant: %v", desc, tt.want)
+				return
+			}
 		})
 	}
 }

pkg/content/content.go

@@ -11,7 +11,7 @@ import (
 )
 
 func Load(data []byte) (schema.ObjectKind, error) {
-	var tm *metav1.TypeMeta
+	var tm metav1.TypeMeta
 	if err := yaml.Unmarshal(data, &tm); err != nil {
 		return nil, err
 	}
@@ -20,5 +20,5 @@ func Load(data []byte) (schema.ObjectKind, error) {
 		return nil, fmt.Errorf("unrecognized content/collection type: %s", tm.GroupVersionKind().String())
 	}
 
-	return tm, nil
+	return &tm, nil
 }

pkg/content/file/config.go

@@ -1,82 +0,0 @@
-package file
-
-import (
-	"bytes"
-	"encoding/json"
-
-	gv1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/partial"
-	gtypes "github.com/google/go-containerregistry/pkg/v1/types"
-
-	"github.com/rancherfederal/hauler/pkg/artifact/types"
-)
-
-var _ partial.Describable = (*config)(nil)
-
-type config struct {
-	Reference   string            `json:"ref"`  // Reference is the reference from where the file was sourced
-	Name        string            `json:"name"` // Name is the files name on disk
-	Annotations map[string]string `json:"annotations,omitempty"`
-	URLs        []string          `json:"urls,omitempty"`
-
-	computed bool
-	size     int64
-	hash     gv1.Hash
-}
-
-func (c config) Descriptor() (gv1.Descriptor, error) {
-	if err := c.compute(); err != nil {
-		return gv1.Descriptor{}, err
-	}
-
-	return gv1.Descriptor{
-		MediaType:   types.FileConfigMediaType,
-		Size:        c.size,
-		Digest:      c.hash,
-		URLs:        c.URLs,
-		Annotations: c.Annotations,
-		// Platform:    nil,
-	}, nil
-}
-
-func (c config) Digest() (gv1.Hash, error) {
-	if err := c.compute(); err != nil {
-		return gv1.Hash{}, err
-	}
-	return c.hash, nil
-}
-
-func (c config) MediaType() (gtypes.MediaType, error) {
-	return types.FileConfigMediaType, nil
-}
-
-func (c config) Size() (int64, error) {
-	if err := c.compute(); err != nil {
-		return 0, err
-	}
-	return c.size, nil
-}
-
-func (c *config) Raw() ([]byte, error) {
-	return json.Marshal(c)
-}
-
-func (c *config) compute() error {
-	if c.computed {
-		return nil
-	}
-
-	data, err := c.Raw()
-	if err != nil {
-		return err
-	}
-
-	h, size, err := gv1.SHA256(bytes.NewBuffer(data))
-	if err != nil {
-		return err
-	}
-
-	c.size = size
-	c.hash = h
-	return nil
-}

pkg/content/file/file.go

@@ -1,107 +0,0 @@
-package file
-
-import (
-	"io"
-	"net/http"
-	"os"
-	"strings"
-
-	gv1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/partial"
-	gtypes "github.com/google/go-containerregistry/pkg/v1/types"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-
-	"github.com/rancherfederal/hauler/pkg/artifact"
-	"github.com/rancherfederal/hauler/pkg/artifact/local"
-	"github.com/rancherfederal/hauler/pkg/artifact/types"
-)
-
-var _ artifact.OCI = (*file)(nil)
-
-type file struct {
-	blob    gv1.Layer
-	config  config
-	blobMap map[gv1.Hash]gv1.Layer
-
-	annotations map[string]string
-}
-
-func NewFile(ref string, filename string) (*file, error) {
-	var getter local.Opener
-	if strings.HasPrefix(ref, "http") || strings.HasPrefix(ref, "https") {
-		getter = remoteOpener(ref)
-	} else {
-		getter = localOpener(ref)
-	}
-
-	annotations := make(map[string]string)
-	annotations[ocispec.AnnotationTitle] = filename // For oras FileStore to recognize
-	annotations[ocispec.AnnotationSource] = ref
-
-	blob, err := local.LayerFromOpener(getter,
-		local.WithMediaType(types.FileLayerMediaType),
-		local.WithAnnotations(annotations))
-	if err != nil {
-		return nil, err
-	}
-
-	f := &file{
-		blob: blob,
-		config: config{
-			Reference: ref,
-			Name:      filename,
-		},
-	}
-	return f, nil
-}
-
-func (f *file) MediaType() string {
-	return types.OCIManifestSchema1
-}
-
-func (f *file) RawConfig() ([]byte, error) {
-	return f.config.Raw()
-}
-
-func (f *file) Layers() ([]gv1.Layer, error) {
-	var layers []gv1.Layer
-	layers = append(layers, f.blob)
-	return layers, nil
-}
-
-func (f *file) Manifest() (*gv1.Manifest, error) {
-	desc, err := partial.Descriptor(f.blob)
-	if err != nil {
-		return nil, err
-	}
-	layerDescs := []gv1.Descriptor{*desc}
-
-	cfgDesc, err := f.config.Descriptor()
-	if err != nil {
-		return nil, err
-	}
-
-	return &gv1.Manifest{
-		SchemaVersion: 2,
-		MediaType:     gtypes.MediaType(f.MediaType()),
-		Config:        cfgDesc,
-		Layers:        layerDescs,
-		Annotations:   f.annotations,
-	}, nil
-}
-
-func localOpener(path string) local.Opener {
-	return func() (io.ReadCloser, error) {
-		return os.Open(path)
-	}
-}
-
-func remoteOpener(url string) local.Opener {
-	return func() (io.ReadCloser, error) {
-		resp, err := http.Get(url)
-		if err != nil {
-			return nil, err
-		}
-		return resp.Body, nil
-	}
-}

pkg/content/file/file_test.go

@@ -1,188 +0,0 @@
-package file_test
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"reflect"
-	"testing"
-
-	"github.com/google/go-containerregistry/pkg/name"
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-
-	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
-	"github.com/rancherfederal/hauler/pkg/artifact/types"
-	"github.com/rancherfederal/hauler/pkg/content/file"
-	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
-)
-
-func TestFile_Copy(t *testing.T) {
-	ctx := context.Background()
-	l := log.NewLogger(os.Stdout)
-	ctx = l.WithContext(ctx)
-
-	tmpdir, err := os.MkdirTemp("", "hauler")
-	if err != nil {
-		t.Error(err)
-	}
-	defer os.Remove(tmpdir)
-
-	// Make a temp file
-	f, err := os.CreateTemp(tmpdir, "tmp")
-	f.Write([]byte("content"))
-	defer f.Close()
-
-	fs := newTestFileServer(tmpdir)
-	fs.Start()
-	defer fs.Stop()
-
-	s := store.NewStore(ctx, tmpdir)
-	s.Open()
-	defer s.Close()
-
-	type args struct {
-		ctx      context.Context
-		registry string
-	}
-
-	tests := []struct {
-		name    string
-		cfg     v1alpha1.File
-		args    args
-		wantErr bool
-	}{
-		{
-			name: "should copy a local file successfully without an explicit name",
-			cfg: v1alpha1.File{
-				Ref:  f.Name(),
-				Name: filepath.Base(f.Name()),
-			},
-			args: args{
-				ctx: ctx,
-			},
-		},
-		{
-			name: "should copy a local file successfully with an explicit name",
-			cfg: v1alpha1.File{
-				Ref:  f.Name(),
-				Name: "my-other-file",
-			},
-			args: args{
-				ctx: ctx,
-			},
-		},
-		{
-			name: "should fail to copy a local file successfully with a malformed explicit name",
-			cfg: v1alpha1.File{
-				Ref:  f.Name(),
-				Name: "my!invalid~@file",
-			},
-			args: args{
-				ctx: ctx,
-			},
-			wantErr: true,
-		},
-		{
-			name: "should copy a remote file successfully without an explicit name",
-			cfg: v1alpha1.File{
-				Ref: fmt.Sprintf("%s/%s", fs.server.URL, filepath.Base(f.Name())),
-			},
-			args: args{
-				ctx: ctx,
-			},
-		},
-		{
-			name: "should copy a remote file successfully with an explicit name",
-			cfg: v1alpha1.File{
-				Ref:  fmt.Sprintf("%s/%s", fs.server.URL, filepath.Base(f.Name())),
-				Name: "my-other-file",
-			},
-			args: args{
-				ctx: ctx,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f, err := file.NewFile(tt.cfg.Ref, tt.cfg.Name)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			ref, err := name.ParseReference("myfile")
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			_, err = s.AddArtifact(ctx, f, ref)
-			if (err != nil) != tt.wantErr {
-				t.Error(err)
-			}
-
-			// if err := validate(tt.cfg.Ref, tt.cfg.Name, m); err != nil {
-			// 	t.Error(err)
-			// }
-		})
-	}
-}
-
-type testFileServer struct {
-	server *httptest.Server
-}
-
-func newTestFileServer(path string) *testFileServer {
-	s := httptest.NewUnstartedServer(http.FileServer(http.Dir(path)))
-	return &testFileServer{server: s}
-}
-
-func (s *testFileServer) Start() *httptest.Server {
-	s.server.Start()
-	return s.server
-}
-
-func (s *testFileServer) Stop() {
-	s.server.Close()
-}
-
-// validate ensure
-func validate(ref string, name string, got *v1.Manifest) error {
-	data, err := os.ReadFile(ref)
-	if err != nil {
-		return err
-	}
-
-	d := digest.FromBytes(data)
-
-	annotations := make(map[string]string)
-	annotations[ocispec.AnnotationTitle] = name
-	annotations[ocispec.AnnotationSource] = ref
-
-	want := &v1.Manifest{
-		SchemaVersion: 2,
-		MediaType:     types.OCIManifestSchema1,
-		Config:        v1.Descriptor{},
-		Layers: []v1.Descriptor{
-			{
-				MediaType: types.FileLayerMediaType,
-				Size:      int64(len(data)),
-				Digest: v1.Hash{
-					Algorithm: d.Algorithm().String(),
-					Hex:       d.Hex(),
-				},
-				Annotations: annotations,
-			},
-		},
-		Annotations: nil,
-	}
-
-	if !reflect.DeepEqual(want.Layers, got.Layers) {
-		return fmt.Errorf("want = (%v) | got = (%v)", want, got)
-	}
-	return nil
-}

pkg/content/image/image.go

@@ -1,43 +0,0 @@
-package image
-
-import (
-	"github.com/google/go-containerregistry/pkg/name"
-	gv1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
-
-	"github.com/rancherfederal/hauler/pkg/artifact"
-)
-
-var _ artifact.OCI = (*image)(nil)
-
-func (i *image) MediaType() string {
-	mt, err := i.Image.MediaType()
-	if err != nil {
-		return ""
-	}
-	return string(mt)
-}
-
-func (i *image) RawConfig() ([]byte, error) {
-	return i.RawConfigFile()
-}
-
-type image struct {
-	gv1.Image
-}
-
-func NewImage(ref string) (*image, error) {
-	r, err := name.ParseReference(ref)
-	if err != nil {
-		return nil, err
-	}
-
-	img, err := remote.Image(r)
-	if err != nil {
-		return nil, err
-	}
-
-	return &image{
-		Image: img,
-	}, nil
-}

pkg/content/image/image_test.go

@@ -1,99 +0,0 @@
-package image_test
-
-import (
-	"context"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/google/go-containerregistry/pkg/name"
-
-	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
-	"github.com/rancherfederal/hauler/pkg/content/image"
-	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
-)
-
-func TestImage_Copy(t *testing.T) {
-	ctx := context.Background()
-	l := log.NewLogger(os.Stdout)
-	ctx = l.WithContext(ctx)
-
-	tmpdir, err := os.MkdirTemp("", "hauler")
-	if err != nil {
-		t.Error(err)
-	}
-	defer os.Remove(tmpdir)
-
-	s := store.NewStore(ctx, tmpdir)
-	s.Open()
-	defer s.Close()
-
-	type args struct {
-		ctx      context.Context
-		registry string
-	}
-	tests := []struct {
-		name    string
-		cfg     v1alpha1.Image
-		args    args
-		wantErr bool
-	}{
-		// TODO: These mostly test functionality we're not responsible for (go-containerregistry), refactor these to only stuff we care about
-		{
-			name: "should work with tagged image",
-			cfg: v1alpha1.Image{
-				Ref: "busybox:1.34.1",
-			},
-			args: args{
-				ctx: ctx,
-				// registry: s.Registry(),
-			},
-			wantErr: false,
-		},
-		{
-			name: "should work with digest image",
-			cfg: v1alpha1.Image{
-				Ref: "busybox@sha256:6066ca124f8c2686b7ae71aa1d6583b28c6dc3df3bdc386f2c89b92162c597d9",
-			},
-			args: args{
-				ctx: ctx,
-				// registry: s.Registry(),
-			},
-			wantErr: false,
-		},
-		{
-			name: "should work with tagged image",
-			cfg: v1alpha1.Image{
-				Ref: "registry:2",
-			},
-			args: args{
-				ctx: ctx,
-				// registry: s.Registry(),
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			i, err := image.NewImage(tt.cfg.Ref)
-			if err != nil {
-				t.Error(err)
-			}
-
-			ref, err := name.ParseReference(path.Join("hauler", filepath.Base(tt.cfg.Ref)))
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if _, err := s.AddArtifact(ctx, i, ref); (err != nil) != tt.wantErr {
-				t.Error(err)
-			}
-
-			// if err := s.Add(tt.args.ctx, i, ref); (err != nil) != tt.wantErr {
-			// 	t.Errorf("Copy() error = %v, wantErr %v", err, tt.wantErr)
-			// }
-		})
-	}
-}

pkg/layout/artifact.go

@@ -1,146 +0,0 @@
-package layout
-
-import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"os"
-	"strings"
-
-	"github.com/google/go-containerregistry/pkg/name"
-	gv1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/layout"
-	gtypes "github.com/google/go-containerregistry/pkg/v1/types"
-	"github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"golang.org/x/sync/errgroup"
-
-	"github.com/rancherfederal/hauler/pkg/artifact"
-)
-
-// Path is a wrapper around layout.Path
-type Path struct {
-	layout.Path
-}
-
-// FromPath returns a new Path or creates one if one doesn't exist
-func FromPath(path string) (Path, error) {
-	p, err := layout.FromPath(path)
-	if os.IsNotExist(err) {
-		p, err = layout.Write(path, empty.Index)
-		if err != nil {
-			return Path{}, err
-		}
-	}
-	return Path{Path: p}, err
-}
-
-// WriteOci will write oci content (artifact.OCI) to the given Path
-func (l Path) WriteOci(o artifact.OCI, reference name.Reference) (ocispec.Descriptor, error) {
-	layers, err := o.Layers()
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	// Write layers concurrently
-	var g errgroup.Group
-	for _, layer := range layers {
-		layer := layer
-		g.Go(func() error {
-			return l.writeLayer(layer)
-		})
-	}
-	if err := g.Wait(); err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	// Write the config
-	cfgBlob, err := o.RawConfig()
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	if err = l.writeBlob(cfgBlob); err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	m, err := o.Manifest()
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	manifest, err := json.Marshal(m)
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	if err := l.writeBlob(manifest); err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	desc := ocispec.Descriptor{
-		MediaType: o.MediaType(),
-		Size:      int64(len(manifest)),
-		Digest:    digest.FromBytes(manifest),
-		Annotations: map[string]string{
-			ocispec.AnnotationRefName: reference.Name(),
-			ocispec.AnnotationTitle:   deregistry(reference).Name(),
-		},
-	}
-
-	if err := l.appendDescriptor(desc); err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	return desc, nil
-}
-
-// writeBlob differs from layer.WriteBlob in that it requires data instead
-func (l Path) writeBlob(data []byte) error {
-	h, _, err := gv1.SHA256(bytes.NewReader(data))
-	if err != nil {
-		return err
-	}
-
-	return l.WriteBlob(h, io.NopCloser(bytes.NewReader(data)))
-}
-
-// writeLayer is a verbatim reimplementation of layout.writeLayer
-func (l Path) writeLayer(layer gv1.Layer) error {
-	d, err := layer.Digest()
-	if err != nil {
-		return err
-	}
-
-	r, err := layer.Compressed()
-	if err != nil {
-		return err
-	}
-
-	return l.WriteBlob(d, r)
-}
-
-// appendDescriptor is a helper that translates a ocispec.Descriptor into a gv1.Descriptor
-func (l Path) appendDescriptor(desc ocispec.Descriptor) error {
-	gdesc := gv1.Descriptor{
-		MediaType: gtypes.MediaType(desc.MediaType),
-		Size:      desc.Size,
-		Digest: gv1.Hash{
-			Algorithm: desc.Digest.Algorithm().String(),
-			Hex:       desc.Digest.Hex(),
-		},
-		URLs:        desc.URLs,
-		Annotations: desc.Annotations,
-	}
-
-	return l.AppendDescriptor(gdesc)
-}
-
-// deregistry removes the registry content from a name.Reference
-func deregistry(ref name.Reference) name.Reference {
-	// No error checking b/c at this point we're already assumed to have a valid enough reference
-	dereg := strings.TrimLeft(strings.ReplaceAll(ref.Name(), ref.Context().RegistryStr(), ""), "/")
-	deref, _ := name.ParseReference(dereg, name.WithDefaultRegistry(""))
-	return deref
-}

pkg/layout/store.go

@@ -1,191 +0,0 @@
-package layout
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/content/local"
-	"github.com/containerd/containerd/remotes/docker"
-	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	orascontent "oras.land/oras-go/pkg/content"
-	"oras.land/oras-go/pkg/oras"
-
-	"github.com/rancherfederal/hauler/pkg/artifact/types"
-)
-
-// interface guards
-var (
-	_ content.Provider = (*OCIStore)(nil)
-	_ content.Ingester = (*OCIStore)(nil)
-)
-
-// OCIStore represents a content compatible store adhering by the oci-layout spec
-type OCIStore struct {
-	content.Store
-
-	root      string
-	index     *ocispec.Index
-	digestMap map[string]ocispec.Descriptor
-}
-
-// Copy placeholder until we migrate to oras 0.5
-// Will loop through each appropriately named index and copy the contents to the desired registry
-func Copy(ctx context.Context, s *OCIStore, registry string) error {
-	for _, desc := range s.index.Manifests {
-		manifestBlobPath, err := s.blobPath(desc.Digest)
-		if err != nil {
-			return err
-		}
-
-		manifestData, err := os.ReadFile(manifestBlobPath)
-		if err != nil {
-			return err
-		}
-
-		m, mdesc, err := loadManifest(manifestData)
-		if err != nil {
-			return err
-		}
-
-		refName, ok := desc.Annotations[ocispec.AnnotationRefName]
-		if !ok {
-			return fmt.Errorf("no name found to push image")
-		}
-
-		rref, err := RelocateReference(refName, registry)
-		if err != nil {
-			return err
-		}
-
-		resolver := docker.NewResolver(docker.ResolverOptions{})
-		_, err = oras.Push(ctx, resolver, rref.Name(), s, m.Layers,
-			oras.WithConfig(m.Config), oras.WithNameValidation(nil), oras.WithManifest(mdesc))
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// NewOCIStore will return a new OCIStore given a path to an oci-layout compatible directory
-func NewOCIStore(path string) (*OCIStore, error) {
-	fs, err := local.NewStore(path)
-	if err != nil {
-		return nil, err
-	}
-
-	store := &OCIStore{
-		Store: fs,
-
-		root: path,
-	}
-
-	if err := store.validateOCILayout(); err != nil {
-		return nil, err
-	}
-	if err := store.LoadIndex(); err != nil {
-		return nil, nil
-	}
-
-	return store, nil
-}
-
-// LoadIndex will load an oci-layout compatible directory
-func (s *OCIStore) LoadIndex() error {
-	path := filepath.Join(s.root, types.OCIImageIndexFile)
-	indexFile, err := os.Open(path)
-	if err != nil {
-		// TODO: Don't just bomb out?
-		return err
-	}
-	defer indexFile.Close()
-
-	if err := json.NewDecoder(indexFile).Decode(&s.index); err != nil {
-		return err
-	}
-
-	s.digestMap = make(map[string]ocispec.Descriptor)
-	for _, desc := range s.index.Manifests {
-		if name := desc.Annotations[ocispec.AnnotationRefName]; name != "" {
-			s.digestMap[name] = desc
-		}
-	}
-
-	return nil
-}
-
-func (s *OCIStore) validateOCILayout() error {
-	layoutFilePath := filepath.Join(s.root, ocispec.ImageLayoutFile)
-	layoutFile, err := os.Open(layoutFilePath)
-	if err != nil {
-		return err
-	}
-	defer layoutFile.Close()
-
-	var layout *ocispec.ImageLayout
-	if err := json.NewDecoder(layoutFile).Decode(&layout); err != nil {
-		return err
-	}
-
-	if layout.Version != ocispec.ImageLayoutVersion {
-		return orascontent.ErrUnsupportedVersion
-	}
-
-	return nil
-}
-
-func (s *OCIStore) blobPath(d digest.Digest) (string, error) {
-	if err := d.Validate(); err != nil {
-		return "", err
-	}
-
-	return filepath.Join(s.root, "blobs", d.Algorithm().String(), d.Hex()), nil
-}
-
-// manifest is a field wrapper around ocispec.Manifest that contains the mediaType field
-type manifest struct {
-	ocispec.Manifest `json:",inline"`
-
-	MediaType string `json:"mediaType"`
-}
-
-// loadManifest
-func loadManifest(data []byte) (ocispec.Manifest, ocispec.Descriptor, error) {
-	var m manifest
-	if err := json.Unmarshal(data, &m); err != nil {
-		return ocispec.Manifest{}, ocispec.Descriptor{}, err
-	}
-
-	desc := ocispec.Descriptor{
-		MediaType: m.MediaType,
-		Digest:    digest.FromBytes(data),
-		Size:      int64(len(data)),
-	}
-
-	return m.Manifest, desc, nil
-}
-
-// RelocateReference returns a name.Reference given a reference and registry
-func RelocateReference(reference string, registry string) (name.Reference, error) {
-	ref, err := name.ParseReference(reference)
-	if err != nil {
-		return nil, err
-	}
-
-	relocated, err := name.ParseReference(ref.Context().RepositoryStr(), name.WithDefaultRegistry(registry))
-	if err != nil {
-		return nil, err
-	}
-
-	if _, err := name.NewDigest(ref.Name()); err == nil {
-		return relocated.Context().Digest(ref.Identifier()), nil
-	}
-	return relocated.Context().Tag(ref.Identifier()), nil
-}

pkg/log/log.go

@@ -14,6 +14,7 @@ type Logger interface {
 	SetLevel(string)
 	With(Fields) *logger
 	WithContext(context.Context) context.Context
+
 	Errorf(string, ...interface{})
 	Infof(string, ...interface{})
 	Warnf(string, ...interface{})

pkg/reference/reference.go

@@ -0,0 +1,67 @@
+// Package reference provides general types to represent oci content within a registry or local oci layout
+// Grammar (stolen mostly from containerd's grammar)
+//
+// 	reference :=
+package reference
+
+import (
+	"strings"
+
+	gname "github.com/google/go-containerregistry/pkg/name"
+)
+
+const (
+	DefaultNamespace = "hauler"
+	DefaultTag       = "latest"
+)
+
+type Reference interface {
+	// FullName is the full name of the reference
+	FullName() string
+
+	// Name is the registryless name
+	Name() string
+}
+
+// NewTagged will create a new docker.NamedTagged given a path-component
+func NewTagged(n string, tag string) (gname.Reference, error) {
+	repo, err := Parse(n)
+	if err != nil {
+		return nil, err
+	}
+
+	return repo.Context().Tag(tag), nil
+}
+
+// Parse will parse a reference and return a name.Reference namespaced with DefaultNamespace if necessary
+func Parse(ref string) (gname.Reference, error) {
+	r, err := gname.ParseReference(ref, gname.WithDefaultRegistry(""), gname.WithDefaultTag(DefaultTag))
+	if err != nil {
+		return nil, err
+	}
+
+	if !strings.ContainsRune(r.String(), '/') {
+		ref = DefaultNamespace + "/" + r.String()
+		return gname.ParseReference(ref, gname.WithDefaultRegistry(""), gname.WithDefaultTag(DefaultTag))
+	}
+
+	return r, nil
+}
+
+// Relocate returns a name.Reference given a reference and registry
+func Relocate(reference string, registry string) (gname.Reference, error) {
+	ref, err := gname.ParseReference(reference)
+	if err != nil {
+		return nil, err
+	}
+
+	relocated, err := gname.ParseReference(ref.Context().RepositoryStr(), gname.WithDefaultRegistry(registry))
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := gname.NewDigest(ref.Name()); err == nil {
+		return relocated.Context().Digest(ref.Identifier()), nil
+	}
+	return relocated.Context().Tag(ref.Identifier()), nil
+}

pkg/reference/reference_test.go

@@ -0,0 +1,57 @@
+package reference_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/rancherfederal/hauler/pkg/reference"
+)
+
+func TestParse(t *testing.T) {
+	type args struct {
+		ref string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "Should add hauler namespace when doesn't exist",
+			args: args{
+				ref: "myfile",
+			},
+			want:    "hauler/myfile:latest",
+			wantErr: false,
+		},
+		{
+			name: "shouldn't modify namespaced reference",
+			args: args{
+				ref: "rancher/rancher:latest",
+			},
+			want:    "rancher/rancher:latest",
+			wantErr: false,
+		},
+		{
+			name: "Shouldn't modify canonical reference",
+			args: args{
+				ref: "index.docker.io/library/registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f",
+			},
+			want:    "index.docker.io/library/registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := reference.Parse(tt.args.ref)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got.Name(), tt.want) {
+				t.Errorf("Parse() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/store/add.go

@@ -1,146 +0,0 @@
-package store
-
-import (
-	"context"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/google/go-containerregistry/pkg/name"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-
-	"github.com/rancherfederal/hauler/pkg/artifact"
-	"github.com/rancherfederal/hauler/pkg/cache"
-	"github.com/rancherfederal/hauler/pkg/layout"
-)
-
-// AddArtifact will add an artifact.OCI to the store
-//  The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form.  Once
-//  saved, the entirety of the layout is copied to the store (which is just a registry).  This allows us to not only use
-//  strict types to define generic content, but provides a processing pipeline suitable for extensibility.  In the
-//  future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout.
-func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference name.Reference) (ocispec.Descriptor, error) {
-	if err := s.precheck(); err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	stg, err := newOciStage()
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	if s.cache != nil {
-		cached := cache.Oci(oci, s.cache)
-		oci = cached
-	}
-
-	pdesc, err := stg.add(ctx, oci, reference)
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	if err := stg.commit(ctx, s); err != nil {
-		return ocispec.Descriptor{}, nil
-	}
-
-	return pdesc, nil
-}
-
-// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting everything in the underlying store directory
-// 	This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents
-// 	To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly includes docker/registry/v2
-// 	in the search dir
-func (s *Store) Flush(ctx context.Context) error {
-	contentDir := filepath.Join(s.DataDir, "docker", "registry", "v2")
-	fs, err := ioutil.ReadDir(contentDir)
-	if !os.IsNotExist(err) && err != nil {
-		return err
-	}
-
-	for _, f := range fs {
-		err := os.RemoveAll(filepath.Join(contentDir, f.Name()))
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// AddCollection .
-func (s *Store) AddCollection(ctx context.Context, coll artifact.Collection) ([]ocispec.Descriptor, error) {
-	if err := s.precheck(); err != nil {
-		return nil, err
-	}
-
-	cnts, err := coll.Contents()
-	if err != nil {
-		return nil, err
-	}
-
-	for ref, o := range cnts {
-		if _, err := s.AddArtifact(ctx, o, ref); err != nil {
-			return nil, nil
-		}
-	}
-
-	return nil, err
-}
-
-type stager interface {
-	// add adds an artifact.OCI to the stage
-	add(artifact.OCI) error
-
-	// commit pushes all the staged contents into the store and closes the stage
-	commit(*Store) error
-
-	// close flushes and closes the stage
-	close() error
-}
-
-type oci struct {
-	layout layout.Path
-	root   string
-}
-
-func (o *oci) add(ctx context.Context, oci artifact.OCI, reference name.Reference) (ocispec.Descriptor, error) {
-	mdesc, err := o.layout.WriteOci(oci, reference)
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-	return mdesc, err
-}
-
-func (o *oci) commit(ctx context.Context, s *Store) error {
-	defer o.close()
-	ts, err := layout.NewOCIStore(o.root)
-	if err != nil {
-		return err
-	}
-
-	if err = layout.Copy(ctx, ts, s.Registry()); err != nil {
-		return err
-	}
-	return err
-}
-
-func (o *oci) close() error {
-	return os.RemoveAll(o.root)
-}
-
-func newOciStage() (*oci, error) {
-	tmpdir, err := os.MkdirTemp("", "hauler")
-	if err != nil {
-		return nil, err
-	}
-
-	l, err := layout.FromPath(tmpdir)
-	if err != nil {
-		return nil, err
-	}
-
-	return &oci{
-		layout: l,
-		root:   tmpdir,
-	}, nil
-}

pkg/store/options.go

@@ -1,20 +0,0 @@
-package store
-
-import "github.com/rancherfederal/hauler/pkg/cache"
-
-// Options defines options for Store
-type Options func(*Store)
-
-// WithCache initializes a Store with a cache.Cache, all content added to the Store will first be cached
-func WithCache(c cache.Cache) Options {
-	return func(s *Store) {
-		s.cache = c
-	}
-}
-
-// WithDefaultRepository sets the default repository to use when none is specified (defaults to "library")
-func WithDefaultRepository(repo string) Options {
-	return func(s *Store) {
-		s.DefaultRepository = repo
-	}
-}

pkg/store/store.go

@@ -1,214 +0,0 @@
-package store
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"regexp"
-	"time"
-
-	"github.com/distribution/distribution/v3/configuration"
-	dcontext "github.com/distribution/distribution/v3/context"
-	"github.com/distribution/distribution/v3/reference"
-	"github.com/distribution/distribution/v3/registry/client"
-	"github.com/distribution/distribution/v3/registry/handlers"
-	"github.com/google/go-containerregistry/pkg/name"
-	"github.com/sirupsen/logrus"
-
-	// Init filesystem distribution storage driver
-	_ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem"
-
-	"github.com/rancherfederal/hauler/pkg/cache"
-)
-
-var (
-	httpRegex = regexp.MustCompile("https?://")
-)
-
-// Store is a simple wrapper around distribution/distribution to enable hauler's use case
-type Store struct {
-	DataDir           string
-	DefaultRepository string
-
-	config  *configuration.Configuration
-	handler http.Handler
-	server  *httptest.Server
-	cache   cache.Cache
-}
-
-// NewStore creates a new registry store, designed strictly for use within haulers embedded operations and _not_ for serving
-func NewStore(ctx context.Context, dataDir string, opts ...Options) *Store {
-	cfg := &configuration.Configuration{
-		Version: "0.1",
-		Storage: configuration.Storage{
-			"cache":      configuration.Parameters{"blobdescriptor": "inmemory"},
-			"filesystem": configuration.Parameters{"rootdirectory": dataDir},
-		},
-	}
-	cfg.Log.Level = "panic"
-	cfg.HTTP.Headers = http.Header{"X-Content-Type-Options": []string{"nosniff"}}
-
-	handler := setupHandler(ctx, cfg)
-
-	s := &Store{
-		DataDir: dataDir,
-		config:  cfg,
-		handler: handler,
-	}
-
-	for _, opt := range opts {
-		opt(s)
-	}
-
-	return s
-}
-
-// Open will create a new server and start it, it's up to the consumer to close it
-func (s *Store) Open() *httptest.Server {
-	server := httptest.NewServer(s.handler)
-	s.server = server
-	return server
-}
-
-// Close stops the server
-func (s *Store) Close() {
-	s.server.Close()
-	s.server = nil
-	return
-}
-
-// List will list all known content tags in the registry
-// TODO: This fn is messy and needs cleanup, this is arguably easier with the catalog api as well
-func (s *Store) List(ctx context.Context) ([]string, error) {
-	reg, err := client.NewRegistry(s.RegistryURL(), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	entries := make(map[string]reference.Named)
-	last := ""
-	for {
-		chunk := make([]string, 20) // randomly chosen number...
-		nf, err := reg.Repositories(ctx, chunk, last)
-		if nf > 0 {
-			last = chunk[nf-1]
-		}
-
-		for _, e := range chunk {
-			if e == "" {
-				continue
-			}
-
-			ref, err := reference.WithName(e)
-			if err != nil {
-				return nil, err
-			}
-			entries[e] = ref
-		}
-		if err == io.EOF {
-			break
-		}
-	}
-
-	var refs []string
-	for ref, named := range entries {
-		repo, err := client.NewRepository(named, s.RegistryURL(), nil)
-		if err != nil {
-			return nil, err
-		}
-
-		tsvc := repo.Tags(ctx)
-		ts, err := tsvc.All(ctx)
-		if err != nil {
-			continue
-		}
-
-		for _, t := range ts {
-			ref, err := name.ParseReference(ref, name.WithDefaultRegistry(""), name.WithDefaultTag(t))
-			if err != nil {
-				return nil, err
-			}
-			refs = append(refs, ref.Name())
-		}
-	}
-
-	return refs, nil
-}
-
-// precheck checks whether server is appropriately started and errors if it's not
-// 		used to safely run Store operations without fear of panics
-func (s *Store) precheck() error {
-	if s.server == nil || s.server.URL == "" {
-		return fmt.Errorf("server is not started yet")
-	}
-	return nil
-}
-
-// Registry returns the registries URL without the protocol, suitable for image relocation operations
-func (s *Store) Registry() string {
-	return httpRegex.ReplaceAllString(s.server.URL, "")
-}
-
-// RegistryURL returns the registries URL
-func (s *Store) RegistryURL() string {
-	return s.server.URL
-}
-
-func alive(path string, handler http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == path {
-			w.Header().Set("Cache-Control", "no-cache")
-			w.WriteHeader(http.StatusOK)
-			return
-		}
-		handler.ServeHTTP(w, r)
-	})
-}
-
-// setupHandler will set up the registry handler
-func setupHandler(ctx context.Context, config *configuration.Configuration) http.Handler {
-	ctx, _ = configureLogging(ctx, config)
-
-	app := handlers.NewApp(ctx, config)
-	app.RegisterHealthChecks()
-	handler := alive("/", app)
-
-	return handler
-}
-
-func configureLogging(ctx context.Context, cfg *configuration.Configuration) (context.Context, context.CancelFunc) {
-	logrus.SetLevel(logLevel(cfg.Log.Level))
-
-	formatter := cfg.Log.Formatter
-	if formatter == "" {
-		formatter = "text"
-	}
-
-	logrus.SetFormatter(&logrus.TextFormatter{
-		TimestampFormat: time.RFC3339Nano,
-	})
-
-	if len(cfg.Log.Fields) > 0 {
-		var fields []interface{}
-		for k := range cfg.Log.Fields {
-			fields = append(fields, k)
-		}
-
-		ctx = dcontext.WithValues(ctx, cfg.Log.Fields)
-		ctx = dcontext.WithLogger(ctx, dcontext.GetLogger(ctx, fields...))
-	}
-
-	dcontext.SetDefaultLogger(dcontext.GetLogger(ctx))
-	return context.WithCancel(ctx)
-}
-
-func logLevel(level configuration.Loglevel) logrus.Level {
-	l, err := logrus.ParseLevel(string(level))
-	if err != nil {
-		l = logrus.InfoLevel
-		logrus.Warnf("error parsing log level %q: %v, using %q", level, err, l)
-	}
-	return l
-}

pkg/store/store_test.go

@@ -1,87 +0,0 @@
-package store
-
-import (
-	"context"
-	"os"
-	"testing"
-
-	"github.com/google/go-containerregistry/pkg/name"
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/random"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
-)
-
-func TestStore_List(t *testing.T) {
-	ctx := context.Background()
-
-	s, err := testStore(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s.Open()
-	defer s.Close()
-
-	r := randomImage(t)
-	addImageToStore(t, s, r, "hauler/tester:latest")
-	addImageToStore(t, s, r, "hauler/tester:non")
-	addImageToStore(t, s, r, "other/ns:more")
-	addImageToStore(t, s, r, "unique/donkey:v1.2.2")
-
-	type args struct {
-		ctx context.Context
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{
-			name:    "should list",
-			args:    args{},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			refs, err := s.List(ctx)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("List() error = %v, wantErr %v", err, tt.wantErr)
-			}
-
-			// TODO: Make this more robust
-			if len(refs) != 4 {
-				t.Errorf("Expected 4, got %d", len(refs))
-			}
-		})
-	}
-}
-
-func testStore(ctx context.Context) (*Store, error) {
-	tmpdir, err := os.MkdirTemp("", "hauler")
-	if err != nil {
-		return nil, err
-	}
-
-	s := NewStore(ctx, tmpdir)
-	return s, nil
-}
-
-func randomImage(t *testing.T) v1.Image {
-	r, err := random.Image(1024, 3)
-	if err != nil {
-		t.Fatalf("random.Image() = %v", err)
-	}
-	return r
-}
-
-func addImageToStore(t *testing.T, s *Store, image v1.Image, reference string) {
-	ref, err := name.ParseReference(reference, name.WithDefaultRegistry(s.Registry()))
-	if err != nil {
-		t.Error(err)
-	}
-
-	if err := remote.Write(ref, image); err != nil {
-		t.Error(err)
-	}
-}

testdata/chart-collection.yaml

@@ -7,5 +7,17 @@ spec:
   charts:
     # charts are also fetched and served as OCI content (currently experimental in helm)
     #   HELM_EXPERIMENTAL_OCI=1 helm chart pull <hauler-registry>/loki:2.6.2
-    - name: loki
-      repoURL: https://grafana.github.io/helm-charts
+#    - name: loki
+#      repoURL: https://grafana.github.io/helm-charts
+
+#    - name: longhorn
+#      repoURL: https://charts.longhorn.io
+
+#    - name: cert-manager
+#      repoURL: https://charts.jetstack.io
+#      version: v1.6.1
+#      extraImages:
+#        - ref: quay.io/jetstack/cert-manager-cainjector:v1.6.1
+
+    - name: podinfo
+      repoURL: https://stefanprodan.github.io/podinfo

testdata/contents.yaml

@@ -5,19 +5,18 @@ metadata:
 spec:
   files:
     # hauler can save/redistribute files on disk (be careful! paths are relative)
-    - ref: testdata/contents.yaml
+    - path: testdata/contents.yaml
 
-    # TODO: when directories are specified, they will be archived and stored as a file
-#    - ref: testdata/
+    # when directories are specified, the directory contents will be archived and stored
+    - path: testdata/
 
     # hauler can also fetch remote content, and will "smartly" identify filenames _when possible_
     #   filename below = "k3s-images.txt"
-    - ref: "https://github.com/k3s-io/k3s/releases/download/v1.22.2%2Bk3s2/k3s-images.txt"
+    - path: "https://github.com/k3s-io/k3s/releases/download/v1.22.2%2Bk3s2/k3s-images.txt"
 
-    # when filenames are not appropriate, a name should be specified
-    #   this will still work, but default to a filename of "get.k3s.io"
-    - ref: https://get.k3s.io
-      name: get-k3s.sh
+    # when discovered filenames are not desired, a file name can be specified
+    - path: https://get.k3s.io
+      name: k3s-init.sh
 
 ---
 apiVersion: content.hauler.cattle.io/v1alpha1
@@ -27,16 +26,16 @@ metadata:
 spec:
   images:
     # images can be referenced shorthanded without a tag
-    - ref: hello-world
+    - name: hello-world
 
     # or namespaced with a tag
-    - ref: rancher/cowsay:latest
+    - name: rancher/cowsay:latest
 
     # or by their digest:
-#    - ref: registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f
+    - name: registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f
 
     # or fully qualified from any OCI compliant registry registry
-    - ref: ghcr.io/fluxcd/flux-cli:v0.22.0
+    - name: ghcr.io/fluxcd/flux-cli:v0.22.0
 
 ---
 apiVersion: content.hauler.cattle.io/v1alpha1