github/hauler
1
0
Fork 0
mirror of https://github.com/hauler-dev/hauler.git synced 2026-08-27 14:07:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

add getter, store, and file tests

Browse Source
This commit is contained in:
Josh Wolf
2021-12-01 22:49:16 -07:00
parent 5855f79156
commit 34836dacb0
17 changed files with 335 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/hauler/cli/download/download.go
+1 -1
View File
@@ -83,7 +83,7 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
// l.Infof("downloaded image [%s] to [%s]", ref.Name(), outputFile)
case consts.FileConfigMediaType:
case consts.FileLocalConfigMediaType:
l.Debugf("identified [file] (%s) content", manifest.Config.MediaType)
fs := mapper.NewStore(o.DestinationDir, file.Mapper())
cmd/hauler/cli/download/download_test.go
-38
View File
@@ -1,38 +0,0 @@
package download
import (
"context"
"testing"
)
func TestCmd(t *testing.T) {
ctx := context.Background()
type args struct {
ctx context.Context
o *Opts
reference string
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "should work",
args: args{
ctx: ctx,
o: &Opts{DestinationDir: ""},
reference: "localhost:3000/hauler/file.txt:latest",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := Cmd(tt.args.ctx, tt.args.o, tt.args.reference); (err != nil) != tt.wantErr {
t.Errorf("Cmd() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
go.mod
+1
View File
@@ -16,6 +16,7 @@ require (
github.com/rancher/wrangler v0.8.4
github.com/rs/zerolog v1.26.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.6.0
github.com/spf13/cobra v1.2.1
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
helm.sh/helm/v3 v3.6.1-0.20211119214113-6a1daecd0c29
go.sum
+1
View File
@@ -1035,6 +1035,7 @@ github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
internal/getter/directory.go
+8 -5
View File
@@ -15,7 +15,11 @@ import (
)
type directory struct {
file
*File
}
func NewDirectory() *directory {
return &directory{File: NewFile()}
}
func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
@@ -57,12 +61,11 @@ func (d directory) Detect(u *url.URL) bool {
return false
}
if fi, err := os.Stat(d.path(u)); err != nil {
return false
} else if !fi.IsDir() {
fi, err := os.Stat(d.path(u))
if err != nil {
return false
}
return true
return fi.IsDir()
}
func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error {
internal/getter/file.go
+26 -12
View File
@@ -7,39 +7,53 @@ import (
"os"
"path/filepath"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type file struct{}
type File struct{}
func (f file) Name(u *url.URL) string {
func NewFile() *File {
return &File{}
}
func (f File) Name(u *url.URL) string {
return filepath.Base(f.path(u))
}
func (f file) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return os.Open(f.path(u))
}
func (f file) Detect(u *url.URL) bool {
func (f File) Detect(u *url.URL) bool {
if len(f.path(u)) == 0 {
return false
}
if fi, err := os.Stat(f.path(u)); err != nil {
return false
} else if fi.IsDir() {
fi, err := os.Stat(f.path(u))
if err != nil {
return false
}
return true
return !fi.IsDir()
}
func (f file) path(u *url.URL) string {
func (f File) path(u *url.URL) string {
return filepath.Join(u.Host, u.Path)
}
func (f file) Config(u *url.URL) artifact.Config {
c := &config{
Reference: u.String(),
func (f File) Config(u *url.URL) artifact.Config {
c := &fileConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c)
}
type fileConfig struct {
config `json:",inline,omitempty"`
}
func (c *fileConfig) MediaType() (types.MediaType, error) {
return consts.FileLocalConfigMediaType, nil
}
internal/getter/getter.go
+9 -12
View File
@@ -6,14 +6,12 @@ import (
"net/url"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/internal/layer"
"github.com/rancherfederal/hauler/pkg/artifact"
htypes "github.com/rancherfederal/hauler/pkg/consts"
)
type Client struct {
@@ -40,10 +38,9 @@ type Getter interface {
func NewClient(opts ClientOptions) *Client {
defaults := map[string]Getter{
"file": new(file),
"directory": new(directory),
"http": new(https),
"https": new(https),
"file": NewFile(),
"directory": NewDirectory(),
"http": NewHttp(),
}
c := &Client{
@@ -95,7 +92,9 @@ func (c *Client) Name(source string) string {
return source
}
for _, g := range c.Getters {
return g.Name(u)
if g.Detect(u) {
return g.Name(u)
}
}
return source
}
@@ -106,7 +105,9 @@ func (c *Client) Config(source string) artifact.Config {
return nil
}
for _, g := range c.Getters {
return g.Config(u)
if g.Detect(u) {
return g.Config(u)
}
}
return nil
}
@@ -115,7 +116,3 @@ type config struct {
Reference string `json:"reference"`
Annotations map[string]string `json:"annotations,omitempty"`
}
func (c *config) MediaType() (types.MediaType, error) {
return htypes.FileConfigMediaType, nil
}
internal/getter/getter_test.go
+129
View File
@@ -0,0 +1,129 @@
package getter_test
import (
"net/url"
"os"
"path/filepath"
"testing"
"github.com/rancherfederal/hauler/internal/getter"
)
func TestClient_Detect(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should identify a file",
args: args{
source: fileWithExt,
},
want: "file",
},
{
name: "should identify a directory",
args: args{
source: rootDir,
},
want: "directory",
},
{
name: "should identify a http",
args: args{
source: "http://my.cool.website",
},
want: "http",
},
{
name: "should identify a http",
args: args{
source: "https://my.cool.website",
},
want: "http",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := identify(c, tt.args.source); got != tt.want {
t.Errorf("identify() = %v, want %v", got, tt.want)
}
})
}
}
func identify(c *getter.Client, source string) string {
u, _ := url.Parse(source)
for t, g := range c.Getters {
if g.Detect(u) {
return t
}
}
return ""
}
func TestClient_Name(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should correctly name a file with an extension",
args: args{
source: fileWithExt,
},
want: "file.yaml",
},
{
name: "should correctly name a directory",
args: args{
source: rootDir,
},
want: rootDir,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := c.Name(tt.args.source); got != tt.want {
t.Errorf("Name() = %v, want %v", got, tt.want)
}
})
}
}
var (
rootDir = "gettertests"
fileWithExt = filepath.Join(rootDir, "file.yaml")
)
func setup(t *testing.T) func() {
if err := os.MkdirAll(rootDir, os.ModePerm); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil {
t.Fatal(err)
}
return func() {
os.RemoveAll(rootDir)
}
}
internal/getter/https.go
+22 -7
View File
@@ -9,12 +9,19 @@ import (
"path/filepath"
"strings"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type https struct{}
type Http struct{}
func (h https) Name(u *url.URL) string {
func NewHttp() *Http {
return &Http{}
}
func (h Http) Name(u *url.URL) string {
resp, err := http.Head(u.String())
if err != nil {
return ""
@@ -34,7 +41,7 @@ func (h https) Name(u *url.URL) string {
return filepath.Base(u.String())
}
func (h https) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
resp, err := http.Get(u.String())
if err != nil {
return nil, err
@@ -42,7 +49,7 @@ func (h https) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return resp.Body, nil
}
func (h https) Detect(u *url.URL) bool {
func (h Http) Detect(u *url.URL) bool {
switch u.Scheme {
case "http", "https":
return true
@@ -50,9 +57,17 @@ func (h https) Detect(u *url.URL) bool {
return false
}
func (h https) Config(u *url.URL) artifact.Config {
c := &config{
Reference: u.String(),
func (h *Http) Config(u *url.URL) artifact.Config {
c := &httpConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c)
}
type httpConfig struct {
config `json:",inline,omitempty"`
}
func (c *httpConfig) MediaType() (types.MediaType, error) {
return consts.FileHttpConfigMediaType, nil
}
pkg/collection/k3s/k3s_test.go
-71
View File
@@ -1,72 +1 @@
package k3s
import (
"context"
"os"
"testing"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
)
// TODO: This is not at all a good test, we really just need to test the added collections functionality (like image scanning)
func TestNewK3s(t *testing.T) {
ctx := context.Background()
l := log.NewLogger(os.Stdout)
ctx = l.WithContext(ctx)
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
t.Error(err)
}
defer os.Remove(tmpdir)
s, err := store.NewStore(tmpdir)
if err != nil {
t.Error(err)
}
type args struct {
version string
}
tests := []struct {
name string
args args
want artifact.Collection
wantErr bool
}{
{
name: "should work",
args: args{
version: "v1.22.2+k3s2",
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NewK3s(tt.args.version)
if (err != nil) != tt.wantErr {
t.Errorf("NewK3s() error = %v, wantErr %v", err, tt.wantErr)
return
}
c, err := got.Contents()
if err != nil {
t.Fatal(err)
}
for r, o := range c {
if _, err := s.AddArtifact(ctx, o, r); err != nil {
t.Fatal(err)
}
}
// if !reflect.DeepEqual(got, tt.want) {
// t.Errorf("NewK3s() got = %v, want %v", got, tt.want)
// }
})
}
}
pkg/consts/types.go
+3 -2
View File
@@ -22,8 +22,9 @@ const (
// FileLayerMediaType is the reserved media type for File content layers
FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1"
// FileConfigMediaType is the reserved media type for File config
FileConfigMediaType = "application/vnd.content.hauler.file.config.v1+json"
// FileLocalConfigMediaType is the reserved media type for File config
FileLocalConfigMediaType = "application/vnd.content.hauler.file.local.config.v1+json"
FileHttpConfigMediaType = "application/vnd.content.hauler.file.http.config.v1+json"
// WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers
WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm"
pkg/content/file/file.go
+10 -5
View File
@@ -20,8 +20,9 @@ import (
var _ artifact.OCI = (*file)(nil)
type file struct {
ref string
client *getter.Client
ref string
client *getter.Client
computed bool
config artifact.Config
blob gv1.Layer
@@ -29,14 +30,18 @@ type file struct {
annotations map[string]string
}
func NewFile(ref string) *file {
// TODO: Allow user to configure this
func NewFile(ref string, opts ...Option) *file {
client := getter.NewClient(getter.ClientOptions{})
return &file{
f := &file{
client: client,
ref: ref,
}
for _, opt := range opts {
opt(f)
}
return f
}
func (f *file) Name(ref string) string {
pkg/content/file/file_test.go
+105 -165
View File
@@ -1,167 +1,107 @@
package file_test
// func TestFile_Copy(t *testing.T) {
// ctx := context.Background()
// l := log.NewLogger(os.Stdout)
// ctx = l.WithContext(ctx)
//
// tmpdir, err := os.MkdirTemp("", "hauler")
// if err != nil {
// t.Error(err)
// }
// defer os.Remove(tmpdir)
//
// // Make a temp file
// f, err := os.CreateTemp(tmpdir, "tmp")
// f.Write([]byte("content"))
// defer f.Close()
//
// fs := newTestFileServer(tmpdir)
// fs.Start()
// defer fs.Stop()
//
// s, err := store.NewBundle(tmpdir)
// if err != nil {
// t.Error(err)
// }
//
// type args struct {
// ctx context.Context
// registry string
// }
//
// tests := []struct {
// name string
// cfg v1alpha1.File
// args args
// wantErr bool
// }{
// {
// name: "should copy a local file successfully without an explicit name",
// cfg: v1alpha1.File{
// Ref: f.Name(),
// Name: filepath.Base(f.Name()),
// },
// args: args{
// ctx: ctx,
// },
// },
// {
// name: "should copy a local file successfully with an explicit name",
// cfg: v1alpha1.File{
// Ref: f.Name(),
// Name: "my-other-file",
// },
// args: args{
// ctx: ctx,
// },
// },
// {
// name: "should fail to copy a local file successfully with a malformed explicit name",
// cfg: v1alpha1.File{
// Ref: f.Name(),
// Name: "my!invalid~@file",
// },
// args: args{
// ctx: ctx,
// },
// wantErr: true,
// },
// {
// name: "should copy a remote file successfully without an explicit name",
// cfg: v1alpha1.File{
// Ref: fmt.Sprintf("%s/%s", fs.server.URL, filepath.Base(f.Name())),
// },
// args: args{
// ctx: ctx,
// },
// },
// {
// name: "should copy a remote file successfully with an explicit name",
// cfg: v1alpha1.File{
// Ref: fmt.Sprintf("%s/%s", fs.server.URL, filepath.Base(f.Name())),
// Name: "my-other-file",
// },
// args: args{
// ctx: ctx,
// },
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// f, err := file.NewFile(tt.cfg.Ref, tt.cfg.Name)
// if err != nil {
// t.Fatal(err)
// }
//
// ref, err := name.ParseReference("myfile")
// if err != nil {
// t.Fatal(err)
// }
//
// _, err = s.AddArtifact(ctx, f, ref)
// if (err != nil) != tt.wantErr {
// t.Error(err)
// }
//
// // if err := validate(tt.cfg.Ref, tt.cfg.Name, m); err != nil {
// // t.Error(err)
// // }
// })
// }
// }
//
// type testFileServer struct {
// server *httptest.Server
// }
//
// func newTestFileServer(path string) *testFileServer {
// s := httptest.NewUnstartedServer(http.FileServer(http.Dir(path)))
// return &testFileServer{server: s}
// }
//
// func (s *testFileServer) Start() *httptest.Server {
// s.server.Start()
// return s.server
// }
//
// func (s *testFileServer) Stop() {
// s.server.Close()
// }
//
// // validate ensure
// func validate(ref string, name string, got *v1.Manifest) error {
// data, err := os.ReadFile(ref)
// if err != nil {
// return err
// }
//
// d := digest.FromBytes(data)
//
// annotations := make(map[string]string)
// annotations[ocispec.AnnotationTitle] = name
// annotations[ocispec.AnnotationSource] = ref
//
// want := &v1.Manifest{
// SchemaVersion: 2,
// MediaType: types.OCIManifestSchema1,
// Config: v1.Descriptor{},
// Layers: []v1.Descriptor{
// {
// MediaType: types.FileLayerMediaType,
// Size: int64(len(data)),
// Digest: v1.Hash{
// Algorithm: d.Algorithm().String(),
// Hex: d.Hex(),
// },
// Annotations: annotations,
// },
// },
// Annotations: nil,
// }
//
// if !reflect.DeepEqual(want.Layers, got.Layers) {
// return fmt.Errorf("want = (%v) | got = (%v)", want, got)
// }
// return nil
// }
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/rancherfederal/hauler/internal/getter"
"github.com/rancherfederal/hauler/pkg/content/file"
)
func Test_file_Layers(t *testing.T) {
filename := "myfile.yaml"
data := []byte(`data`)
mfs := afero.NewMemMapFs()
afero.WriteFile(mfs, filename, data, 0644)
mf := &mockFile{File: getter.NewFile(), fs: mfs}
mockHttp := getter.NewHttp()
mhttp := afero.NewHttpFs(mfs)
fileserver := http.FileServer(mhttp.Dir("."))
http.Handle("/", fileserver)
s := httptest.NewServer(fileserver)
defer s.Close()
mc := &getter.Client{
Options: getter.ClientOptions{},
Getters: map[string]getter.Getter{
"file": mf,
"http": mockHttp,
},
}
tests := []struct {
name string
ref string
want []byte
wantErr bool
}{
{
name: "should load a local file and preserve contents",
ref: filename,
want: data,
wantErr: false,
},
{
name: "should load a remote file and preserve contents",
ref: s.URL + "/" + filename,
want: data,
wantErr: false,
},
// TODO: Add directory test
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := file.NewFile(tt.ref, file.WithClient(mc))
layers, err := f.Layers()
if (err != nil) != tt.wantErr {
t.Errorf("Layers() error = %v, wantErr %v", err, tt.wantErr)
return
}
rc, err := layers[0].Compressed()
if err != nil {
t.Fatal(err)
}
got, err := io.ReadAll(rc)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, tt.want) {
t.Errorf("Layers() got = %v, want %v", layers, tt.want)
}
})
}
}
type mockFile struct {
*getter.File
fs afero.Fs
}
func (m mockFile) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return m.fs.Open(filepath.Join(u.Host, u.Path))
}
func (m mockFile) Detect(u *url.URL) bool {
fi, err := m.fs.Stat(filepath.Join(u.Host, u.Path))
if err != nil {
return false
}
return !fi.IsDir()
}
pkg/content/file/options.go
+11
View File
@@ -0,0 +1,11 @@
package file
import "github.com/rancherfederal/hauler/internal/getter"
type Option func(*file)
func WithClient(c *getter.Client) Option {
return func(f *file) {
f.client = c
}
}
pkg/store/oci.go
-10
View File
@@ -11,7 +11,6 @@ import (
"sync"
ccontent "github.com/containerd/containerd/content"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/remotes"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -25,22 +24,13 @@ import (
var _ target.Target = (*oci)(nil)
type oci struct {
ccontent.Store
root string
index *ocispec.Index
nameMap *sync.Map // map[string]ocispec.Descriptor
}
func NewOCI(root string) (*oci, error) {
fs, err := local.NewStore(root)
if err != nil {
return nil, err
}
return &oci{
Store: fs,
root: root,
nameMap: &sync.Map{},
}, nil
pkg/store/store.go
+1 -1
View File
@@ -65,7 +65,7 @@ func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference str
// Ensure that index.docker.io isn't prepended
ref, err := name.ParseReference(reference, name.WithDefaultRegistry(""), name.WithDefaultTag("latest"))
if err != nil {
return ocispec.Descriptor{}, err
return ocispec.Descriptor{}, errors.Wrap(err, "adding artifact")
}
if err := stage.add(ctx, oci, ref); err != nil {
pkg/store/store_test.go
+8 -8
View File
@@ -47,14 +47,14 @@ func TestStore_AddArtifact(t *testing.T) {
},
wantErr: false,
},
{
name: "should fail with ErrInvalidReference when an invalid reference is provided",
args: args{
ctx: ctx,
reference: "n0tV@l!d:v1",
},
wantErr: true,
},
// {
// name: "should fail when an invalid reference is provided",
// args: args{
// ctx: ctx,
// reference: "n0tV@l!d:v1",
// },
// wantErr: true,
// },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {