factor out core oci logic into independent library (rancherfederal/ocil)

This commit is contained in:
Josh Wolf
2022-01-12 09:47:09 -07:00
parent 96d231efdf
commit 8b372d8a20
39 changed files with 112 additions and 2320 deletions

View File

@@ -1,102 +0,0 @@
package cache
import (
"errors"
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifact"
)
type Cache interface {
Put(v1.Layer) (v1.Layer, error)
Get(v1.Hash) (v1.Layer, error)
}
var ErrLayerNotFound = errors.New("layer not found")
type oci struct {
artifact.OCI
c Cache
}
func Oci(o artifact.OCI, c Cache) artifact.OCI {
return &oci{
OCI: o,
c: c,
}
}
func (o *oci) Layers() ([]v1.Layer, error) {
ls, err := o.OCI.Layers()
if err != nil {
return nil, err
}
var out []v1.Layer
for _, l := range ls {
out = append(out, &lazyLayer{inner: l, c: o.c})
}
return out, nil
}
type lazyLayer struct {
inner v1.Layer
c Cache
}
func (l *lazyLayer) Compressed() (io.ReadCloser, error) {
digest, err := l.inner.Digest()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(digest)
if err != nil {
return nil, err
}
return layer.Compressed()
}
func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) {
diffID, err := l.inner.DiffID()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(diffID)
if err != nil {
return nil, err
}
return layer.Uncompressed()
}
func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) {
var layer v1.Layer
if cl, err := l.c.Get(h); err == nil {
layer = cl
} else if err == ErrLayerNotFound {
rl, err := l.c.Put(l.inner)
if err != nil {
return nil, err
}
layer = rl
} else {
return nil, err
}
return layer, nil
}
func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() }
func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() }

View File

@@ -1,5 +0,0 @@
package cache
/*
This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache
*/

View File

@@ -1,120 +0,0 @@
package cache
import (
"io"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/rancherfederal/hauler/internal/layer"
)
type fs struct {
root string
}
func NewFilesystem(root string) Cache {
return &fs{root: root}
}
func (f *fs) Put(l v1.Layer) (v1.Layer, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
diffID, err := l.DiffID()
if err != nil {
return nil, err
}
return &cachedLayer{
Layer: l,
root: f.root,
digest: digest,
diffID: diffID,
}, nil
}
func (f *fs) Get(h v1.Hash) (v1.Layer, error) {
opener := f.open(h)
l, err := layer.FromOpener(opener)
if os.IsNotExist(err) {
return nil, ErrLayerNotFound
}
return l, err
}
func (f *fs) open(h v1.Hash) layer.Opener {
return func() (io.ReadCloser, error) {
return os.Open(layerpath(f.root, h))
}
}
type cachedLayer struct {
v1.Layer
root string
digest, diffID v1.Hash
}
func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) {
lp := layerpath(l.root, h)
if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil {
return nil, err
}
return os.Create(lp)
}
func (l *cachedLayer) Compressed() (io.ReadCloser, error) {
f, err := l.create(l.digest)
if err != nil {
return nil, nil
}
rc, err := l.Layer.Compressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) {
f, err := l.create(l.diffID)
if err != nil {
return nil, err
}
rc, err := l.Layer.Uncompressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func layerpath(root string, h v1.Hash) string {
return filepath.Join(root, h.Algorithm, h.Hex)
}
type readcloser struct {
t io.Reader
closes []func() error
}
func (rc *readcloser) Read(b []byte) (int, error) {
return rc.t.Read(b)
}
func (rc *readcloser) Close() error {
var err error
for _, c := range rc.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}

View File

@@ -1,165 +0,0 @@
package getter
import (
"archive/tar"
"compress/gzip"
"context"
"io"
"net/url"
"os"
"path/filepath"
"time"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type directory struct {
*File
}
func NewDirectory() *directory {
return &directory{File: NewFile()}
}
func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
tmpfile, err := os.CreateTemp("", "hauler")
if err != nil {
return nil, err
}
digester := digest.Canonical.Digester()
zw := gzip.NewWriter(io.MultiWriter(tmpfile, digester.Hash()))
defer zw.Close()
tarDigester := digest.Canonical.Digester()
if err := tarDir(d.path(u), d.Name(u), io.MultiWriter(zw, tarDigester.Hash()), false); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
if err := tmpfile.Sync(); err != nil {
return nil, err
}
fi, err := os.Open(tmpfile.Name())
if err != nil {
return nil, err
}
// rc := &closer{
// t: io.TeeReader(tmpfile, fi),
// closes: []func() error{fi.Close, tmpfile.Close, zw.Close},
// }
return fi, nil
}
func (d directory) Detect(u *url.URL) bool {
if len(d.path(u)) == 0 {
return false
}
fi, err := os.Stat(d.path(u))
if err != nil {
return false
}
return fi.IsDir()
}
func (d directory) Config(u *url.URL) artifact.Config {
c := &directoryConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileDirectoryConfigMediaType))
}
type directoryConfig struct {
config `json:",inline,omitempty"`
}
func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error {
tw := tar.NewWriter(w)
defer tw.Close()
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Rename path
name, err := filepath.Rel(root, path)
if err != nil {
return err
}
name = filepath.Join(prefix, name)
name = filepath.ToSlash(name)
// Generate header
var link string
mode := info.Mode()
if mode&os.ModeSymlink != 0 {
if link, err = os.Readlink(path); err != nil {
return err
}
}
header, err := tar.FileInfoHeader(info, link)
if err != nil {
return errors.Wrap(err, path)
}
header.Name = name
header.Uid = 0
header.Gid = 0
header.Uname = ""
header.Gname = ""
if stripTimes {
header.ModTime = time.Time{}
header.AccessTime = time.Time{}
header.ChangeTime = time.Time{}
}
// Write file
if err := tw.WriteHeader(header); err != nil {
return errors.Wrap(err, "tar")
}
if mode.IsRegular() {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err := io.Copy(tw, file); err != nil {
return errors.Wrap(err, path)
}
}
return nil
}); err != nil {
return err
}
return nil
}
type closer struct {
t io.Reader
closes []func() error
}
func (c *closer) Read(p []byte) (n int, err error) {
return c.t.Read(p)
}
func (c *closer) Close() error {
var err error
for _, c := range c.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}

View File

@@ -1,53 +0,0 @@
package getter
import (
"context"
"io"
"net/url"
"os"
"path/filepath"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type File struct{}
func NewFile() *File {
return &File{}
}
func (f File) Name(u *url.URL) string {
return filepath.Base(f.path(u))
}
func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return os.Open(f.path(u))
}
func (f File) Detect(u *url.URL) bool {
if len(f.path(u)) == 0 {
return false
}
fi, err := os.Stat(f.path(u))
if err != nil {
return false
}
return !fi.IsDir()
}
func (f File) path(u *url.URL) string {
return filepath.Join(u.Host, u.Path)
}
func (f File) Config(u *url.URL) artifact.Config {
c := &fileConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileLocalConfigMediaType))
}
type fileConfig struct {
config `json:",inline,omitempty"`
}

View File

@@ -1,114 +0,0 @@
package getter
import (
"context"
"io"
"net/url"
v1 "github.com/google/go-containerregistry/pkg/v1"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/internal/layer"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Client struct {
Getters map[string]Getter
Options ClientOptions
}
// TODO: Make some valid ClientOptions
type ClientOptions struct{}
var (
ErrGetterTypeUnknown = errors.New("no getter type found matching reference")
)
type Getter interface {
Open(context.Context, *url.URL) (io.ReadCloser, error)
Detect(*url.URL) bool
Name(*url.URL) string
Config(*url.URL) artifact.Config
}
func NewClient(opts ClientOptions) *Client {
defaults := map[string]Getter{
"file": NewFile(),
"directory": NewDirectory(),
"http": NewHttp(),
}
c := &Client{
Getters: defaults,
Options: opts,
}
return c
}
func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) {
u, err := url.Parse(source)
if err != nil {
return nil, err
}
for _, g := range c.Getters {
if g.Detect(u) {
opener := func() (io.ReadCloser, error) {
return g.Open(ctx, u)
}
annotations := make(map[string]string)
annotations[ocispec.AnnotationTitle] = g.Name(u)
switch g.(type) {
case *directory:
annotations[content.AnnotationUnpack] = "true"
}
l, err := layer.FromOpener(opener,
layer.WithMediaType(consts.FileLayerMediaType),
layer.WithAnnotations(annotations))
if err != nil {
return nil, err
}
return l, nil
}
}
return nil, errors.Wrapf(ErrGetterTypeUnknown, "%s", source)
}
func (c *Client) Name(source string) string {
u, err := url.Parse(source)
if err != nil {
return source
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Name(u)
}
}
return source
}
func (c *Client) Config(source string) artifact.Config {
u, err := url.Parse(source)
if err != nil {
return nil
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Config(u)
}
}
return nil
}
type config struct {
Reference string `json:"reference"`
Annotations map[string]string `json:"annotations,omitempty"`
}

View File

@@ -1,129 +0,0 @@
package getter_test
import (
"net/url"
"os"
"path/filepath"
"testing"
"github.com/rancherfederal/hauler/internal/getter"
)
func TestClient_Detect(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should identify a file",
args: args{
source: fileWithExt,
},
want: "file",
},
{
name: "should identify a directory",
args: args{
source: rootDir,
},
want: "directory",
},
{
name: "should identify a http",
args: args{
source: "http://my.cool.website",
},
want: "http",
},
{
name: "should identify a http",
args: args{
source: "https://my.cool.website",
},
want: "http",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := identify(c, tt.args.source); got != tt.want {
t.Errorf("identify() = %v, want %v", got, tt.want)
}
})
}
}
func identify(c *getter.Client, source string) string {
u, _ := url.Parse(source)
for t, g := range c.Getters {
if g.Detect(u) {
return t
}
}
return ""
}
func TestClient_Name(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should correctly name a file with an extension",
args: args{
source: fileWithExt,
},
want: "file.yaml",
},
{
name: "should correctly name a directory",
args: args{
source: rootDir,
},
want: rootDir,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := c.Name(tt.args.source); got != tt.want {
t.Errorf("Name() = %v, want %v", got, tt.want)
}
})
}
}
var (
rootDir = "gettertests"
fileWithExt = filepath.Join(rootDir, "file.yaml")
)
func setup(t *testing.T) func() {
if err := os.MkdirAll(rootDir, os.ModePerm); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil {
t.Fatal(err)
}
return func() {
os.RemoveAll(rootDir)
}
}

View File

@@ -1,67 +0,0 @@
package getter
import (
"context"
"io"
"mime"
"net/http"
"net/url"
"path/filepath"
"strings"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Http struct{}
func NewHttp() *Http {
return &Http{}
}
func (h Http) Name(u *url.URL) string {
resp, err := http.Head(u.String())
if err != nil {
return ""
}
contentType := resp.Header.Get("Content-Type")
for _, v := range strings.Split(contentType, ",") {
t, _, err := mime.ParseMediaType(v)
if err != nil {
break
}
// TODO: Identify known mimetypes for hints at a filename
_ = t
}
// TODO: Not this
return filepath.Base(u.String())
}
func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
resp, err := http.Get(u.String())
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (h Http) Detect(u *url.URL) bool {
switch u.Scheme {
case "http", "https":
return true
}
return false
}
func (h *Http) Config(u *url.URL) artifact.Config {
c := &httpConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileHttpConfigMediaType))
}
type httpConfig struct {
config `json:",inline,omitempty"`
}

View File

@@ -1,127 +0,0 @@
package layer
import (
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Opener func() (io.ReadCloser, error)
func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) {
var err error
layer := &layer{
mediaType: consts.UnknownLayer,
annotations: make(map[string]string, 1),
}
layer.uncompressedOpener = opener
layer.compressedOpener = func() (io.ReadCloser, error) {
rc, err := opener()
if err != nil {
return nil, err
}
return rc, nil
}
for _, opt := range opts {
opt(layer)
}
if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil {
return nil, err
}
if layer.diffID, _, err = compute(layer.compressedOpener); err != nil {
return nil, err
}
return layer, nil
}
func compute(opener Opener) (v1.Hash, int64, error) {
rc, err := opener()
if err != nil {
return v1.Hash{}, 0, err
}
defer rc.Close()
return v1.SHA256(rc)
}
type Option func(*layer)
func WithMediaType(mt string) Option {
return func(l *layer) {
l.mediaType = mt
}
}
func WithAnnotations(annotations map[string]string) Option {
return func(l *layer) {
if l.annotations == nil {
l.annotations = make(map[string]string)
}
l.annotations = annotations
}
}
type layer struct {
digest v1.Hash
diffID v1.Hash
size int64
compressedOpener Opener
uncompressedOpener Opener
mediaType string
annotations map[string]string
urls []string
}
func (l layer) Descriptor() (*v1.Descriptor, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
mt, err := l.MediaType()
if err != nil {
return nil, err
}
return &v1.Descriptor{
MediaType: mt,
Size: l.size,
Digest: digest,
Annotations: l.annotations,
URLs: l.urls,
// TODO: Allow platforms
Platform: nil,
}, nil
}
func (l layer) Digest() (v1.Hash, error) {
return l.digest, nil
}
func (l layer) DiffID() (v1.Hash, error) {
return l.diffID, nil
}
func (l layer) Compressed() (io.ReadCloser, error) {
return l.compressedOpener()
}
func (l layer) Uncompressed() (io.ReadCloser, error) {
return l.uncompressedOpener()
}
func (l layer) Size() (int64, error) {
return l.size, nil
}
func (l layer) MediaType() (gtypes.MediaType, error) {
return gtypes.MediaType(l.mediaType), nil
}

View File

@@ -6,7 +6,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/ocil/pkg/consts"
)
type Fn func(desc ocispec.Descriptor) (string, error)