github/hauler
1
0
Fork 0
mirror of https://github.com/hauler-dev/hauler.git synced 2026-02-14 18:09:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add cosign for handling image functionality. (#134)

Browse Source 
* pull back in ocil
* updates to OCIL funcs to handle cosign changes
* add cosign logic
* adjust Makefile to be a little more generic
* cli updates to accomodate the cosign additions
* add cosign drop-in funcs
* impl for cosign functions for images & store copy
* fixes and logging for cosign verify <iamge>
* fix cosign verify logging
* update go.mod

Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
This commit is contained in:
Adam Martin
2023-11-03 13:43:32 -04:00
committed by GitHub
parent 337494cefd
commit 4772657548
43 changed files with 2614 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@@ -11,13 +11,13 @@ all: fmt vet install test
build:
mkdir bin;\
$(GO_BUILD_ENV) go build -o bin ./cmd/...;\
GOENV=GOARCH=$(uname -m) CGO_ENABLED=0 go build -o bin ./cmd/...;\
build-all: fmt vet
goreleaser build --rm-dist --snapshot
install:
$(GO_BUILD_ENV) go install
GOENV=GOARCH=$(uname -m) CGO_ENABLED=0 go install ./cmd/...;\
vet:
go vet $(GO_FILES)

2
cmd/hauler/cli/download/download.go
View File

@@ -11,7 +11,7 @@ import (
"oras.land/oras-go/pkg/content"
"oras.land/oras-go/pkg/oras"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/log"

1
cmd/hauler/cli/serve/registry.go
View File

@@ -74,6 +74,7 @@ func (o *RegistryOpts) defaultConfig() *configuration.Configuration {
cfg.HTTP.Addr = fmt.Sprintf(":%d", o.Port)
cfg.HTTP.Headers = http.Header{
"X-Content-Type-Options": []string{"nosniff"},
"Accept": []string{"application/vnd.dsse.envelope.v1+json, application/json"},
}
return cfg

32
cmd/hauler/cli/store/add.go
View File

@@ -4,17 +4,17 @@ import (
"context"
"github.com/google/go-containerregistry/pkg/name"
"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/ocil/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/content/chart"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
)
@@ -62,40 +62,48 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
type AddImageOpts struct {
*RootOpts
Name string
Key string
}
func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
_ = f
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification")
}
func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error {
l := log.FromContext(ctx)
cfg := v1alpha1.Image{
Name: reference,
}
// Check if the user provided a key.
if o.Key != "" {
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name)
if err != nil {
return err
}
l.Infof("signature verified for image [%s]", cfg.Name)
}
return storeImage(ctx, s, cfg)
}
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error {
l := log.FromContext(ctx)
img, err := image.NewImage(i.Name)
if err != nil {
return err
}
r, err := name.ParseReference(i.Name)
if err != nil {
return err
}
desc, err := s.AddOCI(ctx, img, r.Name())
err = cosign.SaveImage(ctx, s, r.Name())
//desc, err := s.AddOCI(ctx, img, r.Name())
if err != nil {
return err
}
l.Infof("added 'image' to store at [%s], with digest [%s]", r.Name(), desc.Digest.String())
l.Infof("added 'image' to store at [%s]", r.Name())
return nil
}

28
cmd/hauler/cli/store/copy.go
View File

@@ -5,14 +5,13 @@ import (
"fmt"
"strings"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
)
type CopyOpts struct {
@@ -36,7 +35,6 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) {
func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error {
l := log.FromContext(ctx)
var descs []ocispec.Descriptor
components := strings.SplitN(targetRef, "://", 2)
switch components[0] {
case "dir":
@@ -44,11 +42,10 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string
fs := content.NewFile(components[1])
defer fs.Close()
ds, err := s.CopyAll(ctx, fs, nil)
_, err := s.CopyAll(ctx, fs, nil)
if err != nil {
return err
}
descs = ds
case "registry":
l.Debugf("identified registry target reference")
@@ -58,29 +55,16 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string
Insecure: o.Insecure,
PlainHTTP: o.PlainHTTP,
}
r, err := content.NewRegistry(ropts)
err := cosign.LoadImage(ctx, s, components[1], ropts)
if err != nil {
return err
}
mapperFn := func(ref string) (string, error) {
r, err := reference.Relocate(ref, components[1])
if err != nil {
return "", err
}
return r.Name(), nil
}
ds, err := s.CopyAll(ctx, r, mapperFn)
if err != nil {
return err
}
descs = ds
default:
return fmt.Errorf("detecting protocol from [%s]", targetRef)
}
l.Infof("Copied [%d] artifacts to [%s]", len(descs), components[1])
l.Infof("Copied artifacts to [%s]", components[1])
return nil
}

8
cmd/hauler/cli/store/extract.go
View File

@@ -2,13 +2,14 @@ package store
import (
"context"
"strings"
"encoding/json"
"fmt"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/log"
@@ -36,7 +37,8 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string
found := false
if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
if reference != r.Name() {
if !strings.Contains(reference, r.Name()) {
return nil
}
found = true
@@ -57,7 +59,7 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string
return err
}
pushedDesc, err := s.Copy(ctx, r.Name(), mapperStore, "")
pushedDesc, err := s.Copy(ctx, reference, mapperStore, "")
if err != nil {
return err
}

4
cmd/hauler/cli/store/flags.go
View File

@@ -6,8 +6,8 @@ import (
"os"
"path/filepath"
"github.com/rancherfederal/ocil/pkg/layer"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/layer"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/log"

12
cmd/hauler/cli/store/info.go
View File

@@ -10,9 +10,9 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/reference"
)
@@ -78,9 +78,11 @@ func buildTable(items ...item) string {
fmt.Fprintf(tw, "---------\t----\t--------\t----\n")
for _, i := range items {
fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n",
i.Reference, i.Type, i.Layers, i.Size,
)
if i.Type != "unknown" {
fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n",
i.Reference, i.Type, i.Layers, i.Size,
)
}
}
tw.Flush()
return b.String()

4
cmd/hauler/cli/store/load.go
View File

@@ -5,8 +5,8 @@ import (
"os"
"github.com/mholt/archiver/v3"
"github.com/rancherfederal/ocil/pkg/content"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/log"

2
cmd/hauler/cli/store/serve.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/distribution/distribution/v3/version"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/internal/server"
)

18
cmd/hauler/cli/store/sync.go
View File

@@ -11,25 +11,28 @@ import (
"helm.sh/helm/v3/pkg/action"
"k8s.io/apimachinery/pkg/util/yaml"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
"github.com/rancherfederal/hauler/pkg/collection/k3s"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
)
type SyncOpts struct {
*RootOpts
ContentFiles []string
Key string
}
func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification")
}
func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
@@ -94,7 +97,18 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
}
for _, i := range cfg.Spec.Images {
err := storeImage(ctx, s, i)
// Check if the user provided a key.
if o.Key != "" {
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, o.Key, i.Name)
if err != nil {
return err
}
l.Infof("signature verified for image [%s]", i.Name)
}
err = storeImage(ctx, s, i)
if err != nil {
return err
}

7
go.mod
View File

@@ -10,12 +10,14 @@ require (
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/mholt/archiver/v3 v3.5.1
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc5
github.com/pkg/errors v0.9.1
github.com/rancherfederal/ocil v0.1.9
github.com/rs/zerolog v1.31.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/afero v1.10.0
github.com/spf13/cobra v1.7.0
golang.org/x/sync v0.4.0
helm.sh/helm/v3 v3.13.0
k8s.io/apimachinery v0.28.2
k8s.io/client-go v0.28.2
@@ -110,7 +112,6 @@ require (
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nwaples/rardecode v1.1.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.2 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
@@ -123,7 +124,6 @@ require (
github.com/rubenv/sql-migrate v1.5.2 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/ulikunitz/xz v0.5.9 // indirect
@@ -140,7 +140,6 @@ require (
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/net v0.13.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect

2
go.sum
View File

@@ -454,8 +454,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
github.com/rancherfederal/ocil v0.1.9 h1:pmiUQCh2HTIMDD9tDj/UqBAAxq4yloLFgd2WnrZnQgc=
github.com/rancherfederal/ocil v0.1.9/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY=
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho=
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U=
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc=

4
internal/mapper/mappers.go
View File

@@ -6,7 +6,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Fn func(desc ocispec.Descriptor) (string, error)
@@ -39,7 +39,7 @@ func Images() map[string]Fn {
return "manifest.json", nil
})
for _, l := range []string{consts.DockerManifestSchema2, consts.OCIManifestSchema1} {
for _, l := range []string{consts.DockerManifestSchema2, consts.DockerManifestListSchema2, consts.OCIManifestSchema1} {
m[l] = manifestMapperFn
}

92
pkg/artifacts/config.go Normal file
View File

@@ -0,0 +1,92 @@
package artifacts
import (
"bytes"
"encoding/json"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ partial.Describable = (*marshallableConfig)(nil)
type Config interface {
// Raw returns the config bytes
Raw() ([]byte, error)
Digest() (v1.Hash, error)
MediaType() (types.MediaType, error)
Size() (int64, error)
}
type Marshallable interface{}
type ConfigOption func(*marshallableConfig)
// ToConfig takes anything that is marshallabe and converts it into a Config
func ToConfig(i Marshallable, opts ...ConfigOption) Config {
mc := &marshallableConfig{Marshallable: i}
for _, o := range opts {
o(mc)
}
return mc
}
func WithConfigMediaType(mediaType string) ConfigOption {
return func(config *marshallableConfig) {
config.mediaType = mediaType
}
}
// marshallableConfig implements Config using helper methods
type marshallableConfig struct {
Marshallable
mediaType string
}
func (c *marshallableConfig) MediaType() (types.MediaType, error) {
mt := c.mediaType
if mt == "" {
mt = consts.UnknownManifest
}
return types.MediaType(mt), nil
}
func (c *marshallableConfig) Raw() ([]byte, error) {
return json.Marshal(c.Marshallable)
}
func (c *marshallableConfig) Digest() (v1.Hash, error) {
return Digest(c)
}
func (c *marshallableConfig) Size() (int64, error) {
return Size(c)
}
type WithRawConfig interface {
Raw() ([]byte, error)
}
func Digest(c WithRawConfig) (v1.Hash, error) {
b, err := c.Raw()
if err != nil {
return v1.Hash{}, err
}
digest, _, err := v1.SHA256(bytes.NewReader(b))
return digest, err
}
func Size(c WithRawConfig) (int64, error) {
b, err := c.Raw()
if err != nil {
return -1, err
}
return int64(len(b)), nil
}

116
pkg/artifacts/file/file.go Normal file
View File

@@ -0,0 +1,116 @@
package file
import (
"context"
gv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/consts"
)
// interface guard
var _ artifacts.OCI = (*File)(nil)
// File implements the OCI interface for File API objects. API spec information is
// stored into the Path field.
type File struct {
Path string
computed bool
client *getter.Client
config artifacts.Config
blob gv1.Layer
manifest *gv1.Manifest
annotations map[string]string
}
func NewFile(path string, opts ...Option) *File {
client := getter.NewClient(getter.ClientOptions{})
f := &File{
client: client,
Path: path,
}
for _, opt := range opts {
opt(f)
}
return f
}
// Name is the name of the file's reference
func (f *File) Name(path string) string {
return f.client.Name(path)
}
func (f *File) MediaType() string {
return consts.OCIManifestSchema1
}
func (f *File) RawConfig() ([]byte, error) {
if err := f.compute(); err != nil {
return nil, err
}
return f.config.Raw()
}
func (f *File) Layers() ([]gv1.Layer, error) {
if err := f.compute(); err != nil {
return nil, err
}
var layers []gv1.Layer
layers = append(layers, f.blob)
return layers, nil
}
func (f *File) Manifest() (*gv1.Manifest, error) {
if err := f.compute(); err != nil {
return nil, err
}
return f.manifest, nil
}
func (f *File) compute() error {
if f.computed {
return nil
}
ctx := context.TODO()
blob, err := f.client.LayerFrom(ctx, f.Path)
if err != nil {
return err
}
layer, err := partial.Descriptor(blob)
if err != nil {
return err
}
cfg := f.client.Config(f.Path)
if cfg == nil {
cfg = f.client.Config(f.Path)
}
cfgDesc, err := partial.Descriptor(cfg)
if err != nil {
return err
}
m := &gv1.Manifest{
SchemaVersion: 2,
MediaType: gtypes.MediaType(f.MediaType()),
Config: *cfgDesc,
Layers: []gv1.Descriptor{*layer},
Annotations: f.annotations,
}
f.manifest = m
f.config = cfg
f.blob = blob
f.computed = true
return nil
}

166
pkg/artifacts/file/file_test.go Normal file
View File

@@ -0,0 +1,166 @@
package file_test
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/consts"
)
var (
filename = "myfile.yaml"
data = []byte(`data`)
ts *httptest.Server
tfs afero.Fs
mc *getter.Client
)
func TestMain(m *testing.M) {
teardown := setup()
defer teardown()
code := m.Run()
os.Exit(code)
}
func Test_file_Config(t *testing.T) {
tests := []struct {
name string
ref string
want string
wantErr bool
}{
{
name: "should properly type local file",
ref: filename,
want: consts.FileLocalConfigMediaType,
wantErr: false,
},
{
name: "should properly type remote file",
ref: ts.URL + "/" + filename,
want: consts.FileHttpConfigMediaType,
wantErr: false,
},
// TODO: Add directory test
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := file.NewFile(tt.ref, file.WithClient(mc))
f.MediaType()
m, err := f.Manifest()
if err != nil {
t.Fatal(err)
}
got := string(m.Config.MediaType)
if got != tt.want {
t.Errorf("unxpected mediatype; got %s, want %s", got, tt.want)
}
})
}
}
func Test_file_Layers(t *testing.T) {
tests := []struct {
name string
ref string
want []byte
wantErr bool
}{
{
name: "should load a local file and preserve contents",
ref: filename,
want: data,
wantErr: false,
},
{
name: "should load a remote file and preserve contents",
ref: ts.URL + "/" + filename,
want: data,
wantErr: false,
},
// TODO: Add directory test
}
for _, tt := range tests {
t.Run(tt.name, func(it *testing.T) {
f := file.NewFile(tt.ref, file.WithClient(mc))
layers, err := f.Layers()
if (err != nil) != tt.wantErr {
it.Fatalf("unexpected Layers() error: got %v, want %v", err, tt.wantErr)
}
rc, err := layers[0].Compressed()
if err != nil {
it.Fatal(err)
}
got, err := io.ReadAll(rc)
if err != nil {
it.Fatal(err)
}
if !bytes.Equal(got, tt.want) {
it.Fatalf("unexpected Layers(): got %v, want %v", layers, tt.want)
}
})
}
}
func setup() func() {
tfs = afero.NewMemMapFs()
afero.WriteFile(tfs, filename, data, 0644)
mf := &mockFile{File: getter.NewFile(), fs: tfs}
mockHttp := getter.NewHttp()
mhttp := afero.NewHttpFs(tfs)
fileserver := http.FileServer(mhttp.Dir("."))
http.Handle("/", fileserver)
ts = httptest.NewServer(fileserver)
mc = &getter.Client{
Options: getter.ClientOptions{},
Getters: map[string]getter.Getter{
"file": mf,
"http": mockHttp,
},
}
teardown := func() {
defer ts.Close()
}
return teardown
}
type mockFile struct {
*getter.File
fs afero.Fs
}
func (m mockFile) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return m.fs.Open(filepath.Join(u.Host, u.Path))
}
func (m mockFile) Detect(u *url.URL) bool {
fi, err := m.fs.Stat(filepath.Join(u.Host, u.Path))
if err != nil {
return false
}
return !fi.IsDir()
}

165
pkg/artifacts/file/getter/directory.go Normal file
View File

@@ -0,0 +1,165 @@
package getter
import (
"archive/tar"
"compress/gzip"
"context"
"io"
"net/url"
"os"
"path/filepath"
"time"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type directory struct {
*File
}
func NewDirectory() *directory {
return &directory{File: NewFile()}
}
func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
tmpfile, err := os.CreateTemp("", "hauler")
if err != nil {
return nil, err
}
digester := digest.Canonical.Digester()
zw := gzip.NewWriter(io.MultiWriter(tmpfile, digester.Hash()))
defer zw.Close()
tarDigester := digest.Canonical.Digester()
if err := tarDir(d.path(u), d.Name(u), io.MultiWriter(zw, tarDigester.Hash()), false); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
if err := tmpfile.Sync(); err != nil {
return nil, err
}
fi, err := os.Open(tmpfile.Name())
if err != nil {
return nil, err
}
// rc := &closer{
// t: io.TeeReader(tmpfile, fi),
// closes: []func() error{fi.Close, tmpfile.Close, zw.Close},
// }
return fi, nil
}
func (d directory) Detect(u *url.URL) bool {
if len(d.path(u)) == 0 {
return false
}
fi, err := os.Stat(d.path(u))
if err != nil {
return false
}
return fi.IsDir()
}
func (d directory) Config(u *url.URL) artifacts.Config {
c := &directoryConfig{
config{Reference: u.String()},
}
return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileDirectoryConfigMediaType))
}
type directoryConfig struct {
config `json:",inline,omitempty"`
}
func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error {
tw := tar.NewWriter(w)
defer tw.Close()
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Rename path
name, err := filepath.Rel(root, path)
if err != nil {
return err
}
name = filepath.Join(prefix, name)
name = filepath.ToSlash(name)
// Generate header
var link string
mode := info.Mode()
if mode&os.ModeSymlink != 0 {
if link, err = os.Readlink(path); err != nil {
return err
}
}
header, err := tar.FileInfoHeader(info, link)
if err != nil {
return errors.Wrap(err, path)
}
header.Name = name
header.Uid = 0
header.Gid = 0
header.Uname = ""
header.Gname = ""
if stripTimes {
header.ModTime = time.Time{}
header.AccessTime = time.Time{}
header.ChangeTime = time.Time{}
}
// Write file
if err := tw.WriteHeader(header); err != nil {
return errors.Wrap(err, "tar")
}
if mode.IsRegular() {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err := io.Copy(tw, file); err != nil {
return errors.Wrap(err, path)
}
}
return nil
}); err != nil {
return err
}
return nil
}
type closer struct {
t io.Reader
closes []func() error
}
func (c *closer) Read(p []byte) (n int, err error) {
return c.t.Read(p)
}
func (c *closer) Close() error {
var err error
for _, c := range c.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}

53
pkg/artifacts/file/getter/file.go Normal file
View File

@@ -0,0 +1,53 @@
package getter
import (
"context"
"io"
"net/url"
"os"
"path/filepath"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type File struct{}
func NewFile() *File {
return &File{}
}
func (f File) Name(u *url.URL) string {
return filepath.Base(f.path(u))
}
func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return os.Open(f.path(u))
}
func (f File) Detect(u *url.URL) bool {
if len(f.path(u)) == 0 {
return false
}
fi, err := os.Stat(f.path(u))
if err != nil {
return false
}
return !fi.IsDir()
}
func (f File) path(u *url.URL) string {
return filepath.Join(u.Host, u.Path)
}
func (f File) Config(u *url.URL) artifacts.Config {
c := &fileConfig{
config{Reference: u.String()},
}
return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileLocalConfigMediaType))
}
type fileConfig struct {
config `json:",inline,omitempty"`
}

148
pkg/artifacts/file/getter/getter.go Normal file
View File

@@ -0,0 +1,148 @@
package getter
import (
"context"
"fmt"
"io"
"net/url"
v1 "github.com/google/go-containerregistry/pkg/v1"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"oras.land/oras-go/pkg/content"
content2 "github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/layer"
)
type Client struct {
Getters map[string]Getter
Options ClientOptions
}
// ClientOptions provides options for the client
type ClientOptions struct {
NameOverride string
}
var (
ErrGetterTypeUnknown = errors.New("no getter type found matching reference")
)
type Getter interface {
Open(context.Context, *url.URL) (io.ReadCloser, error)
Detect(*url.URL) bool
Name(*url.URL) string
Config(*url.URL) content2.Config
}
func NewClient(opts ClientOptions) *Client {
defaults := map[string]Getter{
"file": NewFile(),
"directory": NewDirectory(),
"http": NewHttp(),
}
c := &Client{
Getters: defaults,
Options: opts,
}
return c
}
func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) {
u, err := url.Parse(source)
if err != nil {
return nil, err
}
g, err := c.getterFrom(u)
if err != nil {
if errors.Is(err, ErrGetterTypeUnknown) {
return nil, err
}
return nil, fmt.Errorf("create getter: %w", err)
}
opener := func() (io.ReadCloser, error) {
return g.Open(ctx, u)
}
annotations := make(map[string]string)
annotations[ocispec.AnnotationTitle] = c.Name(source)
switch g.(type) {
case *directory:
annotations[content.AnnotationUnpack] = "true"
}
l, err := layer.FromOpener(opener,
layer.WithMediaType(consts.FileLayerMediaType),
layer.WithAnnotations(annotations))
if err != nil {
return nil, err
}
return l, nil
}
func (c *Client) ContentFrom(ctx context.Context, source string) (io.ReadCloser, error) {
u, err := url.Parse(source)
if err != nil {
return nil, fmt.Errorf("parse source %s: %w", source, err)
}
g, err := c.getterFrom(u)
if err != nil {
if errors.Is(err, ErrGetterTypeUnknown) {
return nil, err
}
return nil, fmt.Errorf("create getter: %w", err)
}
return g.Open(ctx, u)
}
func (c *Client) getterFrom(srcUrl *url.URL) (Getter, error) {
for _, g := range c.Getters {
if g.Detect(srcUrl) {
return g, nil
}
}
return nil, errors.Wrapf(ErrGetterTypeUnknown, "source %s", srcUrl.String())
}
func (c *Client) Name(source string) string {
if c.Options.NameOverride != "" {
return c.Options.NameOverride
}
u, err := url.Parse(source)
if err != nil {
return source
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Name(u)
}
}
return source
}
func (c *Client) Config(source string) content2.Config {
u, err := url.Parse(source)
if err != nil {
return nil
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Config(u)
}
}
return nil
}
type config struct {
Reference string `json:"reference"`
Annotations map[string]string `json:"annotations,omitempty"`
}

139
pkg/artifacts/file/getter/getter_test.go Normal file
View File

@@ -0,0 +1,139 @@
package getter_test
import (
"net/url"
"os"
"path/filepath"
"testing"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
)
func TestClient_Detect(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should identify a file",
args: args{
source: fileWithExt,
},
want: "file",
},
{
name: "should identify a directory",
args: args{
source: rootDir,
},
want: "directory",
},
{
name: "should identify a http",
args: args{
source: "http://my.cool.website",
},
want: "http",
},
{
name: "should identify a http",
args: args{
source: "https://my.cool.website",
},
want: "http",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := identify(c, tt.args.source); got != tt.want {
t.Errorf("identify() = %v, want %v", got, tt.want)
}
})
}
}
func identify(c *getter.Client, source string) string {
u, _ := url.Parse(source)
for t, g := range c.Getters {
if g.Detect(u) {
return t
}
}
return ""
}
func TestClient_Name(t *testing.T) {
teardown := setup(t)
defer teardown()
type args struct {
source string
opts getter.ClientOptions
}
tests := []struct {
name string
args args
want string
}{
{
name: "should correctly name a file with an extension",
args: args{
source: fileWithExt,
opts: getter.ClientOptions{},
},
want: "file.yaml",
},
{
name: "should correctly name a directory",
args: args{
source: rootDir,
opts: getter.ClientOptions{},
},
want: rootDir,
},
{
name: "should correctly override a files name",
args: args{
source: fileWithExt,
opts: getter.ClientOptions{NameOverride: "myfile"},
},
want: "myfile",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := getter.NewClient(tt.args.opts)
if got := c.Name(tt.args.source); got != tt.want {
t.Errorf("Name() = %v, want %v", got, tt.want)
}
})
}
}
var (
rootDir = "gettertests"
fileWithExt = filepath.Join(rootDir, "file.yaml")
)
func setup(t *testing.T) func() {
if err := os.MkdirAll(rootDir, os.ModePerm); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil {
t.Fatal(err)
}
return func() {
os.RemoveAll(rootDir)
}
}

67
pkg/artifacts/file/getter/https.go Normal file
View File

@@ -0,0 +1,67 @@
package getter
import (
"context"
"io"
"mime"
"net/http"
"net/url"
"path/filepath"
"strings"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Http struct{}
func NewHttp() *Http {
return &Http{}
}
func (h Http) Name(u *url.URL) string {
resp, err := http.Head(u.String())
if err != nil {
return ""
}
contentType := resp.Header.Get("Content-Type")
for _, v := range strings.Split(contentType, ",") {
t, _, err := mime.ParseMediaType(v)
if err != nil {
break
}
// TODO: Identify known mimetypes for hints at a filename
_ = t
}
// TODO: Not this
return filepath.Base(u.String())
}
func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
resp, err := http.Get(u.String())
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (h Http) Detect(u *url.URL) bool {
switch u.Scheme {
case "http", "https":
return true
}
return false
}
func (h *Http) Config(u *url.URL) artifacts.Config {
c := &httpConfig{
config{Reference: u.String()},
}
return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileHttpConfigMediaType))
}
type httpConfig struct {
config `json:",inline,omitempty"`
}

26
pkg/artifacts/file/options.go Normal file
View File

@@ -0,0 +1,26 @@
package file
import (
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
)
type Option func(*File)
func WithClient(c *getter.Client) Option {
return func(f *File) {
f.client = c
}
}
func WithConfig(obj interface{}, mediaType string) Option {
return func(f *File) {
f.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType))
}
}
func WithAnnotations(m map[string]string) Option {
return func(f *File) {
f.annotations = m
}
}

53
pkg/artifacts/image/image.go Normal file
View File

@@ -0,0 +1,53 @@
package image
import (
"github.com/google/go-containerregistry/pkg/authn"
gname "github.com/google/go-containerregistry/pkg/name"
gv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/rancherfederal/hauler/pkg/artifacts"
)
var _ artifacts.OCI = (*Image)(nil)
func (i *Image) MediaType() string {
mt, err := i.Image.MediaType()
if err != nil {
return ""
}
return string(mt)
}
func (i *Image) RawConfig() ([]byte, error) {
return i.RawConfigFile()
}
// Image implements the OCI interface for Image API objects. API spec information
// is stored into the Name field.
type Image struct {
Name string
gv1.Image
}
func NewImage(name string, opts ...remote.Option) (*Image, error) {
r, err := gname.ParseReference(name)
if err != nil {
return nil, err
}
defaultOpts := []remote.Option{
remote.WithAuthFromKeychain(authn.DefaultKeychain),
}
opts = append(opts, defaultOpts...)
img, err := remote.Image(r, opts...)
if err != nil {
return nil, err
}
return &Image{
Name: name,
Image: img,
}, nil
}

1
pkg/artifacts/image/image_test.go Normal file
View File

@@ -0,0 +1 @@
package image_test

78
pkg/artifacts/memory/memory.go Normal file
View File

@@ -0,0 +1,78 @@
package memory
import (
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ artifacts.OCI = (*Memory)(nil)
// Memory implements the OCI interface for a generic set of bytes stored in memory.
type Memory struct {
blob v1.Layer
annotations map[string]string
config artifacts.Config
}
type defaultConfig struct {
MediaType string `json:"mediaType,omitempty"`
}
func NewMemory(data []byte, mt string, opts ...Option) *Memory {
blob := static.NewLayer(data, types.MediaType(mt))
cfg := defaultConfig{MediaType: consts.MemoryConfigMediaType}
m := &Memory{
blob: blob,
config: artifacts.ToConfig(cfg),
}
for _, opt := range opts {
opt(m)
}
return m
}
func (m *Memory) MediaType() string {
return consts.OCIManifestSchema1
}
func (m *Memory) Manifest() (*v1.Manifest, error) {
layer, err := partial.Descriptor(m.blob)
if err != nil {
return nil, err
}
cfgDesc, err := partial.Descriptor(m.config)
if err != nil {
return nil, err
}
manifest := &v1.Manifest{
SchemaVersion: 2,
MediaType: types.MediaType(m.MediaType()),
Config: *cfgDesc,
Layers: []v1.Descriptor{*layer},
Annotations: m.annotations,
}
return manifest, nil
}
func (m *Memory) RawConfig() ([]byte, error) {
if m.config == nil {
return []byte(`{}`), nil
}
return m.config.Raw()
}
func (m *Memory) Layers() ([]v1.Layer, error) {
var layers []v1.Layer
layers = append(layers, m.blob)
return layers, nil
}

61
pkg/artifacts/memory/memory_test.go Normal file
View File

@@ -0,0 +1,61 @@
package memory_test
import (
"math/rand"
"testing"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/opencontainers/go-digest"
"github.com/rancherfederal/hauler/pkg/artifacts/memory"
)
func TestMemory_Layers(t *testing.T) {
tests := []struct {
name string
want *v1.Manifest
wantErr bool
}{
{
name: "should preserve content",
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data, m := setup(t)
layers, err := m.Layers()
if err != nil {
t.Fatal(err)
}
if len(layers) != 1 {
t.Fatalf("Expected 1 layer, got %d", len(layers))
}
h, err := layers[0].Digest()
if err != nil {
t.Fatal(err)
}
d := digest.FromBytes(data)
if d.String() != h.String() {
t.Fatalf("bytes do not match, got %s, expected %s", h.String(), d.String())
}
})
}
}
func setup(t *testing.T) ([]byte, *memory.Memory) {
block := make([]byte, 2048)
_, err := rand.Read(block)
if err != nil {
t.Fatal(err)
}
mem := memory.NewMemory(block, "random")
return block, mem
}

17
pkg/artifacts/memory/options.go Normal file
View File

@@ -0,0 +1,17 @@
package memory
import "github.com/rancherfederal/hauler/pkg/artifacts"
type Option func(*Memory)
func WithConfig(obj interface{}, mediaType string) Option {
return func(m *Memory) {
m.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType))
}
}
func WithAnnotations(annotations map[string]string) Option {
return func(m *Memory) {
m.annotations = annotations
}
}

21
pkg/artifacts/ocis.go Normal file
View File

@@ -0,0 +1,21 @@
package artifacts
import "github.com/google/go-containerregistry/pkg/v1"
// OCI is the bare minimum we need to represent an artifact in an oci layout
// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality
// This specific implementation fully encapsulates v1.Layer's within a more generic form
type OCI interface {
MediaType() string
Manifest() (*v1.Manifest, error)
RawConfig() ([]byte, error)
Layers() ([]v1.Layer, error)
}
type OCICollection interface {
// Contents returns the list of contents in the collection
Contents() (map[string]OCI, error)
}

4
pkg/collection/chart/chart.go
View File

@@ -1,8 +1,8 @@
package chart
import (
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"

6
pkg/collection/imagetxt/imagetxt.go
View File

@@ -12,9 +12,9 @@ import (
"github.com/rancherfederal/hauler/pkg/log"
"github.com/google/go-containerregistry/pkg/name"
artifact "github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
artifact "github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
)
type ImageTxt struct {

4
pkg/collection/imagetxt/imagetxt_test.go
View File

@@ -8,8 +8,8 @@ import (
"os"
"testing"
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
)
var (

8
pkg/collection/k3s/k3s.go
View File

@@ -10,12 +10,12 @@ import (
"path"
"strings"
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"github.com/rancherfederal/ocil/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/reference"
)

50
pkg/consts/consts.go Normal file
View File

@@ -0,0 +1,50 @@
package consts
const (
OCIManifestSchema1 = "application/vnd.oci.image.manifest.v1+json"
DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json"
DockerManifestListSchema2 = "application/vnd.docker.distribution.manifest.list.v2+json"
DockerConfigJSON = "application/vnd.docker.container.image.v1+json"
DockerLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip"
DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar"
OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip"
// ChartConfigMediaType is the reserved media type for the Helm chart manifest config
ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
// ChartLayerMediaType is the reserved media type for Helm chart package content
ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"
// ProvLayerMediaType is the reserved media type for Helm chart provenance files
ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov"
// FileLayerMediaType is the reserved media type for File content layers
FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1"
// FileLocalConfigMediaType is the reserved media type for File config
FileLocalConfigMediaType = "application/vnd.content.hauler.file.local.config.v1+json"
FileDirectoryConfigMediaType = "application/vnd.content.hauler.file.directory.config.v1+json"
FileHttpConfigMediaType = "application/vnd.content.hauler.file.http.config.v1+json"
// MemoryConfigMediaType
MemoryConfigMediaType = "application/vnd.content.hauler.memory.config.v1+json"
// WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers
WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm"
// WasmConfigMediaType is the reserved media type for WASM configs
WasmConfigMediaType = "application/vnd.wasm.config.v1+json"
UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json"
UnknownLayer = "application/vnd.content.hauler.unknown.layer"
OCIVendorPrefix = "vnd.oci"
DockerVendorPrefix = "vnd.docker"
HaulerVendorPrefix = "vnd.hauler"
OCIImageIndexFile = "index.json"
KindAnnotationName = "kind"
KindAnnotation = "dev.cosignproject.cosign/image"
)

9
pkg/content/chart/chart.go
View File

@@ -14,15 +14,15 @@ import (
"github.com/google/go-containerregistry/pkg/v1/partial"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"github.com/rancherfederal/ocil/pkg/layer"
"github.com/rancherfederal/hauler/pkg/layer"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ artifacts.OCI = (*Chart)(nil)
@@ -137,7 +137,8 @@ func (h *Chart) RawChartData() ([]byte, error) {
}
// chartData loads the chart contents into memory and returns a NopCloser for the contents
// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB
//
// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB
func (h *Chart) chartData() (gv1.Layer, error) {
info, err := os.Stat(h.path)
if err != nil {

2
pkg/content/chart/chart_test.go
View File

@@ -10,7 +10,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content/chart"
)

272
pkg/content/oci.go Normal file
View File

@@ -0,0 +1,272 @@
package content
import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"sync"
ccontent "github.com/containerd/containerd/content"
"github.com/containerd/containerd/remotes"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/content"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ target.Target = (*OCI)(nil)
type OCI struct {
root string
index *ocispec.Index
nameMap *sync.Map // map[string]ocispec.Descriptor
}
func NewOCI(root string) (*OCI, error) {
o := &OCI{
root: root,
nameMap: &sync.Map{},
}
return o, nil
}
// AddIndex adds a descriptor to the index and updates it
//
// The descriptor must use AnnotationRefName to identify itself
func (o *OCI) AddIndex(desc ocispec.Descriptor) error {
if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
return fmt.Errorf("descriptor must contain a reference from the annotation: %s", ocispec.AnnotationRefName)
}
key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName])
o.nameMap.Store(key, desc)
return o.SaveIndex()
}
// LoadIndex will load the index from disk
func (o *OCI) LoadIndex() error {
path := o.path(consts.OCIImageIndexFile)
idx, err := os.Open(path)
if err != nil {
if !os.IsNotExist(err) {
return err
}
o.index = &ocispec.Index{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
}
return nil
}
defer idx.Close()
if err := json.NewDecoder(idx).Decode(&o.index); err != nil {
return err
}
for _, desc := range o.index.Manifests {
key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName])
if strings.TrimSpace(key) != "--" {
o.nameMap.Store(key, desc)
}
}
return nil
}
// SaveIndex will update the index on disk
func (o *OCI) SaveIndex() error {
var descs []ocispec.Descriptor
o.nameMap.Range(func(name, desc interface{}) bool {
n := desc.(ocispec.Descriptor).Annotations[ocispec.AnnotationRefName]
d := desc.(ocispec.Descriptor)
if d.Annotations == nil {
d.Annotations = make(map[string]string)
}
d.Annotations[ocispec.AnnotationRefName] = n
descs = append(descs, d)
return true
})
// sort index to ensure that images come before any signatures and attestations.
sort.SliceStable(descs, func(i, j int) bool {
kindI := descs[i].Annotations["kind"]
kindJ := descs[j].Annotations["kind"]
// Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top.
if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) {
return true
} else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) {
return false
}
return false // Default: maintain the order.
})
o.index.Manifests = descs
data, err := json.Marshal(o.index)
if err != nil {
return err
}
return os.WriteFile(o.path(consts.OCIImageIndexFile), data, 0644)
}
// Resolve attempts to resolve the reference into a name and descriptor.
//
// The argument `ref` should be a scheme-less URI representing the remote.
// Structurally, it has a host and path. The "host" can be used to directly
// reference a specific host or be matched against a specific handler.
//
// The returned name should be used to identify the referenced entity.
// Dependending on the remote namespace, this may be immutable or mutable.
// While the name may differ from ref, it should itself be a valid ref.
//
// If the resolution fails, an error will be returned.
func (o *OCI) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
if err := o.LoadIndex(); err != nil {
return "", ocispec.Descriptor{}, err
}
d, ok := o.nameMap.Load(ref)
if !ok {
return "", ocispec.Descriptor{}, err
}
desc = d.(ocispec.Descriptor)
return ref, desc, nil
}
// Fetcher returns a new fetcher for the provided reference.
// All content fetched from the returned fetcher will be
// from the namespace referred to by ref.
func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) {
if err := o.LoadIndex(); err != nil {
return nil, err
}
if _, ok := o.nameMap.Load(ref); !ok {
return nil, nil
}
return o, nil
}
func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
readerAt, err := o.blobReaderAt(desc)
if err != nil {
return nil, err
}
return readerAt, nil
}
// Pusher returns a new pusher for the provided reference
// The returned Pusher should satisfy content.Ingester and concurrent attempts
// to push the same blob using the Ingester API should result in ErrUnavailable.
func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) {
if err := o.LoadIndex(); err != nil {
return nil, err
}
var baseRef, hash string
parts := strings.SplitN(ref, "@", 2)
baseRef = parts[0]
if len(parts) > 1 {
hash = parts[1]
}
return &ociPusher{
oci: o,
ref: baseRef,
digest: hash,
}, nil
}
func (o *OCI) Walk(fn func(reference string, desc ocispec.Descriptor) error) error {
if err := o.LoadIndex(); err != nil {
return err
}
var errst []string
o.nameMap.Range(func(key, value interface{}) bool {
if err := fn(key.(string), value.(ocispec.Descriptor)); err != nil {
errst = append(errst, err.Error())
}
return true
})
if errst != nil {
return fmt.Errorf(strings.Join(errst, "; "))
}
return nil
}
func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) {
blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex())
if err != nil {
return nil, err
}
return os.Open(blobPath)
}
func (o *OCI) blobWriterAt(desc ocispec.Descriptor) (*os.File, error) {
blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex())
if err != nil {
return nil, err
}
return os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
}
func (o *OCI) ensureBlob(alg string, hex string) (string, error) {
dir := o.path("blobs", alg)
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return "", err
}
return filepath.Join(dir, hex), nil
}
func (o *OCI) path(elem ...string) string {
complete := []string{string(o.root)}
return filepath.Join(append(complete, elem...)...)
}
type ociPusher struct {
oci *OCI
ref string
digest string
}
// Push returns a content writer for the given resource identified
// by the descriptor.
func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Writer, error) {
switch d.MediaType {
case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex, consts.DockerManifestSchema2, consts.DockerManifestListSchema2:
// if the hash of the content matches that which was provided as the hash for the root, mark it
if p.digest != "" && p.digest == d.Digest.String() {
if err := p.oci.LoadIndex(); err != nil {
return nil, err
}
p.oci.nameMap.Store(p.ref, d)
if err := p.oci.SaveIndex(); err != nil {
return nil, err
}
}
}
blobPath, err := p.oci.ensureBlob(d.Digest.Algorithm().String(), d.Digest.Hex())
if err != nil {
return nil, err
}
if _, err := os.Stat(blobPath); err == nil {
// file already exists, discard (but validate digest)
return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil
}
f, err := os.Create(blobPath)
if err != nil {
return nil, err
}
w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest))
return w, nil
}

286
pkg/cosign/cosign.go Normal file
View File

@@ -0,0 +1,286 @@
package cosign
import (
"fmt"
"io"
"net/http"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"context"
"strings"
"encoding/json"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/reference"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
)
// VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign.
func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string) error {
// Ensure that the cosign binary is installed or download it if needed
cosignBinaryPath, err := ensureCosignBinary(ctx, s)
if err != nil {
return err
}
// Command to verify the signature using Cosign.
cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, ref)
// Run the command and capture its output.
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error verifying signature: %v, output: %s", err, output)
}
return nil
}
// SaveImage saves image and any signatures/attestations to the store.
func SaveImage(ctx context.Context, s *store.Layout, ref string) error {
// Ensure that the cosign binary is installed or download it if needed
cosignBinaryPath, err := ensureCosignBinary(ctx, s)
if err != nil {
return err
}
// Command to verify the signature using Cosign.
cmd := exec.Command(cosignBinaryPath, "save", ref, "--dir", s.Root)
// Run the command and capture its output.
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error adding image to store: %v, output: %s", err, output)
}
return nil
}
// LoadImage loads store to a remote registry.
func LoadImage(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error {
//Ensure that the cosign binary is installed or download it if needed
cosignBinaryPath, err := ensureCosignBinary(ctx, s)
if err != nil {
return err
}
// Command to verify the signature using Cosign.
cmd := exec.Command(cosignBinaryPath, "load", "--registry", registry, "--dir", s.Root)
// Conditionally add extra registry flags.
if ropts.Insecure {
cmd.Args = append(cmd.Args, "--allow-insecure-registry=true")
}
if ropts.PlainHTTP {
cmd.Args = append(cmd.Args, "--allow-http-registry=true")
}
// Run the command and capture its output.
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error adding image to store: %v, output: %s", err, output)
}
return nil
}
// ensureCosignBinary checks if the cosign binary exists in the specified directory and installs it if not.
func ensureCosignBinary(ctx context.Context, s *store.Layout) (string, error) {
l := log.FromContext(ctx)
// Get the current user's information
currentUser, err := user.Current()
if err != nil {
return "", fmt.Errorf("Error: %v\n", err)
}
// Get the user's home directory
homeDir := currentUser.HomeDir
// Construct the path to the .hauler directory
haulerDir := filepath.Join(homeDir, ".hauler")
// Create the .hauler directory if it doesn't exist
if _, err := os.Stat(haulerDir); os.IsNotExist(err) {
// .hauler directory does not exist, create it
if err := os.MkdirAll(haulerDir, 0755); err != nil {
return "", fmt.Errorf("Error creating .hauler directory: %v\n", err)
}
l.Infof("Created .hauler directory at: %s", haulerDir)
}
// Check if the cosign binary exists in the specified directory.
binaryPath := filepath.Join(haulerDir, "cosign")
_, err = os.Stat(binaryPath)
if err == nil {
// Cosign binary is already installed in the specified directory.
return binaryPath, nil
}
// Cosign binary is not found.
l.Infof("Cosign binary not found. Checking to see if it exists in the store...")
// grab binary from store if it exists, otherwise try to download it from GitHub.
// if the binary has to be downloaded, then automatically add it to the store afterwards.
err = copyCosignFromStore(ctx, s, haulerDir)
if err != nil {
l.Warnf("%s", err)
err = downloadCosign(ctx, haulerDir)
if err != nil {
return "", err
}
err = addCosignToStore(ctx, s, binaryPath)
if err != nil {
return "", err
}
}
return binaryPath, nil
}
// used to check if the cosign binary is in the store and if so copy it to the .hauler directory
func copyCosignFromStore(ctx context.Context, s *store.Layout, destDir string) error {
l := log.FromContext(ctx)
ref := "hauler/cosign:latest"
r, err := reference.Parse(ref)
if err != nil {
return err
}
found := false
if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
if !strings.Contains(reference, r.Name()) {
return nil
}
found = true
rc, err := s.Fetch(ctx, desc)
if err != nil {
return err
}
defer rc.Close()
var m ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
}
mapperStore, err := mapper.FromManifest(m, destDir)
if err != nil {
return err
}
pushedDesc, err := s.Copy(ctx, reference, mapperStore, "")
if err != nil {
return err
}
l.Infof("extracted [%s] from store with digest [%s]", ref, pushedDesc.Digest.String())
return nil
}); err != nil {
return err
}
if !found {
return fmt.Errorf("Reference [%s] not found in store. Hauler will attempt to download it from Github.", ref)
}
return nil
}
// adds the cosign binary to the store.
// this is to help with airgapped situations where you cannot access the internet.
func addCosignToStore(ctx context.Context, s *store.Layout, binaryPath string) error {
l := log.FromContext(ctx)
fi := v1alpha1.File{
Path: binaryPath,
}
copts := getter.ClientOptions{
NameOverride: fi.Name,
}
f := file.NewFile(fi.Path, file.WithClient(getter.NewClient(copts)))
ref, err := reference.NewTagged(f.Name(fi.Path), reference.DefaultTag)
if err != nil {
return err
}
desc, err := s.AddOCI(ctx, f, ref.Name())
if err != nil {
return err
}
l.Infof("added 'file' to store at [%s], with digest [%s]", ref.Name(), desc.Digest.String())
return nil
}
// used to check if the cosign binary is in the store and if so copy it to the .hauler directory
func downloadCosign(ctx context.Context, haulerDir string) error {
l := log.FromContext(ctx)
// Define the GitHub release URL and architecture-specific binary name.
releaseURL := "https://github.com/rancher-government-solutions/cosign/releases/latest/download"
// Determine the architecture and add it to the binary name.
arch := runtime.GOARCH
rOS := runtime.GOOS
binaryName := "cosign"
if rOS == "windows" {
binaryName = fmt.Sprintf("cosign-%s-%s.exe", rOS, arch)
} else {
binaryName = fmt.Sprintf("cosign-%s-%s", rOS, arch)
}
// Download the binary.
downloadURL := fmt.Sprintf("%s/%s", releaseURL, binaryName)
resp, err := http.Get(downloadURL)
if err != nil {
return fmt.Errorf("error downloading cosign binary: %v", err)
}
defer resp.Body.Close()
// Create the cosign binary file in the specified directory.
binaryFile, err := os.Create(filepath.Join(haulerDir, binaryName))
if err != nil {
return fmt.Errorf("error creating cosign binary: %v", err)
}
defer binaryFile.Close()
// Copy the downloaded binary to the file.
_, err = io.Copy(binaryFile, resp.Body)
if err != nil {
return fmt.Errorf("error saving cosign binary: %v", err)
}
// Make the binary executable.
if err := os.Chmod(binaryFile.Name(), 0755); err != nil {
return fmt.Errorf("error setting executable permission: %v", err)
}
// Rename the binary to "cosign"
oldBinaryPath := filepath.Join(haulerDir, binaryName)
newBinaryPath := filepath.Join(haulerDir, "cosign")
if err := os.Rename(oldBinaryPath, newBinaryPath); err != nil {
return fmt.Errorf("error renaming cosign binary: %v", err)
}
l.Infof("Cosign binary downloaded and installed to %s", haulerDir)
return nil
}

106
pkg/layer/cache.go Normal file
View File

@@ -0,0 +1,106 @@
package layer
import (
"errors"
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifacts"
)
/*
This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache
*/
type Cache interface {
Put(v1.Layer) (v1.Layer, error)
Get(v1.Hash) (v1.Layer, error)
}
var ErrLayerNotFound = errors.New("layer not found")
type oci struct {
artifacts.OCI
c Cache
}
func OCICache(o artifacts.OCI, c Cache) artifacts.OCI {
return &oci{
OCI: o,
c: c,
}
}
func (o *oci) Layers() ([]v1.Layer, error) {
ls, err := o.OCI.Layers()
if err != nil {
return nil, err
}
var out []v1.Layer
for _, l := range ls {
out = append(out, &lazyLayer{inner: l, c: o.c})
}
return out, nil
}
type lazyLayer struct {
inner v1.Layer
c Cache
}
func (l *lazyLayer) Compressed() (io.ReadCloser, error) {
digest, err := l.inner.Digest()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(digest)
if err != nil {
return nil, err
}
return layer.Compressed()
}
func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) {
diffID, err := l.inner.DiffID()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(diffID)
if err != nil {
return nil, err
}
return layer.Uncompressed()
}
func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) {
var layer v1.Layer
if cl, err := l.c.Get(h); err == nil {
layer = cl
} else if err == ErrLayerNotFound {
rl, err := l.c.Put(l.inner)
if err != nil {
return nil, err
}
layer = rl
} else {
return nil, err
}
return layer, nil
}
func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() }
func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() }

118
pkg/layer/filesystem.go Normal file
View File

@@ -0,0 +1,118 @@
package layer
import (
"io"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
type fs struct {
root string
}
func NewFilesystemCache(root string) Cache {
return &fs{root: root}
}
func (f *fs) Put(l v1.Layer) (v1.Layer, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
diffID, err := l.DiffID()
if err != nil {
return nil, err
}
return &cachedLayer{
Layer: l,
root: f.root,
digest: digest,
diffID: diffID,
}, nil
}
func (f *fs) Get(h v1.Hash) (v1.Layer, error) {
opener := f.open(h)
l, err := FromOpener(opener)
if os.IsNotExist(err) {
return nil, ErrLayerNotFound
}
return l, err
}
func (f *fs) open(h v1.Hash) Opener {
return func() (io.ReadCloser, error) {
return os.Open(layerpath(f.root, h))
}
}
type cachedLayer struct {
v1.Layer
root string
digest, diffID v1.Hash
}
func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) {
lp := layerpath(l.root, h)
if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil {
return nil, err
}
return os.Create(lp)
}
func (l *cachedLayer) Compressed() (io.ReadCloser, error) {
f, err := l.create(l.digest)
if err != nil {
return nil, nil
}
rc, err := l.Layer.Compressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) {
f, err := l.create(l.diffID)
if err != nil {
return nil, err
}
rc, err := l.Layer.Uncompressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func layerpath(root string, h v1.Hash) string {
return filepath.Join(root, h.Algorithm, h.Hex)
}
type readcloser struct {
t io.Reader
closes []func() error
}
func (rc *readcloser) Read(b []byte) (int, error) {
return rc.t.Read(b)
}
func (rc *readcloser) Close() error {
var err error
for _, c := range rc.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}

127
pkg/layer/layer.go Normal file
View File

@@ -0,0 +1,127 @@
package layer
import (
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Opener func() (io.ReadCloser, error)
func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) {
var err error
layer := &layer{
mediaType: consts.UnknownLayer,
annotations: make(map[string]string, 1),
}
layer.uncompressedOpener = opener
layer.compressedOpener = func() (io.ReadCloser, error) {
rc, err := opener()
if err != nil {
return nil, err
}
return rc, nil
}
for _, opt := range opts {
opt(layer)
}
if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil {
return nil, err
}
if layer.diffID, _, err = compute(layer.compressedOpener); err != nil {
return nil, err
}
return layer, nil
}
func compute(opener Opener) (v1.Hash, int64, error) {
rc, err := opener()
if err != nil {
return v1.Hash{}, 0, err
}
defer rc.Close()
return v1.SHA256(rc)
}
type Option func(*layer)
func WithMediaType(mt string) Option {
return func(l *layer) {
l.mediaType = mt
}
}
func WithAnnotations(annotations map[string]string) Option {
return func(l *layer) {
if l.annotations == nil {
l.annotations = make(map[string]string)
}
l.annotations = annotations
}
}
type layer struct {
digest v1.Hash
diffID v1.Hash
size int64
compressedOpener Opener
uncompressedOpener Opener
mediaType string
annotations map[string]string
urls []string
}
func (l layer) Descriptor() (*v1.Descriptor, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
mt, err := l.MediaType()
if err != nil {
return nil, err
}
return &v1.Descriptor{
MediaType: mt,
Size: l.size,
Digest: digest,
Annotations: l.annotations,
URLs: l.urls,
// TODO: Allow platforms
Platform: nil,
}, nil
}
func (l layer) Digest() (v1.Hash, error) {
return l.digest, nil
}
func (l layer) DiffID() (v1.Hash, error) {
return l.diffID, nil
}
func (l layer) Compressed() (io.ReadCloser, error) {
return l.compressedOpener()
}
func (l layer) Uncompressed() (io.ReadCloser, error) {
return l.uncompressedOpener()
}
func (l layer) Size() (int64, error) {
return l.size, nil
}
func (l layer) MediaType() (gtypes.MediaType, error) {
return gtypes.MediaType(l.mediaType), nil
}

262
pkg/store/store.go Normal file
View File

@@ -0,0 +1,262 @@
package store
import (
"context"
"encoding/json"
"io"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/pkg/oras"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/layer"
)
type Layout struct {
*content.OCI
Root string
cache layer.Cache
}
type Options func(*Layout)
func WithCache(c layer.Cache) Options {
return func(l *Layout) {
l.cache = c
}
}
func NewLayout(rootdir string, opts ...Options) (*Layout, error) {
ociStore, err := content.NewOCI(rootdir)
if err != nil {
return nil, err
}
if err := ociStore.LoadIndex(); err != nil {
return nil, err
}
l := &Layout{
Root: rootdir,
OCI: ociStore,
}
for _, opt := range opts {
opt(l)
}
return l, nil
}
// AddOCI adds an artifacts.OCI to the store
//
// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once
// saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use
// strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the
// future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout.
func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (ocispec.Descriptor, error) {
if l.cache != nil {
cached := layer.OCICache(oci, l.cache)
oci = cached
}
// Write manifest blob
m, err := oci.Manifest()
if err != nil {
return ocispec.Descriptor{}, err
}
mdata, err := json.Marshal(m)
if err != nil {
return ocispec.Descriptor{}, err
}
if err := l.writeBlobData(mdata); err != nil {
return ocispec.Descriptor{}, err
}
// Write config blob
cdata, err := oci.RawConfig()
if err != nil {
return ocispec.Descriptor{}, err
}
static.NewLayer(cdata, "")
if err := l.writeBlobData(cdata); err != nil {
return ocispec.Descriptor{}, err
}
// write blob layers concurrently
layers, err := oci.Layers()
if err != nil {
return ocispec.Descriptor{}, err
}
var g errgroup.Group
for _, lyr := range layers {
lyr := lyr
g.Go(func() error {
return l.writeLayer(lyr)
})
}
if err := g.Wait(); err != nil {
return ocispec.Descriptor{}, err
}
// Build index
idx := ocispec.Descriptor{
MediaType: string(m.MediaType),
Digest: digest.FromBytes(mdata),
Size: int64(len(mdata)),
Annotations: map[string]string{
consts.KindAnnotationName: consts.KindAnnotation,
ocispec.AnnotationRefName: ref,
},
URLs: nil,
Platform: nil,
}
return idx, l.OCI.AddIndex(idx)
}
// AddOCICollection .
func (l *Layout) AddOCICollection(ctx context.Context, collection artifacts.OCICollection) ([]ocispec.Descriptor, error) {
cnts, err := collection.Contents()
if err != nil {
return nil, err
}
var descs []ocispec.Descriptor
for ref, oci := range cnts {
desc, err := l.AddOCI(ctx, oci, ref)
if err != nil {
return nil, err
}
descs = append(descs, desc)
}
return descs, nil
}
// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting oci-layout content
//
// This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents
// To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly deletes oci-layout content only
func (l *Layout) Flush(ctx context.Context) error {
blobs := filepath.Join(l.Root, "blobs")
if err := os.RemoveAll(blobs); err != nil {
return err
}
index := filepath.Join(l.Root, "index.json")
if err := os.RemoveAll(index); err != nil {
return err
}
layout := filepath.Join(l.Root, "oci-layout")
if err := os.RemoveAll(layout); err != nil {
return err
}
return nil
}
// Copy will copy a given reference to a given target.Target
//
// This is essentially a wrapper around oras.Copy, but locked to this content store
func (l *Layout) Copy(ctx context.Context, ref string, to target.Target, toRef string) (ocispec.Descriptor, error) {
return oras.Copy(ctx, l.OCI, ref, to, toRef,
oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2, consts.DockerManifestListSchema2))
}
// CopyAll performs bulk copy operations on the stores oci layout to a provided target.Target
func (l *Layout) CopyAll(ctx context.Context, to target.Target, toMapper func(string) (string, error)) ([]ocispec.Descriptor, error) {
var descs []ocispec.Descriptor
err := l.OCI.Walk(func(reference string, desc ocispec.Descriptor) error {
toRef := ""
if toMapper != nil {
tr, err := toMapper(reference)
if err != nil {
return err
}
toRef = tr
}
desc, err := l.Copy(ctx, reference, to, toRef)
if err != nil {
return err
}
descs = append(descs, desc)
return nil
})
if err != nil {
return nil, err
}
return descs, nil
}
// Identify is a helper function that will identify a human-readable content type given a descriptor
func (l *Layout) Identify(ctx context.Context, desc ocispec.Descriptor) string {
rc, err := l.OCI.Fetch(ctx, desc)
if err != nil {
return ""
}
defer rc.Close()
m := struct {
Config struct {
MediaType string `json:"mediaType"`
} `json:"config"`
}{}
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return ""
}
return m.Config.MediaType
}
func (l *Layout) writeBlobData(data []byte) error {
blob := static.NewLayer(data, "") // NOTE: MediaType isn't actually used in the writing
return l.writeLayer(blob)
}
func (l *Layout) writeLayer(layer v1.Layer) error {
d, err := layer.Digest()
if err != nil {
return err
}
r, err := layer.Compressed()
if err != nil {
return err
}
dir := filepath.Join(l.Root, "blobs", d.Algorithm)
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return err
}
blobPath := filepath.Join(dir, d.Hex)
// Skip entirely if something exists, assume layer is present already
if _, err := os.Stat(blobPath); err == nil {
return nil
}
w, err := os.Create(blobPath)
if err != nil {
return err
}
defer w.Close()
_, err = io.Copy(w, r)
return err
}

105
pkg/store/store_test.go Normal file
View File

@@ -0,0 +1,105 @@
package store_test
import (
"context"
"os"
"testing"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/random"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/store"
)
var (
ctx context.Context
root string
)
func TestLayout_AddOCI(t *testing.T) {
teardown := setup(t)
defer teardown()
type args struct {
ref string
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "",
args: args{
ref: "hello/world:v1",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, err := store.NewLayout(root)
if (err != nil) != tt.wantErr {
t.Errorf("NewOCI() error = %v, wantErr %v", err, tt.wantErr)
return
}
moci := genArtifact(t, tt.args.ref)
got, err := s.AddOCI(ctx, moci, tt.args.ref)
if (err != nil) != tt.wantErr {
t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr)
return
}
_ = got
_, err = s.AddOCI(ctx, moci, tt.args.ref)
if err != nil {
t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr)
return
}
})
}
}
func setup(t *testing.T) func() error {
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
t.Fatal(err)
}
root = tmpdir
ctx = context.Background()
return func() error {
os.RemoveAll(tmpdir)
return nil
}
}
type mockArtifact struct {
v1.Image
}
func (m mockArtifact) MediaType() string {
mt, err := m.Image.MediaType()
if err != nil {
return ""
}
return string(mt)
}
func (m mockArtifact) RawConfig() ([]byte, error) {
return m.RawConfigFile()
}
func genArtifact(t *testing.T, ref string) artifacts.OCI {
img, err := random.Image(1024, 3)
if err != nil {
t.Fatal(err)
}
return &mockArtifact{
img,
}
}