diff --git a/Makefile b/Makefile index 398cfbf..bc504c5 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,13 @@ all: fmt vet install test build: mkdir bin;\ - $(GO_BUILD_ENV) go build -o bin ./cmd/...;\ + GOENV=GOARCH=$(uname -m) CGO_ENABLED=0 go build -o bin ./cmd/...;\ build-all: fmt vet goreleaser build --rm-dist --snapshot install: - $(GO_BUILD_ENV) go install + GOENV=GOARCH=$(uname -m) CGO_ENABLED=0 go install ./cmd/...;\ vet: go vet $(GO_FILES) diff --git a/cmd/hauler/cli/download/download.go b/cmd/hauler/cli/download/download.go index b3ec042..d63f1f6 100644 --- a/cmd/hauler/cli/download/download.go +++ b/cmd/hauler/cli/download/download.go @@ -11,7 +11,7 @@ import ( "oras.land/oras-go/pkg/content" "oras.land/oras-go/pkg/oras" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/log" diff --git a/cmd/hauler/cli/serve/registry.go b/cmd/hauler/cli/serve/registry.go index 9336314..99218bd 100644 --- a/cmd/hauler/cli/serve/registry.go +++ b/cmd/hauler/cli/serve/registry.go @@ -74,6 +74,7 @@ func (o *RegistryOpts) defaultConfig() *configuration.Configuration { cfg.HTTP.Addr = fmt.Sprintf(":%d", o.Port) cfg.HTTP.Headers = http.Header{ "X-Content-Type-Options": []string{"nosniff"}, + "Accept": []string{"application/vnd.dsse.envelope.v1+json, application/json"}, } return cfg diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 666839b..a9dceb7 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -4,17 +4,17 @@ import ( "context" "github.com/google/go-containerregistry/pkg/name" - "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" "github.com/spf13/cobra" "helm.sh/helm/v3/pkg/action" - "github.com/rancherfederal/ocil/pkg/artifacts/file" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts/file" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" "github.com/rancherfederal/hauler/pkg/content/chart" + "github.com/rancherfederal/hauler/pkg/cosign" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/reference" ) @@ -62,40 +62,48 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error { type AddImageOpts struct { *RootOpts Name string + Key string } func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() - _ = f + f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification") } func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error { + l := log.FromContext(ctx) cfg := v1alpha1.Image{ Name: reference, } + // Check if the user provided a key. + if o.Key != "" { + // verify signature using the provided key. + err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name) + if err != nil { + return err + } + l.Infof("signature verified for image [%s]", cfg.Name) + } + return storeImage(ctx, s, cfg) } func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error { l := log.FromContext(ctx) - img, err := image.NewImage(i.Name) - if err != nil { - return err - } - r, err := name.ParseReference(i.Name) if err != nil { return err } - desc, err := s.AddOCI(ctx, img, r.Name()) + err = cosign.SaveImage(ctx, s, r.Name()) + //desc, err := s.AddOCI(ctx, img, r.Name()) if err != nil { return err } - l.Infof("added 'image' to store at [%s], with digest [%s]", r.Name(), desc.Digest.String()) + l.Infof("added 'image' to store at [%s]", r.Name()) return nil } diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index ac270d0..158f433 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -5,14 +5,13 @@ import ( "fmt" "strings" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/pkg/content" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/cosign" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/log" - "github.com/rancherfederal/hauler/pkg/reference" ) type CopyOpts struct { @@ -36,7 +35,6 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) { func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error { l := log.FromContext(ctx) - var descs []ocispec.Descriptor components := strings.SplitN(targetRef, "://", 2) switch components[0] { case "dir": @@ -44,11 +42,10 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string fs := content.NewFile(components[1]) defer fs.Close() - ds, err := s.CopyAll(ctx, fs, nil) + _, err := s.CopyAll(ctx, fs, nil) if err != nil { return err } - descs = ds case "registry": l.Debugf("identified registry target reference") @@ -58,29 +55,16 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string Insecure: o.Insecure, PlainHTTP: o.PlainHTTP, } - r, err := content.NewRegistry(ropts) + + err := cosign.LoadImage(ctx, s, components[1], ropts) if err != nil { return err } - mapperFn := func(ref string) (string, error) { - r, err := reference.Relocate(ref, components[1]) - if err != nil { - return "", err - } - return r.Name(), nil - } - - ds, err := s.CopyAll(ctx, r, mapperFn) - if err != nil { - return err - } - descs = ds - default: return fmt.Errorf("detecting protocol from [%s]", targetRef) } - l.Infof("Copied [%d] artifacts to [%s]", len(descs), components[1]) + l.Infof("Copied artifacts to [%s]", components[1]) return nil } diff --git a/cmd/hauler/cli/store/extract.go b/cmd/hauler/cli/store/extract.go index de29b92..3d9630d 100644 --- a/cmd/hauler/cli/store/extract.go +++ b/cmd/hauler/cli/store/extract.go @@ -2,13 +2,14 @@ package store import ( "context" + "strings" "encoding/json" "fmt" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/log" @@ -36,7 +37,8 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string found := false if err := s.Walk(func(reference string, desc ocispec.Descriptor) error { - if reference != r.Name() { + + if !strings.Contains(reference, r.Name()) { return nil } found = true @@ -57,7 +59,7 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string return err } - pushedDesc, err := s.Copy(ctx, r.Name(), mapperStore, "") + pushedDesc, err := s.Copy(ctx, reference, mapperStore, "") if err != nil { return err } diff --git a/cmd/hauler/cli/store/flags.go b/cmd/hauler/cli/store/flags.go index dbb1b7f..3fcb27d 100644 --- a/cmd/hauler/cli/store/flags.go +++ b/cmd/hauler/cli/store/flags.go @@ -6,8 +6,8 @@ import ( "os" "path/filepath" - "github.com/rancherfederal/ocil/pkg/layer" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/layer" + "github.com/rancherfederal/hauler/pkg/store" "github.com/spf13/cobra" "github.com/rancherfederal/hauler/pkg/log" diff --git a/cmd/hauler/cli/store/info.go b/cmd/hauler/cli/store/info.go index 74cdce7..f4de62d 100644 --- a/cmd/hauler/cli/store/info.go +++ b/cmd/hauler/cli/store/info.go @@ -10,9 +10,9 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/reference" ) @@ -78,9 +78,11 @@ func buildTable(items ...item) string { fmt.Fprintf(tw, "---------\t----\t--------\t----\n") for _, i := range items { - fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", - i.Reference, i.Type, i.Layers, i.Size, - ) + if i.Type != "unknown" { + fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", + i.Reference, i.Type, i.Layers, i.Size, + ) + } } tw.Flush() return b.String() diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index a9a4774..5de6a94 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -5,8 +5,8 @@ import ( "os" "github.com/mholt/archiver/v3" - "github.com/rancherfederal/ocil/pkg/content" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/content" + "github.com/rancherfederal/hauler/pkg/store" "github.com/spf13/cobra" "github.com/rancherfederal/hauler/pkg/log" diff --git a/cmd/hauler/cli/store/serve.go b/cmd/hauler/cli/store/serve.go index 6f39441..a6ebad7 100644 --- a/cmd/hauler/cli/store/serve.go +++ b/cmd/hauler/cli/store/serve.go @@ -14,7 +14,7 @@ import ( "github.com/distribution/distribution/v3/version" "github.com/spf13/cobra" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/internal/server" ) diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 5eb4641..b8b9044 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -11,25 +11,28 @@ import ( "helm.sh/helm/v3/pkg/action" "k8s.io/apimachinery/pkg/util/yaml" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" tchart "github.com/rancherfederal/hauler/pkg/collection/chart" "github.com/rancherfederal/hauler/pkg/collection/imagetxt" "github.com/rancherfederal/hauler/pkg/collection/k3s" "github.com/rancherfederal/hauler/pkg/content" + "github.com/rancherfederal/hauler/pkg/cosign" "github.com/rancherfederal/hauler/pkg/log" ) type SyncOpts struct { *RootOpts ContentFiles []string + Key string } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files") + f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification") } func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { @@ -94,7 +97,18 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { } for _, i := range cfg.Spec.Images { - err := storeImage(ctx, s, i) + + // Check if the user provided a key. + if o.Key != "" { + // verify signature using the provided key. + err := cosign.VerifySignature(ctx, s, o.Key, i.Name) + if err != nil { + return err + } + l.Infof("signature verified for image [%s]", i.Name) + } + + err = storeImage(ctx, s, i) if err != nil { return err } diff --git a/go.mod b/go.mod index 72c4f2e..f5ac8c5 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,14 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/mholt/archiver/v3 v3.5.1 + github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc5 github.com/pkg/errors v0.9.1 - github.com/rancherfederal/ocil v0.1.9 github.com/rs/zerolog v1.31.0 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.7.0 + golang.org/x/sync v0.4.0 helm.sh/helm/v3 v3.13.0 k8s.io/apimachinery v0.28.2 k8s.io/client-go v0.28.2 @@ -110,7 +112,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nwaples/rardecode v1.1.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/prometheus/client_golang v1.16.0 // indirect @@ -123,7 +124,6 @@ require ( github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ulikunitz/xz v0.5.9 // indirect @@ -140,7 +140,6 @@ require ( golang.org/x/crypto v0.13.0 // indirect golang.org/x/net v0.13.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect diff --git a/go.sum b/go.sum index d326077..28b3d18 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rancherfederal/ocil v0.1.9 h1:pmiUQCh2HTIMDD9tDj/UqBAAxq4yloLFgd2WnrZnQgc= -github.com/rancherfederal/ocil v0.1.9/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= diff --git a/internal/mapper/mappers.go b/internal/mapper/mappers.go index 1cc60d4..7942f88 100644 --- a/internal/mapper/mappers.go +++ b/internal/mapper/mappers.go @@ -6,7 +6,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/pkg/target" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" ) type Fn func(desc ocispec.Descriptor) (string, error) @@ -39,7 +39,7 @@ func Images() map[string]Fn { return "manifest.json", nil }) - for _, l := range []string{consts.DockerManifestSchema2, consts.OCIManifestSchema1} { + for _, l := range []string{consts.DockerManifestSchema2, consts.DockerManifestListSchema2, consts.OCIManifestSchema1} { m[l] = manifestMapperFn } diff --git a/pkg/artifacts/config.go b/pkg/artifacts/config.go new file mode 100644 index 0000000..b25bb44 --- /dev/null +++ b/pkg/artifacts/config.go @@ -0,0 +1,92 @@ +package artifacts + +import ( + "bytes" + "encoding/json" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/consts" +) + +var _ partial.Describable = (*marshallableConfig)(nil) + +type Config interface { + // Raw returns the config bytes + Raw() ([]byte, error) + + Digest() (v1.Hash, error) + + MediaType() (types.MediaType, error) + + Size() (int64, error) +} + +type Marshallable interface{} + +type ConfigOption func(*marshallableConfig) + +// ToConfig takes anything that is marshallabe and converts it into a Config +func ToConfig(i Marshallable, opts ...ConfigOption) Config { + mc := &marshallableConfig{Marshallable: i} + for _, o := range opts { + o(mc) + } + return mc +} + +func WithConfigMediaType(mediaType string) ConfigOption { + return func(config *marshallableConfig) { + config.mediaType = mediaType + } +} + +// marshallableConfig implements Config using helper methods +type marshallableConfig struct { + Marshallable + + mediaType string +} + +func (c *marshallableConfig) MediaType() (types.MediaType, error) { + mt := c.mediaType + if mt == "" { + mt = consts.UnknownManifest + } + return types.MediaType(mt), nil +} + +func (c *marshallableConfig) Raw() ([]byte, error) { + return json.Marshal(c.Marshallable) +} + +func (c *marshallableConfig) Digest() (v1.Hash, error) { + return Digest(c) +} + +func (c *marshallableConfig) Size() (int64, error) { + return Size(c) +} + +type WithRawConfig interface { + Raw() ([]byte, error) +} + +func Digest(c WithRawConfig) (v1.Hash, error) { + b, err := c.Raw() + if err != nil { + return v1.Hash{}, err + } + digest, _, err := v1.SHA256(bytes.NewReader(b)) + return digest, err +} + +func Size(c WithRawConfig) (int64, error) { + b, err := c.Raw() + if err != nil { + return -1, err + } + return int64(len(b)), nil +} diff --git a/pkg/artifacts/file/file.go b/pkg/artifacts/file/file.go new file mode 100644 index 0000000..3be515e --- /dev/null +++ b/pkg/artifacts/file/file.go @@ -0,0 +1,116 @@ +package file + +import ( + "context" + + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/consts" +) + +// interface guard +var _ artifacts.OCI = (*File)(nil) + +// File implements the OCI interface for File API objects. API spec information is +// stored into the Path field. +type File struct { + Path string + + computed bool + client *getter.Client + config artifacts.Config + blob gv1.Layer + manifest *gv1.Manifest + annotations map[string]string +} + +func NewFile(path string, opts ...Option) *File { + client := getter.NewClient(getter.ClientOptions{}) + + f := &File{ + client: client, + Path: path, + } + + for _, opt := range opts { + opt(f) + } + return f +} + +// Name is the name of the file's reference +func (f *File) Name(path string) string { + return f.client.Name(path) +} + +func (f *File) MediaType() string { + return consts.OCIManifestSchema1 +} + +func (f *File) RawConfig() ([]byte, error) { + if err := f.compute(); err != nil { + return nil, err + } + return f.config.Raw() +} + +func (f *File) Layers() ([]gv1.Layer, error) { + if err := f.compute(); err != nil { + return nil, err + } + var layers []gv1.Layer + layers = append(layers, f.blob) + return layers, nil +} + +func (f *File) Manifest() (*gv1.Manifest, error) { + if err := f.compute(); err != nil { + return nil, err + } + return f.manifest, nil +} + +func (f *File) compute() error { + if f.computed { + return nil + } + + ctx := context.TODO() + blob, err := f.client.LayerFrom(ctx, f.Path) + if err != nil { + return err + } + + layer, err := partial.Descriptor(blob) + if err != nil { + return err + } + + cfg := f.client.Config(f.Path) + if cfg == nil { + cfg = f.client.Config(f.Path) + } + + cfgDesc, err := partial.Descriptor(cfg) + if err != nil { + return err + } + + m := &gv1.Manifest{ + SchemaVersion: 2, + MediaType: gtypes.MediaType(f.MediaType()), + Config: *cfgDesc, + Layers: []gv1.Descriptor{*layer}, + Annotations: f.annotations, + } + + f.manifest = m + f.config = cfg + f.blob = blob + f.computed = true + return nil +} diff --git a/pkg/artifacts/file/file_test.go b/pkg/artifacts/file/file_test.go new file mode 100644 index 0000000..729b5c4 --- /dev/null +++ b/pkg/artifacts/file/file_test.go @@ -0,0 +1,166 @@ +package file_test + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + + "github.com/rancherfederal/hauler/pkg/artifacts/file" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/consts" +) + +var ( + filename = "myfile.yaml" + data = []byte(`data`) + + ts *httptest.Server + tfs afero.Fs + mc *getter.Client +) + +func TestMain(m *testing.M) { + teardown := setup() + defer teardown() + code := m.Run() + os.Exit(code) +} + +func Test_file_Config(t *testing.T) { + tests := []struct { + name string + ref string + want string + wantErr bool + }{ + { + name: "should properly type local file", + ref: filename, + want: consts.FileLocalConfigMediaType, + wantErr: false, + }, + { + name: "should properly type remote file", + ref: ts.URL + "/" + filename, + want: consts.FileHttpConfigMediaType, + wantErr: false, + }, + // TODO: Add directory test + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := file.NewFile(tt.ref, file.WithClient(mc)) + + f.MediaType() + + m, err := f.Manifest() + if err != nil { + t.Fatal(err) + } + + got := string(m.Config.MediaType) + if got != tt.want { + t.Errorf("unxpected mediatype; got %s, want %s", got, tt.want) + } + }) + } +} + +func Test_file_Layers(t *testing.T) { + tests := []struct { + name string + ref string + want []byte + wantErr bool + }{ + { + name: "should load a local file and preserve contents", + ref: filename, + want: data, + wantErr: false, + }, + { + name: "should load a remote file and preserve contents", + ref: ts.URL + "/" + filename, + want: data, + wantErr: false, + }, + // TODO: Add directory test + } + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + f := file.NewFile(tt.ref, file.WithClient(mc)) + + layers, err := f.Layers() + if (err != nil) != tt.wantErr { + it.Fatalf("unexpected Layers() error: got %v, want %v", err, tt.wantErr) + } + + rc, err := layers[0].Compressed() + if err != nil { + it.Fatal(err) + } + + got, err := io.ReadAll(rc) + if err != nil { + it.Fatal(err) + } + + if !bytes.Equal(got, tt.want) { + it.Fatalf("unexpected Layers(): got %v, want %v", layers, tt.want) + } + }) + } +} + +func setup() func() { + tfs = afero.NewMemMapFs() + afero.WriteFile(tfs, filename, data, 0644) + + mf := &mockFile{File: getter.NewFile(), fs: tfs} + + mockHttp := getter.NewHttp() + mhttp := afero.NewHttpFs(tfs) + fileserver := http.FileServer(mhttp.Dir(".")) + http.Handle("/", fileserver) + ts = httptest.NewServer(fileserver) + + mc = &getter.Client{ + Options: getter.ClientOptions{}, + Getters: map[string]getter.Getter{ + "file": mf, + "http": mockHttp, + }, + } + + teardown := func() { + defer ts.Close() + } + + return teardown +} + +type mockFile struct { + *getter.File + fs afero.Fs +} + +func (m mockFile) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + return m.fs.Open(filepath.Join(u.Host, u.Path)) +} + +func (m mockFile) Detect(u *url.URL) bool { + fi, err := m.fs.Stat(filepath.Join(u.Host, u.Path)) + if err != nil { + return false + } + return !fi.IsDir() +} diff --git a/pkg/artifacts/file/getter/directory.go b/pkg/artifacts/file/getter/directory.go new file mode 100644 index 0000000..ab640d8 --- /dev/null +++ b/pkg/artifacts/file/getter/directory.go @@ -0,0 +1,165 @@ +package getter + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +type directory struct { + *File +} + +func NewDirectory() *directory { + return &directory{File: NewFile()} +} + +func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + tmpfile, err := os.CreateTemp("", "hauler") + if err != nil { + return nil, err + } + + digester := digest.Canonical.Digester() + zw := gzip.NewWriter(io.MultiWriter(tmpfile, digester.Hash())) + defer zw.Close() + + tarDigester := digest.Canonical.Digester() + if err := tarDir(d.path(u), d.Name(u), io.MultiWriter(zw, tarDigester.Hash()), false); err != nil { + return nil, err + } + + if err := zw.Close(); err != nil { + return nil, err + } + if err := tmpfile.Sync(); err != nil { + return nil, err + } + + fi, err := os.Open(tmpfile.Name()) + if err != nil { + return nil, err + } + + // rc := &closer{ + // t: io.TeeReader(tmpfile, fi), + // closes: []func() error{fi.Close, tmpfile.Close, zw.Close}, + // } + return fi, nil +} + +func (d directory) Detect(u *url.URL) bool { + if len(d.path(u)) == 0 { + return false + } + + fi, err := os.Stat(d.path(u)) + if err != nil { + return false + } + return fi.IsDir() +} + +func (d directory) Config(u *url.URL) artifacts.Config { + c := &directoryConfig{ + config{Reference: u.String()}, + } + return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileDirectoryConfigMediaType)) +} + +type directoryConfig struct { + config `json:",inline,omitempty"` +} + +func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error { + tw := tar.NewWriter(w) + defer tw.Close() + if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rename path + name, err := filepath.Rel(root, path) + if err != nil { + return err + } + name = filepath.Join(prefix, name) + name = filepath.ToSlash(name) + + // Generate header + var link string + mode := info.Mode() + if mode&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return err + } + } + header, err := tar.FileInfoHeader(info, link) + if err != nil { + return errors.Wrap(err, path) + } + header.Name = name + header.Uid = 0 + header.Gid = 0 + header.Uname = "" + header.Gname = "" + + if stripTimes { + header.ModTime = time.Time{} + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} + } + + // Write file + if err := tw.WriteHeader(header); err != nil { + return errors.Wrap(err, "tar") + } + if mode.IsRegular() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + if _, err := io.Copy(tw, file); err != nil { + return errors.Wrap(err, path) + } + } + + return nil + }); err != nil { + return err + } + return nil +} + +type closer struct { + t io.Reader + closes []func() error +} + +func (c *closer) Read(p []byte) (n int, err error) { + return c.t.Read(p) +} + +func (c *closer) Close() error { + var err error + for _, c := range c.closes { + lastErr := c() + if err == nil { + err = lastErr + } + } + return err +} diff --git a/pkg/artifacts/file/getter/file.go b/pkg/artifacts/file/getter/file.go new file mode 100644 index 0000000..222a80c --- /dev/null +++ b/pkg/artifacts/file/getter/file.go @@ -0,0 +1,53 @@ +package getter + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +type File struct{} + +func NewFile() *File { + return &File{} +} + +func (f File) Name(u *url.URL) string { + return filepath.Base(f.path(u)) +} + +func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + return os.Open(f.path(u)) +} + +func (f File) Detect(u *url.URL) bool { + if len(f.path(u)) == 0 { + return false + } + + fi, err := os.Stat(f.path(u)) + if err != nil { + return false + } + return !fi.IsDir() +} + +func (f File) path(u *url.URL) string { + return filepath.Join(u.Host, u.Path) +} + +func (f File) Config(u *url.URL) artifacts.Config { + c := &fileConfig{ + config{Reference: u.String()}, + } + return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileLocalConfigMediaType)) +} + +type fileConfig struct { + config `json:",inline,omitempty"` +} diff --git a/pkg/artifacts/file/getter/getter.go b/pkg/artifacts/file/getter/getter.go new file mode 100644 index 0000000..ff2e0cf --- /dev/null +++ b/pkg/artifacts/file/getter/getter.go @@ -0,0 +1,148 @@ +package getter + +import ( + "context" + "fmt" + "io" + "net/url" + + v1 "github.com/google/go-containerregistry/pkg/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "oras.land/oras-go/pkg/content" + + content2 "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/hauler/pkg/layer" +) + +type Client struct { + Getters map[string]Getter + Options ClientOptions +} + +// ClientOptions provides options for the client +type ClientOptions struct { + NameOverride string +} + +var ( + ErrGetterTypeUnknown = errors.New("no getter type found matching reference") +) + +type Getter interface { + Open(context.Context, *url.URL) (io.ReadCloser, error) + + Detect(*url.URL) bool + + Name(*url.URL) string + + Config(*url.URL) content2.Config +} + +func NewClient(opts ClientOptions) *Client { + defaults := map[string]Getter{ + "file": NewFile(), + "directory": NewDirectory(), + "http": NewHttp(), + } + + c := &Client{ + Getters: defaults, + Options: opts, + } + return c +} + +func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) { + u, err := url.Parse(source) + if err != nil { + return nil, err + } + + g, err := c.getterFrom(u) + if err != nil { + if errors.Is(err, ErrGetterTypeUnknown) { + return nil, err + } + return nil, fmt.Errorf("create getter: %w", err) + } + + opener := func() (io.ReadCloser, error) { + return g.Open(ctx, u) + } + + annotations := make(map[string]string) + annotations[ocispec.AnnotationTitle] = c.Name(source) + + switch g.(type) { + case *directory: + annotations[content.AnnotationUnpack] = "true" + } + + l, err := layer.FromOpener(opener, + layer.WithMediaType(consts.FileLayerMediaType), + layer.WithAnnotations(annotations)) + if err != nil { + return nil, err + } + return l, nil +} + +func (c *Client) ContentFrom(ctx context.Context, source string) (io.ReadCloser, error) { + u, err := url.Parse(source) + if err != nil { + return nil, fmt.Errorf("parse source %s: %w", source, err) + } + g, err := c.getterFrom(u) + if err != nil { + if errors.Is(err, ErrGetterTypeUnknown) { + return nil, err + } + return nil, fmt.Errorf("create getter: %w", err) + } + return g.Open(ctx, u) +} + +func (c *Client) getterFrom(srcUrl *url.URL) (Getter, error) { + for _, g := range c.Getters { + if g.Detect(srcUrl) { + return g, nil + } + } + return nil, errors.Wrapf(ErrGetterTypeUnknown, "source %s", srcUrl.String()) +} + +func (c *Client) Name(source string) string { + if c.Options.NameOverride != "" { + return c.Options.NameOverride + } + u, err := url.Parse(source) + if err != nil { + return source + } + for _, g := range c.Getters { + if g.Detect(u) { + return g.Name(u) + } + } + return source +} + +func (c *Client) Config(source string) content2.Config { + u, err := url.Parse(source) + if err != nil { + return nil + } + for _, g := range c.Getters { + if g.Detect(u) { + return g.Config(u) + } + } + return nil +} + +type config struct { + Reference string `json:"reference"` + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/pkg/artifacts/file/getter/getter_test.go b/pkg/artifacts/file/getter/getter_test.go new file mode 100644 index 0000000..e0da758 --- /dev/null +++ b/pkg/artifacts/file/getter/getter_test.go @@ -0,0 +1,139 @@ +package getter_test + +import ( + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" +) + +func TestClient_Detect(t *testing.T) { + teardown := setup(t) + defer teardown() + + c := getter.NewClient(getter.ClientOptions{}) + + type args struct { + source string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "should identify a file", + args: args{ + source: fileWithExt, + }, + want: "file", + }, + { + name: "should identify a directory", + args: args{ + source: rootDir, + }, + want: "directory", + }, + { + name: "should identify a http", + args: args{ + source: "http://my.cool.website", + }, + want: "http", + }, + { + name: "should identify a http", + args: args{ + source: "https://my.cool.website", + }, + want: "http", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := identify(c, tt.args.source); got != tt.want { + t.Errorf("identify() = %v, want %v", got, tt.want) + } + }) + } +} + +func identify(c *getter.Client, source string) string { + u, _ := url.Parse(source) + for t, g := range c.Getters { + if g.Detect(u) { + return t + } + } + return "" +} + +func TestClient_Name(t *testing.T) { + teardown := setup(t) + defer teardown() + + type args struct { + source string + opts getter.ClientOptions + } + tests := []struct { + name string + args args + want string + }{ + { + name: "should correctly name a file with an extension", + args: args{ + source: fileWithExt, + opts: getter.ClientOptions{}, + }, + want: "file.yaml", + }, + { + name: "should correctly name a directory", + args: args{ + source: rootDir, + opts: getter.ClientOptions{}, + }, + want: rootDir, + }, + { + name: "should correctly override a files name", + args: args{ + source: fileWithExt, + opts: getter.ClientOptions{NameOverride: "myfile"}, + }, + want: "myfile", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := getter.NewClient(tt.args.opts) + if got := c.Name(tt.args.source); got != tt.want { + t.Errorf("Name() = %v, want %v", got, tt.want) + } + }) + } +} + +var ( + rootDir = "gettertests" + fileWithExt = filepath.Join(rootDir, "file.yaml") +) + +func setup(t *testing.T) func() { + if err := os.MkdirAll(rootDir, os.ModePerm); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil { + t.Fatal(err) + } + + return func() { + os.RemoveAll(rootDir) + } +} diff --git a/pkg/artifacts/file/getter/https.go b/pkg/artifacts/file/getter/https.go new file mode 100644 index 0000000..3fb1128 --- /dev/null +++ b/pkg/artifacts/file/getter/https.go @@ -0,0 +1,67 @@ +package getter + +import ( + "context" + "io" + "mime" + "net/http" + "net/url" + "path/filepath" + "strings" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +type Http struct{} + +func NewHttp() *Http { + return &Http{} +} + +func (h Http) Name(u *url.URL) string { + resp, err := http.Head(u.String()) + if err != nil { + return "" + } + + contentType := resp.Header.Get("Content-Type") + for _, v := range strings.Split(contentType, ",") { + t, _, err := mime.ParseMediaType(v) + if err != nil { + break + } + // TODO: Identify known mimetypes for hints at a filename + _ = t + } + + // TODO: Not this + return filepath.Base(u.String()) +} + +func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + resp, err := http.Get(u.String()) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +func (h Http) Detect(u *url.URL) bool { + switch u.Scheme { + case "http", "https": + return true + } + return false +} + +func (h *Http) Config(u *url.URL) artifacts.Config { + c := &httpConfig{ + config{Reference: u.String()}, + } + return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileHttpConfigMediaType)) +} + +type httpConfig struct { + config `json:",inline,omitempty"` +} diff --git a/pkg/artifacts/file/options.go b/pkg/artifacts/file/options.go new file mode 100644 index 0000000..3efdd63 --- /dev/null +++ b/pkg/artifacts/file/options.go @@ -0,0 +1,26 @@ +package file + +import ( + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" +) + +type Option func(*File) + +func WithClient(c *getter.Client) Option { + return func(f *File) { + f.client = c + } +} + +func WithConfig(obj interface{}, mediaType string) Option { + return func(f *File) { + f.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType)) + } +} + +func WithAnnotations(m map[string]string) Option { + return func(f *File) { + f.annotations = m + } +} diff --git a/pkg/artifacts/image/image.go b/pkg/artifacts/image/image.go new file mode 100644 index 0000000..a4a0cfd --- /dev/null +++ b/pkg/artifacts/image/image.go @@ -0,0 +1,53 @@ +package image + +import ( + "github.com/google/go-containerregistry/pkg/authn" + gname "github.com/google/go-containerregistry/pkg/name" + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/rancherfederal/hauler/pkg/artifacts" +) + +var _ artifacts.OCI = (*Image)(nil) + +func (i *Image) MediaType() string { + mt, err := i.Image.MediaType() + if err != nil { + return "" + } + return string(mt) +} + +func (i *Image) RawConfig() ([]byte, error) { + return i.RawConfigFile() +} + +// Image implements the OCI interface for Image API objects. API spec information +// is stored into the Name field. +type Image struct { + Name string + gv1.Image +} + +func NewImage(name string, opts ...remote.Option) (*Image, error) { + r, err := gname.ParseReference(name) + if err != nil { + return nil, err + } + + defaultOpts := []remote.Option{ + remote.WithAuthFromKeychain(authn.DefaultKeychain), + } + opts = append(opts, defaultOpts...) + + img, err := remote.Image(r, opts...) + if err != nil { + return nil, err + } + + return &Image{ + Name: name, + Image: img, + }, nil +} diff --git a/pkg/artifacts/image/image_test.go b/pkg/artifacts/image/image_test.go new file mode 100644 index 0000000..aa66c33 --- /dev/null +++ b/pkg/artifacts/image/image_test.go @@ -0,0 +1 @@ +package image_test diff --git a/pkg/artifacts/memory/memory.go b/pkg/artifacts/memory/memory.go new file mode 100644 index 0000000..05bb752 --- /dev/null +++ b/pkg/artifacts/memory/memory.go @@ -0,0 +1,78 @@ +package memory + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +var _ artifacts.OCI = (*Memory)(nil) + +// Memory implements the OCI interface for a generic set of bytes stored in memory. +type Memory struct { + blob v1.Layer + annotations map[string]string + config artifacts.Config +} + +type defaultConfig struct { + MediaType string `json:"mediaType,omitempty"` +} + +func NewMemory(data []byte, mt string, opts ...Option) *Memory { + blob := static.NewLayer(data, types.MediaType(mt)) + + cfg := defaultConfig{MediaType: consts.MemoryConfigMediaType} + m := &Memory{ + blob: blob, + config: artifacts.ToConfig(cfg), + } + + for _, opt := range opts { + opt(m) + } + return m +} + +func (m *Memory) MediaType() string { + return consts.OCIManifestSchema1 +} + +func (m *Memory) Manifest() (*v1.Manifest, error) { + layer, err := partial.Descriptor(m.blob) + if err != nil { + return nil, err + } + + cfgDesc, err := partial.Descriptor(m.config) + if err != nil { + return nil, err + } + + manifest := &v1.Manifest{ + SchemaVersion: 2, + MediaType: types.MediaType(m.MediaType()), + Config: *cfgDesc, + Layers: []v1.Descriptor{*layer}, + Annotations: m.annotations, + } + + return manifest, nil +} + +func (m *Memory) RawConfig() ([]byte, error) { + if m.config == nil { + return []byte(`{}`), nil + } + return m.config.Raw() +} + +func (m *Memory) Layers() ([]v1.Layer, error) { + var layers []v1.Layer + layers = append(layers, m.blob) + return layers, nil +} diff --git a/pkg/artifacts/memory/memory_test.go b/pkg/artifacts/memory/memory_test.go new file mode 100644 index 0000000..529ce9d --- /dev/null +++ b/pkg/artifacts/memory/memory_test.go @@ -0,0 +1,61 @@ +package memory_test + +import ( + "math/rand" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/opencontainers/go-digest" + + "github.com/rancherfederal/hauler/pkg/artifacts/memory" +) + +func TestMemory_Layers(t *testing.T) { + tests := []struct { + name string + want *v1.Manifest + wantErr bool + }{ + { + name: "should preserve content", + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, m := setup(t) + + layers, err := m.Layers() + if err != nil { + t.Fatal(err) + } + + if len(layers) != 1 { + t.Fatalf("Expected 1 layer, got %d", len(layers)) + } + + h, err := layers[0].Digest() + if err != nil { + t.Fatal(err) + } + + d := digest.FromBytes(data) + + if d.String() != h.String() { + t.Fatalf("bytes do not match, got %s, expected %s", h.String(), d.String()) + } + }) + } +} + +func setup(t *testing.T) ([]byte, *memory.Memory) { + block := make([]byte, 2048) + _, err := rand.Read(block) + if err != nil { + t.Fatal(err) + } + + mem := memory.NewMemory(block, "random") + return block, mem +} diff --git a/pkg/artifacts/memory/options.go b/pkg/artifacts/memory/options.go new file mode 100644 index 0000000..032aa3c --- /dev/null +++ b/pkg/artifacts/memory/options.go @@ -0,0 +1,17 @@ +package memory + +import "github.com/rancherfederal/hauler/pkg/artifacts" + +type Option func(*Memory) + +func WithConfig(obj interface{}, mediaType string) Option { + return func(m *Memory) { + m.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType)) + } +} + +func WithAnnotations(annotations map[string]string) Option { + return func(m *Memory) { + m.annotations = annotations + } +} diff --git a/pkg/artifacts/ocis.go b/pkg/artifacts/ocis.go new file mode 100644 index 0000000..c1fe25b --- /dev/null +++ b/pkg/artifacts/ocis.go @@ -0,0 +1,21 @@ +package artifacts + +import "github.com/google/go-containerregistry/pkg/v1" + +// OCI is the bare minimum we need to represent an artifact in an oci layout +// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality +// This specific implementation fully encapsulates v1.Layer's within a more generic form +type OCI interface { + MediaType() string + + Manifest() (*v1.Manifest, error) + + RawConfig() ([]byte, error) + + Layers() ([]v1.Layer, error) +} + +type OCICollection interface { + // Contents returns the list of contents in the collection + Contents() (map[string]OCI, error) +} diff --git a/pkg/collection/chart/chart.go b/pkg/collection/chart/chart.go index 032ce40..05be6c3 100644 --- a/pkg/collection/chart/chart.go +++ b/pkg/collection/chart/chart.go @@ -1,8 +1,8 @@ package chart import ( - "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/image" "helm.sh/helm/v3/pkg/action" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" diff --git a/pkg/collection/imagetxt/imagetxt.go b/pkg/collection/imagetxt/imagetxt.go index 0143a7c..b9fc405 100644 --- a/pkg/collection/imagetxt/imagetxt.go +++ b/pkg/collection/imagetxt/imagetxt.go @@ -12,9 +12,9 @@ import ( "github.com/rancherfederal/hauler/pkg/log" "github.com/google/go-containerregistry/pkg/name" - artifact "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + artifact "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/artifacts/image" ) type ImageTxt struct { diff --git a/pkg/collection/imagetxt/imagetxt_test.go b/pkg/collection/imagetxt/imagetxt_test.go index 0dfa8ef..8dcee13 100644 --- a/pkg/collection/imagetxt/imagetxt_test.go +++ b/pkg/collection/imagetxt/imagetxt_test.go @@ -8,8 +8,8 @@ import ( "os" "testing" - "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/image" ) var ( diff --git a/pkg/collection/k3s/k3s.go b/pkg/collection/k3s/k3s.go index 28aae22..890db13 100644 --- a/pkg/collection/k3s/k3s.go +++ b/pkg/collection/k3s/k3s.go @@ -10,12 +10,12 @@ import ( "path" "strings" - "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/image" - "github.com/rancherfederal/ocil/pkg/artifacts/file" + "github.com/rancherfederal/hauler/pkg/artifacts/file" - "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" "github.com/rancherfederal/hauler/pkg/reference" ) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go new file mode 100644 index 0000000..0263408 --- /dev/null +++ b/pkg/consts/consts.go @@ -0,0 +1,50 @@ +package consts + +const ( + OCIManifestSchema1 = "application/vnd.oci.image.manifest.v1+json" + DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json" + DockerManifestListSchema2 = "application/vnd.docker.distribution.manifest.list.v2+json" + + DockerConfigJSON = "application/vnd.docker.container.image.v1+json" + DockerLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip" + DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" + DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar" + OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip" + + // ChartConfigMediaType is the reserved media type for the Helm chart manifest config + ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json" + + // ChartLayerMediaType is the reserved media type for Helm chart package content + ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip" + + // ProvLayerMediaType is the reserved media type for Helm chart provenance files + ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov" + + // FileLayerMediaType is the reserved media type for File content layers + FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1" + + // FileLocalConfigMediaType is the reserved media type for File config + FileLocalConfigMediaType = "application/vnd.content.hauler.file.local.config.v1+json" + FileDirectoryConfigMediaType = "application/vnd.content.hauler.file.directory.config.v1+json" + FileHttpConfigMediaType = "application/vnd.content.hauler.file.http.config.v1+json" + + // MemoryConfigMediaType + MemoryConfigMediaType = "application/vnd.content.hauler.memory.config.v1+json" + + // WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers + WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm" + + // WasmConfigMediaType is the reserved media type for WASM configs + WasmConfigMediaType = "application/vnd.wasm.config.v1+json" + + UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json" + UnknownLayer = "application/vnd.content.hauler.unknown.layer" + + OCIVendorPrefix = "vnd.oci" + DockerVendorPrefix = "vnd.docker" + HaulerVendorPrefix = "vnd.hauler" + OCIImageIndexFile = "index.json" + + KindAnnotationName = "kind" + KindAnnotation = "dev.cosignproject.cosign/image" +) diff --git a/pkg/content/chart/chart.go b/pkg/content/chart/chart.go index 329cf12..ae01d65 100644 --- a/pkg/content/chart/chart.go +++ b/pkg/content/chart/chart.go @@ -14,15 +14,15 @@ import ( "github.com/google/go-containerregistry/pkg/v1/partial" gtypes "github.com/google/go-containerregistry/pkg/v1/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/rancherfederal/ocil/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" - "github.com/rancherfederal/ocil/pkg/layer" + "github.com/rancherfederal/hauler/pkg/layer" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" ) var _ artifacts.OCI = (*Chart)(nil) @@ -137,7 +137,8 @@ func (h *Chart) RawChartData() ([]byte, error) { } // chartData loads the chart contents into memory and returns a NopCloser for the contents -// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB +// +// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB func (h *Chart) chartData() (gv1.Layer, error) { info, err := os.Stat(h.path) if err != nil { diff --git a/pkg/content/chart/chart_test.go b/pkg/content/chart/chart_test.go index 7128c8f..462f5eb 100644 --- a/pkg/content/chart/chart_test.go +++ b/pkg/content/chart/chart_test.go @@ -10,7 +10,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "helm.sh/helm/v3/pkg/action" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" "github.com/rancherfederal/hauler/pkg/content/chart" ) diff --git a/pkg/content/oci.go b/pkg/content/oci.go new file mode 100644 index 0000000..1c488dd --- /dev/null +++ b/pkg/content/oci.go @@ -0,0 +1,272 @@ +package content + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + ccontent "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/pkg/content" + "oras.land/oras-go/pkg/target" + + "github.com/rancherfederal/hauler/pkg/consts" +) + +var _ target.Target = (*OCI)(nil) + +type OCI struct { + root string + index *ocispec.Index + nameMap *sync.Map // map[string]ocispec.Descriptor +} + +func NewOCI(root string) (*OCI, error) { + o := &OCI{ + root: root, + nameMap: &sync.Map{}, + } + return o, nil +} + +// AddIndex adds a descriptor to the index and updates it +// +// The descriptor must use AnnotationRefName to identify itself +func (o *OCI) AddIndex(desc ocispec.Descriptor) error { + if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok { + return fmt.Errorf("descriptor must contain a reference from the annotation: %s", ocispec.AnnotationRefName) + } + key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName]) + o.nameMap.Store(key, desc) + return o.SaveIndex() +} + +// LoadIndex will load the index from disk +func (o *OCI) LoadIndex() error { + path := o.path(consts.OCIImageIndexFile) + idx, err := os.Open(path) + if err != nil { + if !os.IsNotExist(err) { + return err + } + o.index = &ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + } + return nil + } + defer idx.Close() + + if err := json.NewDecoder(idx).Decode(&o.index); err != nil { + return err + } + + for _, desc := range o.index.Manifests { + key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName]) + if strings.TrimSpace(key) != "--" { + o.nameMap.Store(key, desc) + } + } + return nil +} + +// SaveIndex will update the index on disk +func (o *OCI) SaveIndex() error { + var descs []ocispec.Descriptor + o.nameMap.Range(func(name, desc interface{}) bool { + n := desc.(ocispec.Descriptor).Annotations[ocispec.AnnotationRefName] + d := desc.(ocispec.Descriptor) + + if d.Annotations == nil { + d.Annotations = make(map[string]string) + } + d.Annotations[ocispec.AnnotationRefName] = n + descs = append(descs, d) + return true + }) + + // sort index to ensure that images come before any signatures and attestations. + sort.SliceStable(descs, func(i, j int) bool { + kindI := descs[i].Annotations["kind"] + kindJ := descs[j].Annotations["kind"] + + // Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top. + if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) { + return true + } else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) { + return false + } + return false // Default: maintain the order. + }) + + o.index.Manifests = descs + data, err := json.Marshal(o.index) + if err != nil { + return err + } + return os.WriteFile(o.path(consts.OCIImageIndexFile), data, 0644) +} + +// Resolve attempts to resolve the reference into a name and descriptor. +// +// The argument `ref` should be a scheme-less URI representing the remote. +// Structurally, it has a host and path. The "host" can be used to directly +// reference a specific host or be matched against a specific handler. +// +// The returned name should be used to identify the referenced entity. +// Dependending on the remote namespace, this may be immutable or mutable. +// While the name may differ from ref, it should itself be a valid ref. +// +// If the resolution fails, an error will be returned. +func (o *OCI) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) { + if err := o.LoadIndex(); err != nil { + return "", ocispec.Descriptor{}, err + } + d, ok := o.nameMap.Load(ref) + if !ok { + return "", ocispec.Descriptor{}, err + } + desc = d.(ocispec.Descriptor) + return ref, desc, nil +} + +// Fetcher returns a new fetcher for the provided reference. +// All content fetched from the returned fetcher will be +// from the namespace referred to by ref. +func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { + if err := o.LoadIndex(); err != nil { + return nil, err + } + if _, ok := o.nameMap.Load(ref); !ok { + return nil, nil + } + return o, nil +} + +func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + readerAt, err := o.blobReaderAt(desc) + if err != nil { + return nil, err + } + return readerAt, nil +} + +// Pusher returns a new pusher for the provided reference +// The returned Pusher should satisfy content.Ingester and concurrent attempts +// to push the same blob using the Ingester API should result in ErrUnavailable. +func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { + if err := o.LoadIndex(); err != nil { + return nil, err + } + + var baseRef, hash string + parts := strings.SplitN(ref, "@", 2) + baseRef = parts[0] + if len(parts) > 1 { + hash = parts[1] + } + return &ociPusher{ + oci: o, + ref: baseRef, + digest: hash, + }, nil +} + +func (o *OCI) Walk(fn func(reference string, desc ocispec.Descriptor) error) error { + if err := o.LoadIndex(); err != nil { + return err + } + + var errst []string + o.nameMap.Range(func(key, value interface{}) bool { + if err := fn(key.(string), value.(ocispec.Descriptor)); err != nil { + errst = append(errst, err.Error()) + } + return true + }) + if errst != nil { + return fmt.Errorf(strings.Join(errst, "; ")) + } + return nil +} + +func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) { + blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex()) + if err != nil { + return nil, err + } + return os.Open(blobPath) +} + +func (o *OCI) blobWriterAt(desc ocispec.Descriptor) (*os.File, error) { + blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex()) + if err != nil { + return nil, err + } + return os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644) +} + +func (o *OCI) ensureBlob(alg string, hex string) (string, error) { + dir := o.path("blobs", alg) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return "", err + } + return filepath.Join(dir, hex), nil +} + +func (o *OCI) path(elem ...string) string { + complete := []string{string(o.root)} + return filepath.Join(append(complete, elem...)...) +} + +type ociPusher struct { + oci *OCI + ref string + digest string +} + +// Push returns a content writer for the given resource identified +// by the descriptor. +func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Writer, error) { + switch d.MediaType { + case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex, consts.DockerManifestSchema2, consts.DockerManifestListSchema2: + // if the hash of the content matches that which was provided as the hash for the root, mark it + if p.digest != "" && p.digest == d.Digest.String() { + if err := p.oci.LoadIndex(); err != nil { + return nil, err + } + p.oci.nameMap.Store(p.ref, d) + if err := p.oci.SaveIndex(); err != nil { + return nil, err + } + } + } + + blobPath, err := p.oci.ensureBlob(d.Digest.Algorithm().String(), d.Digest.Hex()) + if err != nil { + return nil, err + } + + if _, err := os.Stat(blobPath); err == nil { + // file already exists, discard (but validate digest) + return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil + } + + f, err := os.Create(blobPath) + if err != nil { + return nil, err + } + + w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest)) + return w, nil +} diff --git a/pkg/cosign/cosign.go b/pkg/cosign/cosign.go new file mode 100644 index 0000000..91b06a2 --- /dev/null +++ b/pkg/cosign/cosign.go @@ -0,0 +1,286 @@ +package cosign + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "os/user" + "path/filepath" + "runtime" + "context" + "strings" + "encoding/json" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/pkg/content" + "github.com/rancherfederal/hauler/pkg/store" + "github.com/rancherfederal/hauler/pkg/log" + "github.com/rancherfederal/hauler/internal/mapper" + "github.com/rancherfederal/hauler/pkg/reference" + "github.com/rancherfederal/hauler/pkg/artifacts/file" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" +) + +// VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign. +func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string) error { + + // Ensure that the cosign binary is installed or download it if needed + cosignBinaryPath, err := ensureCosignBinary(ctx, s) + if err != nil { + return err + } + + // Command to verify the signature using Cosign. + cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, ref) + + // Run the command and capture its output. + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error verifying signature: %v, output: %s", err, output) + } + + return nil +} + +// SaveImage saves image and any signatures/attestations to the store. +func SaveImage(ctx context.Context, s *store.Layout, ref string) error { + + // Ensure that the cosign binary is installed or download it if needed + cosignBinaryPath, err := ensureCosignBinary(ctx, s) + if err != nil { + return err + } + + // Command to verify the signature using Cosign. + cmd := exec.Command(cosignBinaryPath, "save", ref, "--dir", s.Root) + + // Run the command and capture its output. + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error adding image to store: %v, output: %s", err, output) + } + + return nil +} + +// LoadImage loads store to a remote registry. +func LoadImage(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error { + + //Ensure that the cosign binary is installed or download it if needed + cosignBinaryPath, err := ensureCosignBinary(ctx, s) + if err != nil { + return err + } + + // Command to verify the signature using Cosign. + cmd := exec.Command(cosignBinaryPath, "load", "--registry", registry, "--dir", s.Root) + + // Conditionally add extra registry flags. + if ropts.Insecure { + cmd.Args = append(cmd.Args, "--allow-insecure-registry=true") + } + if ropts.PlainHTTP { + cmd.Args = append(cmd.Args, "--allow-http-registry=true") + } + + // Run the command and capture its output. + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error adding image to store: %v, output: %s", err, output) + } + + return nil +} + +// ensureCosignBinary checks if the cosign binary exists in the specified directory and installs it if not. +func ensureCosignBinary(ctx context.Context, s *store.Layout) (string, error) { + l := log.FromContext(ctx) + + // Get the current user's information + currentUser, err := user.Current() + if err != nil { + return "", fmt.Errorf("Error: %v\n", err) + } + + // Get the user's home directory + homeDir := currentUser.HomeDir + + // Construct the path to the .hauler directory + haulerDir := filepath.Join(homeDir, ".hauler") + + // Create the .hauler directory if it doesn't exist + if _, err := os.Stat(haulerDir); os.IsNotExist(err) { + // .hauler directory does not exist, create it + if err := os.MkdirAll(haulerDir, 0755); err != nil { + return "", fmt.Errorf("Error creating .hauler directory: %v\n", err) + } + l.Infof("Created .hauler directory at: %s", haulerDir) + } + + // Check if the cosign binary exists in the specified directory. + binaryPath := filepath.Join(haulerDir, "cosign") + _, err = os.Stat(binaryPath) + if err == nil { + // Cosign binary is already installed in the specified directory. + return binaryPath, nil + } + + // Cosign binary is not found. + l.Infof("Cosign binary not found. Checking to see if it exists in the store...") + + // grab binary from store if it exists, otherwise try to download it from GitHub. + // if the binary has to be downloaded, then automatically add it to the store afterwards. + err = copyCosignFromStore(ctx, s, haulerDir) + if err != nil { + l.Warnf("%s", err) + err = downloadCosign(ctx, haulerDir) + if err != nil { + return "", err + } + err = addCosignToStore(ctx, s, binaryPath) + if err != nil { + return "", err + } + } + + return binaryPath, nil +} + +// used to check if the cosign binary is in the store and if so copy it to the .hauler directory +func copyCosignFromStore(ctx context.Context, s *store.Layout, destDir string) error { + l := log.FromContext(ctx) + + ref := "hauler/cosign:latest" + r, err := reference.Parse(ref) + if err != nil { + return err + } + + found := false + if err := s.Walk(func(reference string, desc ocispec.Descriptor) error { + + if !strings.Contains(reference, r.Name()) { + return nil + } + found = true + + rc, err := s.Fetch(ctx, desc) + if err != nil { + return err + } + defer rc.Close() + + var m ocispec.Manifest + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return err + } + + mapperStore, err := mapper.FromManifest(m, destDir) + if err != nil { + return err + } + + pushedDesc, err := s.Copy(ctx, reference, mapperStore, "") + if err != nil { + return err + } + + l.Infof("extracted [%s] from store with digest [%s]", ref, pushedDesc.Digest.String()) + + return nil + }); err != nil { + return err + } + + if !found { + return fmt.Errorf("Reference [%s] not found in store. Hauler will attempt to download it from Github.", ref) + } + + return nil +} + +// adds the cosign binary to the store. +// this is to help with airgapped situations where you cannot access the internet. +func addCosignToStore(ctx context.Context, s *store.Layout, binaryPath string) error { + l := log.FromContext(ctx) + + fi := v1alpha1.File{ + Path: binaryPath, + } + + copts := getter.ClientOptions{ + NameOverride: fi.Name, + } + + f := file.NewFile(fi.Path, file.WithClient(getter.NewClient(copts))) + ref, err := reference.NewTagged(f.Name(fi.Path), reference.DefaultTag) + if err != nil { + return err + } + + desc, err := s.AddOCI(ctx, f, ref.Name()) + if err != nil { + return err + } + + l.Infof("added 'file' to store at [%s], with digest [%s]", ref.Name(), desc.Digest.String()) + return nil +} + + +// used to check if the cosign binary is in the store and if so copy it to the .hauler directory +func downloadCosign(ctx context.Context, haulerDir string) error { + l := log.FromContext(ctx) + + // Define the GitHub release URL and architecture-specific binary name. + releaseURL := "https://github.com/rancher-government-solutions/cosign/releases/latest/download" + + // Determine the architecture and add it to the binary name. + arch := runtime.GOARCH + rOS := runtime.GOOS + binaryName := "cosign" + if rOS == "windows" { + binaryName = fmt.Sprintf("cosign-%s-%s.exe", rOS, arch) + } else { + binaryName = fmt.Sprintf("cosign-%s-%s", rOS, arch) + } + + // Download the binary. + downloadURL := fmt.Sprintf("%s/%s", releaseURL, binaryName) + resp, err := http.Get(downloadURL) + if err != nil { + return fmt.Errorf("error downloading cosign binary: %v", err) + } + defer resp.Body.Close() + + // Create the cosign binary file in the specified directory. + binaryFile, err := os.Create(filepath.Join(haulerDir, binaryName)) + if err != nil { + return fmt.Errorf("error creating cosign binary: %v", err) + } + defer binaryFile.Close() + + // Copy the downloaded binary to the file. + _, err = io.Copy(binaryFile, resp.Body) + if err != nil { + return fmt.Errorf("error saving cosign binary: %v", err) + } + + // Make the binary executable. + if err := os.Chmod(binaryFile.Name(), 0755); err != nil { + return fmt.Errorf("error setting executable permission: %v", err) + } + + // Rename the binary to "cosign" + oldBinaryPath := filepath.Join(haulerDir, binaryName) + newBinaryPath := filepath.Join(haulerDir, "cosign") + if err := os.Rename(oldBinaryPath, newBinaryPath); err != nil { + return fmt.Errorf("error renaming cosign binary: %v", err) + } + + l.Infof("Cosign binary downloaded and installed to %s", haulerDir) + return nil +} \ No newline at end of file diff --git a/pkg/layer/cache.go b/pkg/layer/cache.go new file mode 100644 index 0000000..45d27f7 --- /dev/null +++ b/pkg/layer/cache.go @@ -0,0 +1,106 @@ +package layer + +import ( + "errors" + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifacts" +) + +/* +This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache +*/ + +type Cache interface { + Put(v1.Layer) (v1.Layer, error) + + Get(v1.Hash) (v1.Layer, error) +} + +var ErrLayerNotFound = errors.New("layer not found") + +type oci struct { + artifacts.OCI + + c Cache +} + +func OCICache(o artifacts.OCI, c Cache) artifacts.OCI { + return &oci{ + OCI: o, + c: c, + } +} + +func (o *oci) Layers() ([]v1.Layer, error) { + ls, err := o.OCI.Layers() + if err != nil { + return nil, err + } + + var out []v1.Layer + for _, l := range ls { + out = append(out, &lazyLayer{inner: l, c: o.c}) + } + return out, nil +} + +type lazyLayer struct { + inner v1.Layer + c Cache +} + +func (l *lazyLayer) Compressed() (io.ReadCloser, error) { + digest, err := l.inner.Digest() + if err != nil { + return nil, err + } + + layer, err := l.getOrPut(digest) + if err != nil { + return nil, err + } + + return layer.Compressed() +} + +func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) { + diffID, err := l.inner.DiffID() + if err != nil { + return nil, err + } + + layer, err := l.getOrPut(diffID) + if err != nil { + return nil, err + } + + return layer.Uncompressed() +} + +func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) { + var layer v1.Layer + if cl, err := l.c.Get(h); err == nil { + layer = cl + + } else if err == ErrLayerNotFound { + rl, err := l.c.Put(l.inner) + if err != nil { + return nil, err + } + layer = rl + + } else { + return nil, err + } + + return layer, nil +} + +func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() } +func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() } diff --git a/pkg/layer/filesystem.go b/pkg/layer/filesystem.go new file mode 100644 index 0000000..8330fdb --- /dev/null +++ b/pkg/layer/filesystem.go @@ -0,0 +1,118 @@ +package layer + +import ( + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +type fs struct { + root string +} + +func NewFilesystemCache(root string) Cache { + return &fs{root: root} +} + +func (f *fs) Put(l v1.Layer) (v1.Layer, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + diffID, err := l.DiffID() + if err != nil { + return nil, err + } + return &cachedLayer{ + Layer: l, + root: f.root, + digest: digest, + diffID: diffID, + }, nil +} + +func (f *fs) Get(h v1.Hash) (v1.Layer, error) { + opener := f.open(h) + l, err := FromOpener(opener) + if os.IsNotExist(err) { + return nil, ErrLayerNotFound + } + return l, err +} + +func (f *fs) open(h v1.Hash) Opener { + return func() (io.ReadCloser, error) { + return os.Open(layerpath(f.root, h)) + } +} + +type cachedLayer struct { + v1.Layer + + root string + digest, diffID v1.Hash +} + +func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) { + lp := layerpath(l.root, h) + if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil { + return nil, err + } + return os.Create(lp) +} + +func (l *cachedLayer) Compressed() (io.ReadCloser, error) { + f, err := l.create(l.digest) + if err != nil { + return nil, nil + } + rc, err := l.Layer.Compressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) { + f, err := l.create(l.diffID) + if err != nil { + return nil, err + } + rc, err := l.Layer.Uncompressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +func layerpath(root string, h v1.Hash) string { + return filepath.Join(root, h.Algorithm, h.Hex) +} + +type readcloser struct { + t io.Reader + closes []func() error +} + +func (rc *readcloser) Read(b []byte) (int, error) { + return rc.t.Read(b) +} + +func (rc *readcloser) Close() error { + var err error + for _, c := range rc.closes { + lastErr := c() + if err == nil { + err = lastErr + } + } + return err +} diff --git a/pkg/layer/layer.go b/pkg/layer/layer.go new file mode 100644 index 0000000..fdfbf6a --- /dev/null +++ b/pkg/layer/layer.go @@ -0,0 +1,127 @@ +package layer + +import ( + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/consts" +) + +type Opener func() (io.ReadCloser, error) + +func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) { + var err error + + layer := &layer{ + mediaType: consts.UnknownLayer, + annotations: make(map[string]string, 1), + } + + layer.uncompressedOpener = opener + layer.compressedOpener = func() (io.ReadCloser, error) { + rc, err := opener() + if err != nil { + return nil, err + } + + return rc, nil + } + + for _, opt := range opts { + opt(layer) + } + + if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil { + return nil, err + } + + if layer.diffID, _, err = compute(layer.compressedOpener); err != nil { + return nil, err + } + + return layer, nil +} + +func compute(opener Opener) (v1.Hash, int64, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, 0, err + } + defer rc.Close() + return v1.SHA256(rc) +} + +type Option func(*layer) + +func WithMediaType(mt string) Option { + return func(l *layer) { + l.mediaType = mt + } +} + +func WithAnnotations(annotations map[string]string) Option { + return func(l *layer) { + if l.annotations == nil { + l.annotations = make(map[string]string) + } + l.annotations = annotations + } +} + +type layer struct { + digest v1.Hash + diffID v1.Hash + size int64 + compressedOpener Opener + uncompressedOpener Opener + mediaType string + annotations map[string]string + urls []string +} + +func (l layer) Descriptor() (*v1.Descriptor, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + mt, err := l.MediaType() + if err != nil { + return nil, err + } + return &v1.Descriptor{ + MediaType: mt, + Size: l.size, + Digest: digest, + Annotations: l.annotations, + URLs: l.urls, + + // TODO: Allow platforms + Platform: nil, + }, nil +} + +func (l layer) Digest() (v1.Hash, error) { + return l.digest, nil +} + +func (l layer) DiffID() (v1.Hash, error) { + return l.diffID, nil +} + +func (l layer) Compressed() (io.ReadCloser, error) { + return l.compressedOpener() +} + +func (l layer) Uncompressed() (io.ReadCloser, error) { + return l.uncompressedOpener() +} + +func (l layer) Size() (int64, error) { + return l.size, nil +} + +func (l layer) MediaType() (gtypes.MediaType, error) { + return gtypes.MediaType(l.mediaType), nil +} diff --git a/pkg/store/store.go b/pkg/store/store.go new file mode 100644 index 0000000..ef4b2d5 --- /dev/null +++ b/pkg/store/store.go @@ -0,0 +1,262 @@ +package store + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" + "oras.land/oras-go/pkg/oras" + "oras.land/oras-go/pkg/target" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/hauler/pkg/content" + "github.com/rancherfederal/hauler/pkg/layer" +) + +type Layout struct { + *content.OCI + Root string + cache layer.Cache +} + +type Options func(*Layout) + +func WithCache(c layer.Cache) Options { + return func(l *Layout) { + l.cache = c + } +} + +func NewLayout(rootdir string, opts ...Options) (*Layout, error) { + ociStore, err := content.NewOCI(rootdir) + if err != nil { + return nil, err + } + + if err := ociStore.LoadIndex(); err != nil { + return nil, err + } + + l := &Layout{ + Root: rootdir, + OCI: ociStore, + } + + for _, opt := range opts { + opt(l) + } + + return l, nil +} + +// AddOCI adds an artifacts.OCI to the store +// +// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once +// saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use +// strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the +// future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout. +func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (ocispec.Descriptor, error) { + if l.cache != nil { + cached := layer.OCICache(oci, l.cache) + oci = cached + } + + // Write manifest blob + m, err := oci.Manifest() + if err != nil { + return ocispec.Descriptor{}, err + } + + mdata, err := json.Marshal(m) + if err != nil { + return ocispec.Descriptor{}, err + } + if err := l.writeBlobData(mdata); err != nil { + return ocispec.Descriptor{}, err + } + + // Write config blob + cdata, err := oci.RawConfig() + if err != nil { + return ocispec.Descriptor{}, err + } + + static.NewLayer(cdata, "") + + if err := l.writeBlobData(cdata); err != nil { + return ocispec.Descriptor{}, err + } + + // write blob layers concurrently + layers, err := oci.Layers() + if err != nil { + return ocispec.Descriptor{}, err + } + + var g errgroup.Group + for _, lyr := range layers { + lyr := lyr + g.Go(func() error { + return l.writeLayer(lyr) + }) + } + if err := g.Wait(); err != nil { + return ocispec.Descriptor{}, err + } + + // Build index + idx := ocispec.Descriptor{ + MediaType: string(m.MediaType), + Digest: digest.FromBytes(mdata), + Size: int64(len(mdata)), + Annotations: map[string]string{ + consts.KindAnnotationName: consts.KindAnnotation, + ocispec.AnnotationRefName: ref, + }, + URLs: nil, + Platform: nil, + } + + return idx, l.OCI.AddIndex(idx) +} + +// AddOCICollection . +func (l *Layout) AddOCICollection(ctx context.Context, collection artifacts.OCICollection) ([]ocispec.Descriptor, error) { + cnts, err := collection.Contents() + if err != nil { + return nil, err + } + + var descs []ocispec.Descriptor + for ref, oci := range cnts { + desc, err := l.AddOCI(ctx, oci, ref) + if err != nil { + return nil, err + } + descs = append(descs, desc) + } + return descs, nil +} + +// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting oci-layout content +// +// This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents +// To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly deletes oci-layout content only +func (l *Layout) Flush(ctx context.Context) error { + blobs := filepath.Join(l.Root, "blobs") + if err := os.RemoveAll(blobs); err != nil { + return err + } + + index := filepath.Join(l.Root, "index.json") + if err := os.RemoveAll(index); err != nil { + return err + } + + layout := filepath.Join(l.Root, "oci-layout") + if err := os.RemoveAll(layout); err != nil { + return err + } + + return nil +} + +// Copy will copy a given reference to a given target.Target +// +// This is essentially a wrapper around oras.Copy, but locked to this content store +func (l *Layout) Copy(ctx context.Context, ref string, to target.Target, toRef string) (ocispec.Descriptor, error) { + return oras.Copy(ctx, l.OCI, ref, to, toRef, + oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2, consts.DockerManifestListSchema2)) +} + +// CopyAll performs bulk copy operations on the stores oci layout to a provided target.Target +func (l *Layout) CopyAll(ctx context.Context, to target.Target, toMapper func(string) (string, error)) ([]ocispec.Descriptor, error) { + var descs []ocispec.Descriptor + err := l.OCI.Walk(func(reference string, desc ocispec.Descriptor) error { + toRef := "" + if toMapper != nil { + tr, err := toMapper(reference) + if err != nil { + return err + } + toRef = tr + } + + desc, err := l.Copy(ctx, reference, to, toRef) + if err != nil { + return err + } + + descs = append(descs, desc) + return nil + }) + if err != nil { + return nil, err + } + return descs, nil +} + +// Identify is a helper function that will identify a human-readable content type given a descriptor +func (l *Layout) Identify(ctx context.Context, desc ocispec.Descriptor) string { + rc, err := l.OCI.Fetch(ctx, desc) + if err != nil { + return "" + } + defer rc.Close() + + m := struct { + Config struct { + MediaType string `json:"mediaType"` + } `json:"config"` + }{} + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return "" + } + + return m.Config.MediaType +} + +func (l *Layout) writeBlobData(data []byte) error { + blob := static.NewLayer(data, "") // NOTE: MediaType isn't actually used in the writing + return l.writeLayer(blob) +} + +func (l *Layout) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + dir := filepath.Join(l.Root, "blobs", d.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + blobPath := filepath.Join(dir, d.Hex) + // Skip entirely if something exists, assume layer is present already + if _, err := os.Stat(blobPath); err == nil { + return nil + } + + w, err := os.Create(blobPath) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, r) + return err +} diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go new file mode 100644 index 0000000..54b4319 --- /dev/null +++ b/pkg/store/store_test.go @@ -0,0 +1,105 @@ +package store_test + +import ( + "context" + "os" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/store" +) + +var ( + ctx context.Context + root string +) + +func TestLayout_AddOCI(t *testing.T) { + teardown := setup(t) + defer teardown() + + type args struct { + ref string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "", + args: args{ + ref: "hello/world:v1", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, err := store.NewLayout(root) + if (err != nil) != tt.wantErr { + t.Errorf("NewOCI() error = %v, wantErr %v", err, tt.wantErr) + return + } + moci := genArtifact(t, tt.args.ref) + + got, err := s.AddOCI(ctx, moci, tt.args.ref) + if (err != nil) != tt.wantErr { + t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr) + return + } + _ = got + + _, err = s.AddOCI(ctx, moci, tt.args.ref) + if err != nil { + t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func setup(t *testing.T) func() error { + tmpdir, err := os.MkdirTemp("", "hauler") + if err != nil { + t.Fatal(err) + } + root = tmpdir + + ctx = context.Background() + + return func() error { + os.RemoveAll(tmpdir) + return nil + } +} + +type mockArtifact struct { + v1.Image +} + +func (m mockArtifact) MediaType() string { + mt, err := m.Image.MediaType() + if err != nil { + return "" + } + return string(mt) +} + +func (m mockArtifact) RawConfig() ([]byte, error) { + return m.RawConfigFile() +} + +func genArtifact(t *testing.T, ref string) artifacts.OCI { + img, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + + return &mockArtifact{ + img, + } +}