github/hauler
1
0
Fork 0
mirror of https://github.com/hauler-dev/hauler.git synced 2026-08-18 20:07:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

factor out core oci logic into independent library (rancherfederal/ocil)

Browse Source
This commit is contained in:
Josh Wolf
2022-01-12 09:47:09 -07:00
parent 96d231efdf
commit 8b372d8a20
39 changed files with 112 additions and 2320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/hauler/cli/cli.go
+7 -6
View File
@@ -6,11 +6,12 @@ import (
"os"
"path/filepath"
"github.com/rancherfederal/ocil/pkg/layer"
"github.com/spf13/cobra"
cache2 "github.com/rancherfederal/hauler/internal/cache"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
)
type rootOpts struct {
@@ -55,7 +56,7 @@ func New() *cobra.Command {
return cmd
}
func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
func (o *rootOpts) getStore(ctx context.Context) (*store.Layout, error) {
l := log.FromContext(ctx)
dir := o.storeDir
@@ -80,14 +81,14 @@ func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
return nil, err
}
s, err := store.NewStore(abs, store.WithCache(c))
s, err := store.NewLayout(abs, store.WithCache(c))
if err != nil {
return nil, err
}
return s, nil
}
func (o *rootOpts) getCache(ctx context.Context) (cache2.Cache, error) {
func (o *rootOpts) getCache(ctx context.Context) (layer.Cache, error) {
dir := o.cacheDir
if dir == "" {
@@ -105,6 +106,6 @@ func (o *rootOpts) getCache(ctx context.Context) (cache2.Cache, error) {
dir = abs
}
c := cache2.NewFilesystem(dir)
c := layer.NewFilesystemCache(dir)
return c, nil
}
cmd/hauler/cli/download/download.go
+2 -1
View File
@@ -11,8 +11,9 @@ import (
"oras.land/oras-go/pkg/content"
"oras.land/oras-go/pkg/oras"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
)
cmd/hauler/cli/store/add.go
+14 -12
View File
@@ -7,13 +7,15 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/content/chart"
"github.com/rancherfederal/hauler/pkg/content/file"
"github.com/rancherfederal/hauler/pkg/content/image"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
"github.com/rancherfederal/hauler/pkg/store"
)
type AddFileOpts struct {
@@ -25,7 +27,7 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) {
f.StringVarP(&o.Name, "name", "n", "", "(Optional) Name to assign to file in store")
}
func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference string) error {
func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Layout, reference string) error {
cfg := v1alpha1.File{
Path: reference,
}
@@ -33,7 +35,7 @@ func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference s
return storeFile(ctx, s, cfg)
}
func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error {
func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
l := log.FromContext(ctx)
f := file.NewFile(fi.Path)
@@ -42,7 +44,7 @@ func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error {
return err
}
desc, err := s.AddArtifact(ctx, f, ref.Name())
desc, err := s.AddOCI(ctx, f, ref.Name())
if err != nil {
return err
}
@@ -60,7 +62,7 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
_ = f
}
func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference string) error {
func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error {
cfg := v1alpha1.Image{
Name: reference,
}
@@ -68,7 +70,7 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference
return storeImage(ctx, s, cfg)
}
func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error {
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error {
l := log.FromContext(ctx)
oci, err := image.NewImage(i.Name)
@@ -81,7 +83,7 @@ func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error {
return err
}
desc, err := s.AddArtifact(ctx, oci, r.Name())
desc, err := s.AddOCI(ctx, oci, r.Name())
if err != nil {
return err
}
@@ -104,7 +106,7 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
f.StringVar(&o.Version, "version", "", "(Optional) Version of the chart to download, defaults to latest if not specified")
}
func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName string) error {
func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Layout, chartName string) error {
path := ""
if _, err := os.Stat(chartName); err == nil {
path = chartName
@@ -119,7 +121,7 @@ func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName
return storeChart(ctx, s, cfg)
}
func storeChart(ctx context.Context, s *store.Store, cfg v1alpha1.Chart) error {
func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart) error {
l := log.FromContext(ctx)
oci, err := chart.NewChart(cfg)
@@ -131,7 +133,7 @@ func storeChart(ctx context.Context, s *store.Store, cfg v1alpha1.Chart) error {
if err != nil {
return err
}
desc, err := s.AddArtifact(ctx, oci, ref.Name())
desc, err := s.AddOCI(ctx, oci, ref.Name())
if err != nil {
return err
}
cmd/hauler/cli/store/copy.go
+7 -5
View File
@@ -9,8 +9,10 @@ import (
"github.com/spf13/cobra"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/reference"
)
type CopyOpts struct {
@@ -31,7 +33,7 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) {
f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry")
}
func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, targetRef string) error {
func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error {
l := log.FromContext(ctx)
var descs []ocispec.Descriptor
@@ -60,12 +62,12 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, targetRef string)
return err
}
mapperFn := func(reference string) (string, error) {
ref, err := store.RelocateReference(reference, components[1])
mapperFn := func(ref string) (string, error) {
r, err := reference.Relocate(ref, components[1])
if err != nil {
return "", err
}
return ref.Name(), nil
return r.Name(), nil
}
ds, err := s.CopyAll(ctx, r, mapperFn)
cmd/hauler/cli/store/extract.go
+5 -4
View File
@@ -8,10 +8,11 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
"github.com/rancherfederal/hauler/pkg/store"
)
type ExtractOpts struct {
@@ -24,7 +25,7 @@ func (o *ExtractOpts) AddArgs(cmd *cobra.Command) {
f.StringVarP(&o.DestinationDir, "output", "o", "", "Directory to save contents to (defaults to current directory)")
}
func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, ref string) error {
func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string) error {
l := log.FromContext(ctx)
r, err := reference.Parse(ref)
@@ -33,13 +34,13 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, ref string)
}
found := false
if err := s.Content.Walk(func(reference string, desc ocispec.Descriptor) error {
if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
if reference != r.Name() {
return nil
}
found = true
rc, err := s.Content.Fetch(ctx, desc)
rc, err := s.Fetch(ctx, desc)
if err != nil {
return err
}
cmd/hauler/cli/store/info.go
+8 -6
View File
@@ -10,9 +10,11 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/reference"
"github.com/rancherfederal/hauler/pkg/store"
)
type InfoOpts struct {
@@ -28,14 +30,14 @@ func (o *InfoOpts) AddFlags(cmd *cobra.Command) {
// TODO: Regex/globbing
}
func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Store) error {
func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
var items []item
if err := s.Content.Walk(func(ref string, desc ocispec.Descriptor) error {
if err := s.Walk(func(ref string, desc ocispec.Descriptor) error {
if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
return nil
}
rc, err := s.Content.Fetch(ctx, desc)
rc, err := s.Fetch(ctx, desc)
if err != nil {
return err
}
@@ -97,7 +99,7 @@ type item struct {
Size string
}
func newItem(s *store.Store, desc ocispec.Descriptor, m ocispec.Manifest) item {
func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item {
var size int64 = 0
for _, l := range m.Layers {
size = +l.Size
cmd/hauler/cli/store/serve.go
+3 -2
View File
@@ -14,8 +14,9 @@ import (
"github.com/distribution/distribution/v3/version"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/internal/server"
"github.com/rancherfederal/hauler/pkg/store"
)
type ServeOpts struct {
@@ -37,7 +38,7 @@ func (o *ServeOpts) AddFlags(cmd *cobra.Command) {
}
// ServeCmd serves the embedded registry almost identically to how distribution/v3 does it
func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error {
func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Layout) error {
ctx = dcontext.WithVersion(ctx, version.Version)
tr := server.NewTempRegistry(ctx, o.RootDir)
cmd/hauler/cli/store/sync.go
+5 -4
View File
@@ -10,12 +10,13 @@ import (
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/util/yaml"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
"github.com/rancherfederal/hauler/pkg/collection/k3s"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
)
type SyncOpts struct {
@@ -28,7 +29,7 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
}
func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
l := log.FromContext(ctx)
// Start from an empty store (contents are cached elsewhere)
@@ -120,7 +121,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
return err
}
if _, err := s.AddCollection(ctx, k); err != nil {
if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}
@@ -136,7 +137,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
return err
}
if _, err := s.AddCollection(ctx, tc); err != nil {
if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
}
go.mod
+6 -8
View File
@@ -10,14 +10,12 @@ require (
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/mholt/archiver/v3 v3.5.1
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2
github.com/pkg/errors v0.9.1
github.com/rancherfederal/ocil v0.1.6
github.com/rs/zerolog v1.26.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.6.0
github.com/spf13/cobra v1.2.1
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
helm.sh/helm/v3 v3.6.1-0.20211203140009-0e6e936ab84e
k8s.io/apimachinery v0.22.4
k8s.io/client-go v0.22.4
@@ -48,7 +46,6 @@ require (
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.10.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.11+incompatible // indirect
@@ -112,13 +109,14 @@ require (
github.com/nwaples/rardecode v1.1.0 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.15.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/prometheus/common v0.30.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
@@ -126,7 +124,6 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/ulikunitz/xz v0.5.9 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -139,9 +136,10 @@ require (
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20211111162719-482062a4217b // indirect
go.sum
+9 -3
View File
@@ -852,8 +852,9 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug=
github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -864,9 +865,12 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rancherfederal/ocil v0.1.6 h1:PncaQJkny10WHr3bYOuM6uXVUUdMki/xooNF0XiMGvo=
github.com/rancherfederal/ocil v0.1.6/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -1153,6 +1157,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
@@ -1296,8 +1301,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
internal/cache/cache.go
Vendored
-102
View File
@@ -1,102 +0,0 @@
package cache
import (
"errors"
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifact"
)
type Cache interface {
Put(v1.Layer) (v1.Layer, error)
Get(v1.Hash) (v1.Layer, error)
}
var ErrLayerNotFound = errors.New("layer not found")
type oci struct {
artifact.OCI
c Cache
}
func Oci(o artifact.OCI, c Cache) artifact.OCI {
return &oci{
OCI: o,
c: c,
}
}
func (o *oci) Layers() ([]v1.Layer, error) {
ls, err := o.OCI.Layers()
if err != nil {
return nil, err
}
var out []v1.Layer
for _, l := range ls {
out = append(out, &lazyLayer{inner: l, c: o.c})
}
return out, nil
}
type lazyLayer struct {
inner v1.Layer
c Cache
}
func (l *lazyLayer) Compressed() (io.ReadCloser, error) {
digest, err := l.inner.Digest()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(digest)
if err != nil {
return nil, err
}
return layer.Compressed()
}
func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) {
diffID, err := l.inner.DiffID()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(diffID)
if err != nil {
return nil, err
}
return layer.Uncompressed()
}
func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) {
var layer v1.Layer
if cl, err := l.c.Get(h); err == nil {
layer = cl
} else if err == ErrLayerNotFound {
rl, err := l.c.Put(l.inner)
if err != nil {
return nil, err
}
layer = rl
} else {
return nil, err
}
return layer, nil
}
func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() }
func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() }
internal/cache/doc.go
Vendored
-5
View File
@@ -1,5 +0,0 @@
package cache
/*
This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache
*/
internal/cache/filesystem.go
Vendored
-120
View File
@@ -1,120 +0,0 @@
package cache
import (
"io"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/rancherfederal/hauler/internal/layer"
)
type fs struct {
root string
}
func NewFilesystem(root string) Cache {
return &fs{root: root}
}
func (f *fs) Put(l v1.Layer) (v1.Layer, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
diffID, err := l.DiffID()
if err != nil {
return nil, err
}
return &cachedLayer{
Layer: l,
root: f.root,
digest: digest,
diffID: diffID,
}, nil
}
func (f *fs) Get(h v1.Hash) (v1.Layer, error) {
opener := f.open(h)
l, err := layer.FromOpener(opener)
if os.IsNotExist(err) {
return nil, ErrLayerNotFound
}
return l, err
}
func (f *fs) open(h v1.Hash) layer.Opener {
return func() (io.ReadCloser, error) {
return os.Open(layerpath(f.root, h))
}
}
type cachedLayer struct {
v1.Layer
root string
digest, diffID v1.Hash
}
func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) {
lp := layerpath(l.root, h)
if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil {
return nil, err
}
return os.Create(lp)
}
func (l *cachedLayer) Compressed() (io.ReadCloser, error) {
f, err := l.create(l.digest)
if err != nil {
return nil, nil
}
rc, err := l.Layer.Compressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) {
f, err := l.create(l.diffID)
if err != nil {
return nil, err
}
rc, err := l.Layer.Uncompressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func layerpath(root string, h v1.Hash) string {
return filepath.Join(root, h.Algorithm, h.Hex)
}
type readcloser struct {
t io.Reader
closes []func() error
}
func (rc *readcloser) Read(b []byte) (int, error) {
return rc.t.Read(b)
}
func (rc *readcloser) Close() error {
var err error
for _, c := range rc.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}
internal/getter/directory.go
-165
View File
@@ -1,165 +0,0 @@
package getter
import (
"archive/tar"
"compress/gzip"
"context"
"io"
"net/url"
"os"
"path/filepath"
"time"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type directory struct {
*File
}
func NewDirectory() *directory {
return &directory{File: NewFile()}
}
func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
tmpfile, err := os.CreateTemp("", "hauler")
if err != nil {
return nil, err
}
digester := digest.Canonical.Digester()
zw := gzip.NewWriter(io.MultiWriter(tmpfile, digester.Hash()))
defer zw.Close()
tarDigester := digest.Canonical.Digester()
if err := tarDir(d.path(u), d.Name(u), io.MultiWriter(zw, tarDigester.Hash()), false); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
if err := tmpfile.Sync(); err != nil {
return nil, err
}
fi, err := os.Open(tmpfile.Name())
if err != nil {
return nil, err
}
// rc := &closer{
// t: io.TeeReader(tmpfile, fi),
// closes: []func() error{fi.Close, tmpfile.Close, zw.Close},
// }
return fi, nil
}
func (d directory) Detect(u *url.URL) bool {
if len(d.path(u)) == 0 {
return false
}
fi, err := os.Stat(d.path(u))
if err != nil {
return false
}
return fi.IsDir()
}
func (d directory) Config(u *url.URL) artifact.Config {
c := &directoryConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileDirectoryConfigMediaType))
}
type directoryConfig struct {
config `json:",inline,omitempty"`
}
func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error {
tw := tar.NewWriter(w)
defer tw.Close()
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Rename path
name, err := filepath.Rel(root, path)
if err != nil {
return err
}
name = filepath.Join(prefix, name)
name = filepath.ToSlash(name)
// Generate header
var link string
mode := info.Mode()
if mode&os.ModeSymlink != 0 {
if link, err = os.Readlink(path); err != nil {
return err
}
}
header, err := tar.FileInfoHeader(info, link)
if err != nil {
return errors.Wrap(err, path)
}
header.Name = name
header.Uid = 0
header.Gid = 0
header.Uname = ""
header.Gname = ""
if stripTimes {
header.ModTime = time.Time{}
header.AccessTime = time.Time{}
header.ChangeTime = time.Time{}
}
// Write file
if err := tw.WriteHeader(header); err != nil {
return errors.Wrap(err, "tar")
}
if mode.IsRegular() {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err := io.Copy(tw, file); err != nil {
return errors.Wrap(err, path)
}
}
return nil
}); err != nil {
return err
}
return nil
}
type closer struct {
t io.Reader
closes []func() error
}
func (c *closer) Read(p []byte) (n int, err error) {
return c.t.Read(p)
}
func (c *closer) Close() error {
var err error
for _, c := range c.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}
internal/getter/file.go
-53
View File
@@ -1,53 +0,0 @@
package getter
import (
"context"
"io"
"net/url"
"os"
"path/filepath"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type File struct{}
func NewFile() *File {
return &File{}
}
func (f File) Name(u *url.URL) string {
return filepath.Base(f.path(u))
}
func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return os.Open(f.path(u))
}
func (f File) Detect(u *url.URL) bool {
if len(f.path(u)) == 0 {
return false
}
fi, err := os.Stat(f.path(u))
if err != nil {
return false
}
return !fi.IsDir()
}
func (f File) path(u *url.URL) string {
return filepath.Join(u.Host, u.Path)
}
func (f File) Config(u *url.URL) artifact.Config {
c := &fileConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileLocalConfigMediaType))
}
type fileConfig struct {
config `json:",inline,omitempty"`
}
internal/getter/getter.go
-114
View File
@@ -1,114 +0,0 @@
package getter
import (
"context"
"io"
"net/url"
v1 "github.com/google/go-containerregistry/pkg/v1"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/internal/layer"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Client struct {
Getters map[string]Getter
Options ClientOptions
}
// TODO: Make some valid ClientOptions
type ClientOptions struct{}
var (
ErrGetterTypeUnknown = errors.New("no getter type found matching reference")
)
type Getter interface {
Open(context.Context, *url.URL) (io.ReadCloser, error)
Detect(*url.URL) bool
Name(*url.URL) string
Config(*url.URL) artifact.Config
}
func NewClient(opts ClientOptions) *Client {
defaults := map[string]Getter{
"file": NewFile(),
"directory": NewDirectory(),
"http": NewHttp(),
}
c := &Client{
Getters: defaults,
Options: opts,
}
return c
}
func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) {
u, err := url.Parse(source)
if err != nil {
return nil, err
}
for _, g := range c.Getters {
if g.Detect(u) {
opener := func() (io.ReadCloser, error) {
return g.Open(ctx, u)
}
annotations := make(map[string]string)
annotations[ocispec.AnnotationTitle] = g.Name(u)
switch g.(type) {
case *directory:
annotations[content.AnnotationUnpack] = "true"
}
l, err := layer.FromOpener(opener,
layer.WithMediaType(consts.FileLayerMediaType),
layer.WithAnnotations(annotations))
if err != nil {
return nil, err
}
return l, nil
}
}
return nil, errors.Wrapf(ErrGetterTypeUnknown, "%s", source)
}
func (c *Client) Name(source string) string {
u, err := url.Parse(source)
if err != nil {
return source
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Name(u)
}
}
return source
}
func (c *Client) Config(source string) artifact.Config {
u, err := url.Parse(source)
if err != nil {
return nil
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Config(u)
}
}
return nil
}
type config struct {
Reference string `json:"reference"`
Annotations map[string]string `json:"annotations,omitempty"`
}
internal/getter/getter_test.go
-129
View File
@@ -1,129 +0,0 @@
package getter_test
import (
"net/url"
"os"
"path/filepath"
"testing"
"github.com/rancherfederal/hauler/internal/getter"
)
func TestClient_Detect(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should identify a file",
args: args{
source: fileWithExt,
},
want: "file",
},
{
name: "should identify a directory",
args: args{
source: rootDir,
},
want: "directory",
},
{
name: "should identify a http",
args: args{
source: "http://my.cool.website",
},
want: "http",
},
{
name: "should identify a http",
args: args{
source: "https://my.cool.website",
},
want: "http",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := identify(c, tt.args.source); got != tt.want {
t.Errorf("identify() = %v, want %v", got, tt.want)
}
})
}
}
func identify(c *getter.Client, source string) string {
u, _ := url.Parse(source)
for t, g := range c.Getters {
if g.Detect(u) {
return t
}
}
return ""
}
func TestClient_Name(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should correctly name a file with an extension",
args: args{
source: fileWithExt,
},
want: "file.yaml",
},
{
name: "should correctly name a directory",
args: args{
source: rootDir,
},
want: rootDir,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := c.Name(tt.args.source); got != tt.want {
t.Errorf("Name() = %v, want %v", got, tt.want)
}
})
}
}
var (
rootDir = "gettertests"
fileWithExt = filepath.Join(rootDir, "file.yaml")
)
func setup(t *testing.T) func() {
if err := os.MkdirAll(rootDir, os.ModePerm); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil {
t.Fatal(err)
}
return func() {
os.RemoveAll(rootDir)
}
}
internal/getter/https.go
-67
View File
@@ -1,67 +0,0 @@
package getter
import (
"context"
"io"
"mime"
"net/http"
"net/url"
"path/filepath"
"strings"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Http struct{}
func NewHttp() *Http {
return &Http{}
}
func (h Http) Name(u *url.URL) string {
resp, err := http.Head(u.String())
if err != nil {
return ""
}
contentType := resp.Header.Get("Content-Type")
for _, v := range strings.Split(contentType, ",") {
t, _, err := mime.ParseMediaType(v)
if err != nil {
break
}
// TODO: Identify known mimetypes for hints at a filename
_ = t
}
// TODO: Not this
return filepath.Base(u.String())
}
func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
resp, err := http.Get(u.String())
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (h Http) Detect(u *url.URL) bool {
switch u.Scheme {
case "http", "https":
return true
}
return false
}
func (h *Http) Config(u *url.URL) artifact.Config {
c := &httpConfig{
config{Reference: u.String()},
}
return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileHttpConfigMediaType))
}
type httpConfig struct {
config `json:",inline,omitempty"`
}
internal/layer/layer.go
-127
View File
@@ -1,127 +0,0 @@
package layer
import (
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Opener func() (io.ReadCloser, error)
func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) {
var err error
layer := &layer{
mediaType: consts.UnknownLayer,
annotations: make(map[string]string, 1),
}
layer.uncompressedOpener = opener
layer.compressedOpener = func() (io.ReadCloser, error) {
rc, err := opener()
if err != nil {
return nil, err
}
return rc, nil
}
for _, opt := range opts {
opt(layer)
}
if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil {
return nil, err
}
if layer.diffID, _, err = compute(layer.compressedOpener); err != nil {
return nil, err
}
return layer, nil
}
func compute(opener Opener) (v1.Hash, int64, error) {
rc, err := opener()
if err != nil {
return v1.Hash{}, 0, err
}
defer rc.Close()
return v1.SHA256(rc)
}
type Option func(*layer)
func WithMediaType(mt string) Option {
return func(l *layer) {
l.mediaType = mt
}
}
func WithAnnotations(annotations map[string]string) Option {
return func(l *layer) {
if l.annotations == nil {
l.annotations = make(map[string]string)
}
l.annotations = annotations
}
}
type layer struct {
digest v1.Hash
diffID v1.Hash
size int64
compressedOpener Opener
uncompressedOpener Opener
mediaType string
annotations map[string]string
urls []string
}
func (l layer) Descriptor() (*v1.Descriptor, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
mt, err := l.MediaType()
if err != nil {
return nil, err
}
return &v1.Descriptor{
MediaType: mt,
Size: l.size,
Digest: digest,
Annotations: l.annotations,
URLs: l.urls,
// TODO: Allow platforms
Platform: nil,
}, nil
}
func (l layer) Digest() (v1.Hash, error) {
return l.digest, nil
}
func (l layer) DiffID() (v1.Hash, error) {
return l.diffID, nil
}
func (l layer) Compressed() (io.ReadCloser, error) {
return l.compressedOpener()
}
func (l layer) Uncompressed() (io.ReadCloser, error) {
return l.uncompressedOpener()
}
func (l layer) Size() (int64, error) {
return l.size, nil
}
func (l layer) MediaType() (gtypes.MediaType, error) {
return gtypes.MediaType(l.mediaType), nil
}
internal/mapper/mappers.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/ocil/pkg/consts"
)
type Fn func(desc ocispec.Descriptor) (string, error)
pkg/artifact/config.go
-92
View File
@@ -1,92 +0,0 @@
package artifact
import (
"bytes"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
"k8s.io/apimachinery/pkg/util/json"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ partial.Describable = (*marshallableConfig)(nil)
type Config interface {
// Raw returns the config bytes
Raw() ([]byte, error)
Digest() (v1.Hash, error)
MediaType() (types.MediaType, error)
Size() (int64, error)
}
type Marshallable interface{}
type ConfigOption func(*marshallableConfig)
// ToConfig takes anything that is marshallabe and converts it into a Config
func ToConfig(i Marshallable, opts ...ConfigOption) Config {
mc := &marshallableConfig{Marshallable: i}
for _, o := range opts {
o(mc)
}
return mc
}
func WithConfigMediaType(mediaType string) ConfigOption {
return func(config *marshallableConfig) {
config.mediaType = mediaType
}
}
// marshallableConfig implements Config using helper methods
type marshallableConfig struct {
Marshallable
mediaType string
}
func (c *marshallableConfig) MediaType() (types.MediaType, error) {
mt := c.mediaType
if mt == "" {
mt = consts.UnknownManifest
}
return types.MediaType(mt), nil
}
func (c *marshallableConfig) Raw() ([]byte, error) {
return json.Marshal(c.Marshallable)
}
func (c *marshallableConfig) Digest() (v1.Hash, error) {
return Digest(c)
}
func (c *marshallableConfig) Size() (int64, error) {
return Size(c)
}
type WithRawConfig interface {
Raw() ([]byte, error)
}
func Digest(c WithRawConfig) (v1.Hash, error) {
b, err := c.Raw()
if err != nil {
return v1.Hash{}, err
}
digest, _, err := v1.SHA256(bytes.NewReader(b))
return digest, err
}
func Size(c WithRawConfig) (int64, error) {
b, err := c.Raw()
if err != nil {
return -1, err
}
return int64(len(b)), nil
}
pkg/artifact/oci.go
-23
View File
@@ -1,23 +0,0 @@
package artifact
import (
"github.com/google/go-containerregistry/pkg/v1"
)
// OCI is the bare minimum we need to represent an artifact in an oci layout
// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality
// This specific implementation fully encapsulates v1.Layer's within a more generic form
type OCI interface {
MediaType() string
Manifest() (*v1.Manifest, error)
RawConfig() ([]byte, error)
Layers() ([]v1.Layer, error)
}
type Collection interface {
// Contents returns the list of contents in the collection
Contents() (map[string]OCI, error)
}
pkg/collection/chart/chart.go
+8 -7
View File
@@ -1,13 +1,14 @@
package chart
import (
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/content/chart"
"github.com/rancherfederal/hauler/pkg/content/image"
)
var _ artifact.Collection = (*tchart)(nil)
var _ artifacts.OCICollection = (*tchart)(nil)
// tchart is a thick chart that includes all the dependent images as well as the chart itself
type tchart struct {
@@ -15,10 +16,10 @@ type tchart struct {
config v1alpha1.ThickChart
computed bool
contents map[string]artifact.OCI
contents map[string]artifacts.OCI
}
func NewThickChart(cfg v1alpha1.ThickChart) (artifact.Collection, error) {
func NewThickChart(cfg v1alpha1.ThickChart) (artifacts.OCICollection, error) {
o, err := chart.NewChart(cfg.Chart)
if err != nil {
return nil, err
@@ -27,11 +28,11 @@ func NewThickChart(cfg v1alpha1.ThickChart) (artifact.Collection, error) {
return &tchart{
chart: o,
config: cfg,
contents: make(map[string]artifact.OCI),
contents: make(map[string]artifacts.OCI),
}, nil
}
func (c *tchart) Contents() (map[string]artifact.OCI, error) {
func (c *tchart) Contents() (map[string]artifacts.OCI, error) {
if err := c.compute(); err != nil {
return nil, err
}
pkg/collection/k3s/k3s.go
+11 -9
View File
@@ -10,13 +10,15 @@ import (
"path"
"strings"
"github.com/rancherfederal/hauler/internal/getter"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/content/file"
"github.com/rancherfederal/hauler/pkg/content/image"
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/ocil/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
)
var _ artifact.Collection = (*k3s)(nil)
var _ artifacts.OCICollection = (*k3s)(nil)
const (
releaseUrl = "https://github.com/k3s-io/k3s/releases/download"
@@ -36,19 +38,19 @@ type k3s struct {
arch string
computed bool
contents map[string]artifact.OCI
contents map[string]artifacts.OCI
channels map[string]string
client *getter.Client
}
func NewK3s(version string) (artifact.Collection, error) {
func NewK3s(version string) (artifacts.OCICollection, error) {
return &k3s{
version: version,
contents: make(map[string]artifact.OCI),
contents: make(map[string]artifacts.OCI),
}, nil
}
func (k *k3s) Contents() (map[string]artifact.OCI, error) {
func (k *k3s) Contents() (map[string]artifacts.OCI, error) {
if err := k.compute(); err != nil {
return nil, err
}
pkg/consts/consts.go
-43
View File
@@ -1,43 +0,0 @@
package consts
const (
OCIManifestSchema1 = "application/vnd.oci.image.manifest.v1+json"
DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json"
DockerConfigJSON = "application/vnd.docker.container.image.v1+json"
DockerLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip"
DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar"
OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip"
// ChartConfigMediaType is the reserved media type for the Helm chart manifest config
ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
// ChartLayerMediaType is the reserved media type for Helm chart package content
ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"
// ProvLayerMediaType is the reserved media type for Helm chart provenance files
ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov"
// FileLayerMediaType is the reserved media type for File content layers
FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1"
// FileLocalConfigMediaType is the reserved media type for File config
FileLocalConfigMediaType = "application/vnd.content.hauler.file.local.config.v1+json"
FileDirectoryConfigMediaType = "application/vnd.content.hauler.file.directory.config.v1+json"
FileHttpConfigMediaType = "application/vnd.content.hauler.file.http.config.v1+json"
// WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers
WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm"
// WasmConfigMediaType is the reserved media type for WASM configs
WasmConfigMediaType = "application/vnd.wasm.config.v1+json"
UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json"
UnknownLayer = "application/vnd.content.hauler.unknown.layer"
OCIVendorPrefix = "vnd.oci"
DockerVendorPrefix = "vnd.docker"
HaulerVendorPrefix = "vnd.hauler"
OCIImageIndexFile = "index.json"
)
pkg/content/chart/chart.go
+6 -4
View File
@@ -14,18 +14,20 @@ import (
"github.com/google/go-containerregistry/pkg/v1/partial"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rancherfederal/ocil/pkg/artifacts"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"github.com/rancherfederal/hauler/internal/layer"
"github.com/rancherfederal/ocil/pkg/layer"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ artifact.OCI = (*Chart)(nil)
var _ artifacts.OCI = (*Chart)(nil)
// Chart implements the OCI interface for Chart API objects. API spec values are
// stored into the Repo, Name, and Version fields.
pkg/content/chart/chart_test.go
+2 -1
View File
@@ -10,7 +10,8 @@ import (
"github.com/mholt/archiver/v3"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content/chart"
)
pkg/content/file/file.go
-116
View File
@@ -1,116 +0,0 @@
package file
import (
"context"
gv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/internal/getter"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
// interface guard
var _ artifact.OCI = (*File)(nil)
// File implements the OCI interface for File API objects. API spec information is
// stored into the Path field.
type File struct {
Path string
client *getter.Client
computed bool
config artifact.Config
blob gv1.Layer
manifest *gv1.Manifest
annotations map[string]string
}
func NewFile(path string, opts ...Option) *File {
client := getter.NewClient(getter.ClientOptions{})
f := &File{
client: client,
Path: path,
}
for _, opt := range opts {
opt(f)
}
return f
}
func (f *File) Name(path string) string {
return f.client.Name(path)
}
func (f *File) MediaType() string {
return consts.OCIManifestSchema1
}
func (f *File) RawConfig() ([]byte, error) {
if err := f.compute(); err != nil {
return nil, err
}
return f.config.Raw()
}
func (f *File) Layers() ([]gv1.Layer, error) {
if err := f.compute(); err != nil {
return nil, err
}
var layers []gv1.Layer
layers = append(layers, f.blob)
return layers, nil
}
func (f *File) Manifest() (*gv1.Manifest, error) {
if err := f.compute(); err != nil {
return nil, err
}
return f.manifest, nil
}
func (f *File) compute() error {
if f.computed {
return nil
}
ctx := context.Background()
blob, err := f.client.LayerFrom(ctx, f.Path)
if err != nil {
return err
}
layer, err := partial.Descriptor(blob)
if err != nil {
return err
}
cfg := f.client.Config(f.Path)
if cfg == nil {
cfg = f.client.Config(f.Path)
}
cfgDesc, err := partial.Descriptor(cfg)
if err != nil {
return err
}
m := &gv1.Manifest{
SchemaVersion: 2,
MediaType: gtypes.MediaType(f.MediaType()),
Config: *cfgDesc,
Layers: []gv1.Descriptor{*layer},
Annotations: f.annotations,
}
f.manifest = m
f.config = cfg
f.blob = blob
f.computed = true
return nil
}
pkg/content/file/file_test.go
-166
View File
@@ -1,166 +0,0 @@
package file_test
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/rancherfederal/hauler/internal/getter"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content/file"
)
var (
filename = "myfile.yaml"
data = []byte(`data`)
ts *httptest.Server
tfs afero.Fs
mc *getter.Client
)
func TestMain(m *testing.M) {
teardown := setup()
defer teardown()
code := m.Run()
os.Exit(code)
}
func Test_file_Config(t *testing.T) {
tests := []struct {
name string
ref string
want string
wantErr bool
}{
{
name: "should properly type local file",
ref: filename,
want: consts.FileLocalConfigMediaType,
wantErr: false,
},
{
name: "should properly type remote file",
ref: ts.URL + "/" + filename,
want: consts.FileHttpConfigMediaType,
wantErr: false,
},
// TODO: Add directory test
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := file.NewFile(tt.ref, file.WithClient(mc))
f.MediaType()
m, err := f.Manifest()
if err != nil {
t.Fatal(err)
}
got := string(m.Config.MediaType)
if got != tt.want {
t.Errorf("unxpected mediatype; got %s, want %s", got, tt.want)
}
})
}
}
func Test_file_Layers(t *testing.T) {
tests := []struct {
name string
ref string
want []byte
wantErr bool
}{
{
name: "should load a local file and preserve contents",
ref: filename,
want: data,
wantErr: false,
},
{
name: "should load a remote file and preserve contents",
ref: ts.URL + "/" + filename,
want: data,
wantErr: false,
},
// TODO: Add directory test
}
for _, tt := range tests {
t.Run(tt.name, func(it *testing.T) {
f := file.NewFile(tt.ref, file.WithClient(mc))
layers, err := f.Layers()
if (err != nil) != tt.wantErr {
it.Fatalf("unexpected Layers() error: got %v, want %v", err, tt.wantErr)
}
rc, err := layers[0].Compressed()
if err != nil {
it.Fatal(err)
}
got, err := io.ReadAll(rc)
if err != nil {
it.Fatal(err)
}
if !bytes.Equal(got, tt.want) {
it.Fatalf("unexpected Layers(): got %v, want %v", layers, tt.want)
}
})
}
}
func setup() func() {
tfs = afero.NewMemMapFs()
afero.WriteFile(tfs, filename, data, 0644)
mf := &mockFile{File: getter.NewFile(), fs: tfs}
mockHttp := getter.NewHttp()
mhttp := afero.NewHttpFs(tfs)
fileserver := http.FileServer(mhttp.Dir("."))
http.Handle("/", fileserver)
ts = httptest.NewServer(fileserver)
mc = &getter.Client{
Options: getter.ClientOptions{},
Getters: map[string]getter.Getter{
"file": mf,
"http": mockHttp,
},
}
teardown := func() {
defer ts.Close()
}
return teardown
}
type mockFile struct {
*getter.File
fs afero.Fs
}
func (m mockFile) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return m.fs.Open(filepath.Join(u.Host, u.Path))
}
func (m mockFile) Detect(u *url.URL) bool {
fi, err := m.fs.Stat(filepath.Join(u.Host, u.Path))
if err != nil {
return false
}
return !fi.IsDir()
}
pkg/content/file/options.go
-26
View File
@@ -1,26 +0,0 @@
package file
import (
"github.com/rancherfederal/hauler/internal/getter"
"github.com/rancherfederal/hauler/pkg/artifact"
)
type Option func(*File)
func WithClient(c *getter.Client) Option {
return func(f *File) {
f.client = c
}
}
func WithConfig(obj interface{}, mediaType string) Option {
return func(f *File) {
f.config = artifact.ToConfig(obj, artifact.WithConfigMediaType(mediaType))
}
}
func WithAnnotations(m map[string]string) Option {
return func(f *File) {
f.annotations = m
}
}
pkg/content/image/image.go
-53
View File
@@ -1,53 +0,0 @@
package image
import (
"github.com/google/go-containerregistry/pkg/authn"
gname "github.com/google/go-containerregistry/pkg/name"
gv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/rancherfederal/hauler/pkg/artifact"
)
var _ artifact.OCI = (*Image)(nil)
func (i *Image) MediaType() string {
mt, err := i.Image.MediaType()
if err != nil {
return ""
}
return string(mt)
}
func (i *Image) RawConfig() ([]byte, error) {
return i.RawConfigFile()
}
// Image implements the OCI interface for Image API objects. API spec information
// is stored into the Name field.
type Image struct {
Name string
gv1.Image
}
func NewImage(name string, opts ...remote.Option) (*Image, error) {
r, err := gname.ParseReference(name)
if err != nil {
return nil, err
}
defaultOpts := []remote.Option{
remote.WithAuthFromKeychain(authn.DefaultKeychain),
}
opts = append(opts, defaultOpts...)
img, err := remote.Image(r, opts...)
if err != nil {
return nil, err
}
return &Image{
Name: name,
Image: img,
}, nil
}
pkg/content/image/image_test.go
-1
View File
@@ -1 +0,0 @@
package image_test
pkg/reference/reference.go
+18
View File
@@ -47,3 +47,21 @@ func Parse(ref string) (gname.Reference, error) {
return r, nil
}
// Relocate returns a name.Reference given a reference and registry
func Relocate(reference string, registry string) (gname.Reference, error) {
ref, err := gname.ParseReference(reference)
if err != nil {
return nil, err
}
relocated, err := gname.ParseReference(ref.Context().RepositoryStr(), gname.WithDefaultRegistry(registry))
if err != nil {
return nil, err
}
if _, err := gname.NewDigest(ref.Name()); err == nil {
return relocated.Context().Digest(ref.Identifier()), nil
}
return relocated.Context().Tag(ref.Identifier()), nil
}
pkg/store/errors.go
-7
View File
@@ -1,7 +0,0 @@
package store
import "github.com/pkg/errors"
var (
ErrInvalidReference = errors.New("invalid reference")
)
pkg/store/oci.go
-236
View File
@@ -1,236 +0,0 @@
package store
import (
"context"
"encoding/json"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
ccontent "github.com/containerd/containerd/content"
"github.com/containerd/containerd/remotes"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/content"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ target.Target = (*OCI)(nil)
type OCI struct {
root string
index *ocispec.Index
nameMap *sync.Map // map[string]ocispec.Descriptor
}
func NewOCI(root string) (*OCI, error) {
o := &OCI{
root: root,
nameMap: &sync.Map{},
}
return o, nil
}
func (o *OCI) LoadIndex() error {
path := o.path(consts.OCIImageIndexFile)
idx, err := os.Open(path)
if err != nil {
if !os.IsNotExist(err) {
return err
}
o.index = &ocispec.Index{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
}
return nil
}
defer idx.Close()
if err := json.NewDecoder(idx).Decode(&o.index); err != nil {
return err
}
for _, desc := range o.index.Manifests {
if name := desc.Annotations[ocispec.AnnotationRefName]; name != "" {
o.nameMap.Store(name, desc)
}
}
return nil
}
func (o *OCI) SaveIndex() error {
var descs []ocispec.Descriptor
o.nameMap.Range(func(name, desc interface{}) bool {
n := name.(string)
d := desc.(ocispec.Descriptor)
if d.Annotations == nil {
d.Annotations = make(map[string]string)
}
d.Annotations[ocispec.AnnotationRefName] = n
descs = append(descs, d)
return true
})
o.index.Manifests = descs
data, err := json.Marshal(o.index)
if err != nil {
return err
}
return os.WriteFile(o.path(consts.OCIImageIndexFile), data, 0644)
}
// Resolve attempts to resolve the reference into a name and descriptor.
//
// The argument `ref` should be a scheme-less URI representing the remote.
// Structurally, it has a host and path. The "host" can be used to directly
// reference a specific host or be matched against a specific handler.
//
// The returned name should be used to identify the referenced entity.
// Dependending on the remote namespace, this may be immutable or mutable.
// While the name may differ from ref, it should itself be a valid ref.
//
// If the resolution fails, an error will be returned.
func (o *OCI) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
if err := o.LoadIndex(); err != nil {
return "", ocispec.Descriptor{}, err
}
d, ok := o.nameMap.Load(ref)
if !ok {
return "", ocispec.Descriptor{}, err
}
desc = d.(ocispec.Descriptor)
return ref, desc, nil
}
// Fetcher returns a new fetcher for the provided reference.
// All content fetched from the returned fetcher will be
// from the namespace referred to by ref.
func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) {
if err := o.LoadIndex(); err != nil {
return nil, err
}
if _, ok := o.nameMap.Load(ref); !ok {
return nil, nil
}
return o, nil
}
func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
readerAt, err := o.blobReaderAt(desc)
if err != nil {
return nil, err
}
return readerAt, nil
}
// Pusher returns a new pusher for the provided reference
// The returned Pusher should satisfy content.Ingester and concurrent attempts
// to push the same blob using the Ingester API should result in ErrUnavailable.
func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) {
if err := o.LoadIndex(); err != nil {
return nil, err
}
var baseRef, hash string
parts := strings.SplitN(ref, "@", 2)
baseRef = parts[0]
if len(parts) > 1 {
hash = parts[1]
}
return &ociPusher{
oci: o,
ref: baseRef,
digest: hash,
}, nil
}
func (o *OCI) Walk(fn func(reference string, desc ocispec.Descriptor) error) error {
if err := o.LoadIndex(); err != nil {
return err
}
o.nameMap.Range(func(key, value interface{}) bool {
if err := fn(key.(string), value.(ocispec.Descriptor)); err != nil {
return false
}
return true
})
return nil
}
func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) {
blobPath, err := o.ensureBlob(desc)
if err != nil {
return nil, err
}
return os.Open(blobPath)
}
func (o *OCI) blobWriterAt(desc ocispec.Descriptor) (*os.File, error) {
blobPath, err := o.ensureBlob(desc)
if err != nil {
return nil, err
}
return os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
}
func (o *OCI) ensureBlob(desc ocispec.Descriptor) (string, error) {
dir := o.path("blobs", desc.Digest.Algorithm().String())
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return "", err
}
return filepath.Join(dir, desc.Digest.Hex()), nil
}
func (o *OCI) path(elem ...string) string {
complete := []string{string(o.root)}
return filepath.Join(append(complete, elem...)...)
}
type ociPusher struct {
oci *OCI
ref string
digest string
}
// Push returns a content writer for the given resource identified
// by the descriptor.
func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Writer, error) {
switch d.MediaType {
case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex, consts.DockerManifestSchema2:
// if the hash of the content matches that which was provided as the hash for the root, mark it
if p.digest != "" && p.digest == d.Digest.String() {
if err := p.oci.LoadIndex(); err != nil {
return nil, err
}
p.oci.nameMap.Store(p.ref, d)
if err := p.oci.SaveIndex(); err != nil {
return nil, err
}
}
}
blobPath, err := p.oci.ensureBlob(d)
if err != nil {
return nil, err
}
if _, err := os.Stat(blobPath); err == nil {
// file already exists, discard (but validate digest)
return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil
}
f, err := os.Create(blobPath)
if err != nil {
return nil, err
}
w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest))
return w, nil
}
pkg/store/options.go
-13
View File
@@ -1,13 +0,0 @@
package store
import (
"github.com/rancherfederal/hauler/internal/cache"
)
type Options func(*Store)
func WithCache(c cache.Cache) Options {
return func(b *Store) {
b.cache = c
}
}
pkg/store/stager.go
-192
View File
@@ -1,192 +0,0 @@
package store
import (
"context"
"encoding/json"
"io"
"os"
"path/filepath"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/pkg/oras"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
)
type stager interface {
add(ctx context.Context, oci artifact.OCI, ref name.Reference) error
commit(ctx context.Context, b *Store) (ocispec.Descriptor, error)
flush(ctx context.Context) error
}
var _ stager = (*layout)(nil)
func newLayout() (*layout, error) {
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
return nil, err
}
return &layout{
root: tmpdir,
blobs: make(map[digest.Digest]v1.Layer),
}, nil
}
type layout struct {
root string
descs []ocispec.Descriptor
blobs map[digest.Digest]v1.Layer
}
func (l *layout) commit(ctx context.Context, b *Store) (ocispec.Descriptor, error) {
defer l.flush(ctx)
var g errgroup.Group
for d, blob := range l.blobs {
blob := blob
d := d
g.Go(func() error {
rc, err := blob.Compressed()
if err != nil {
return err
}
return l.writeBlob(d, rc)
})
}
if err := g.Wait(); err != nil {
return ocispec.Descriptor{}, err
}
idx := ocispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
Manifests: l.descs,
Annotations: nil,
}
data, err := json.Marshal(idx)
if err != nil {
return ocispec.Descriptor{}, err
}
if err := os.WriteFile(l.path("index.json"), data, 0666); err != nil {
return ocispec.Descriptor{}, err
}
src, err := NewOCI(l.path(""))
if err != nil {
return ocispec.Descriptor{}, err
}
dst, err := NewOCI(b.Root)
if err != nil {
return ocispec.Descriptor{}, err
}
if err := src.LoadIndex(); err != nil {
return ocispec.Descriptor{}, err
}
m := src.index.Manifests[0]
ref := m.Annotations[ocispec.AnnotationRefName]
desc, err := oras.Copy(ctx, src, ref, dst, "",
oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2))
if err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "comitting staging layout")
}
if err := dst.SaveIndex(); err != nil {
return ocispec.Descriptor{}, err
}
return desc, err
}
func (l *layout) flush(ctx context.Context) error {
return os.RemoveAll(l.path(""))
}
func (l *layout) add(ctx context.Context, oci artifact.OCI, ref name.Reference) error {
m, err := oci.Manifest()
if err != nil {
return err
}
mdata, err := json.Marshal(m)
if err != nil {
return err
}
mdigest := digest.FromBytes(mdata)
l.blobs[mdigest] = static.NewLayer(mdata, m.MediaType)
cdata, err := oci.RawConfig()
if err != nil {
return err
}
l.blobs[digest.FromBytes(cdata)] = static.NewLayer(cdata, "")
layers, err := oci.Layers()
if err != nil {
return err
}
for _, layer := range layers {
h, err := layer.Digest()
if err != nil {
return err
}
d := digest.NewDigestFromHex(h.Algorithm, h.Hex)
l.blobs[d] = layer
}
mdesc := ocispec.Descriptor{
MediaType: oci.MediaType(),
Digest: mdigest,
Size: int64(len(mdata)),
Annotations: map[string]string{
ocispec.AnnotationRefName: ref.Name(),
},
URLs: nil,
Platform: nil,
}
l.descs = append(l.descs, mdesc)
return nil
}
func (l *layout) writeBlob(d digest.Digest, rc io.ReadCloser) error {
dir := l.path("blobs", d.Algorithm().String())
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return err
}
file := filepath.Join(dir, d.Hex())
if _, err := os.Stat(file); err == nil {
return err
}
w, err := os.Create(file)
if err != nil {
return err
}
defer w.Close()
_, err = io.Copy(w, rc)
return err
}
func (l *layout) path(elem ...string) string {
complete := []string{string(l.root)}
return filepath.Join(append(complete, elem...)...)
}
pkg/store/store.go
-184
View File
@@ -1,184 +0,0 @@
package store
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"github.com/google/go-containerregistry/pkg/name"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/oras"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/internal/cache"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/reference"
)
type Store struct {
Root string
Content *OCI
cache cache.Cache
}
func NewStore(rootdir string, opts ...Options) (*Store, error) {
ociStore, err := NewOCI(rootdir)
if err != nil {
return nil, err
}
b := &Store{
Root: rootdir,
Content: ociStore,
}
for _, opt := range opts {
opt(b)
}
return b, nil
}
// AddArtifact will add an artifact.OCI to the store
// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once
// saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use
// strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the
// future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout.
func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, r string) (ocispec.Descriptor, error) {
stage, err := newLayout()
if err != nil {
return ocispec.Descriptor{}, err
}
if s.cache != nil {
cached := cache.Oci(oci, s.cache)
oci = cached
}
// Validate we have a locatable reference
ref, err := reference.Parse(r)
if err != nil {
return ocispec.Descriptor{}, err
}
if err := stage.add(ctx, oci, ref); err != nil {
return ocispec.Descriptor{}, fmt.Errorf("adding artifact: %w", err)
}
return stage.commit(ctx, s)
}
// AddCollection .
func (s *Store) AddCollection(ctx context.Context, coll artifact.Collection) ([]ocispec.Descriptor, error) {
cnts, err := coll.Contents()
if err != nil {
return nil, err
}
var descs []ocispec.Descriptor
for ref, oci := range cnts {
ds, err := s.AddArtifact(ctx, oci, ref)
if err != nil {
return nil, err
}
descs = append(descs, ds)
}
return descs, nil
}
// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting oci-layout content
// This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents
// To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly deletes oci-layout content only
func (s *Store) Flush(ctx context.Context) error {
blobs := filepath.Join(s.Root, "blobs")
if err := os.RemoveAll(blobs); err != nil {
return err
}
index := filepath.Join(s.Root, "index.json")
if err := os.RemoveAll(index); err != nil {
return err
}
layout := filepath.Join(s.Root, "oci-layout")
if err := os.RemoveAll(layout); err != nil {
return err
}
return nil
}
// Copy will copy a given reference to a given target.Target
// This is essentially a wrapper around oras.Copy, but locked to this content store
func (s *Store) Copy(ctx context.Context, ref string, to target.Target, toRef string) (ocispec.Descriptor, error) {
return oras.Copy(ctx, s.Content, ref, to, toRef,
oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2))
}
// CopyAll performs bulk copy operations on the stores oci layout to a provided target.Target
func (s *Store) CopyAll(ctx context.Context, to target.Target, toMapper func(string) (string, error)) ([]ocispec.Descriptor, error) {
var descs []ocispec.Descriptor
err := s.Content.Walk(func(reference string, desc ocispec.Descriptor) error {
toRef := ""
if toMapper != nil {
tr, err := toMapper(reference)
if err != nil {
return err
}
toRef = tr
}
desc, err := s.Copy(ctx, reference, to, toRef)
if err != nil {
return err
}
descs = append(descs, desc)
return nil
})
if err != nil {
return nil, err
}
return descs, nil
}
// Identify is a helper function that will identify a human-readable content type given a descriptor
func (s *Store) Identify(ctx context.Context, desc ocispec.Descriptor) string {
rc, err := s.Content.Fetch(ctx, desc)
if err != nil {
return ""
}
defer rc.Close()
m := struct {
Config struct {
MediaType string `json:"mediaType"`
} `json:"config"`
}{}
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return ""
}
return m.Config.MediaType
}
// RelocateReference returns a name.Reference given a reference and registry
func RelocateReference(reference string, registry string) (name.Reference, error) {
ref, err := name.ParseReference(reference)
if err != nil {
return nil, err
}
relocated, err := name.ParseReference(ref.Context().RepositoryStr(), name.WithDefaultRegistry(registry))
if err != nil {
return nil, err
}
if _, err := name.NewDigest(ref.Name()); err == nil {
return relocated.Context().Digest(ref.Identifier()), nil
}
return relocated.Context().Tag(ref.Identifier()), nil
}
pkg/store/store_test.go
-213
View File
@@ -1,213 +0,0 @@
package store_test
import (
"bytes"
"context"
"encoding/json"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/google/go-containerregistry/pkg/name"
gv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/random"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/content"
"oras.land/oras-go/pkg/target"
_ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem"
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
"github.com/rancherfederal/hauler/internal/server"
"github.com/rancherfederal/hauler/pkg/artifact"
"github.com/rancherfederal/hauler/pkg/store"
)
func TestStore_AddArtifact(t *testing.T) {
ctx := context.Background()
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
t.Fatal(err)
}
s, err := store.NewStore(tmpdir)
if err != nil {
t.Fatal(err)
}
type args struct {
ctx context.Context
reference string
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "should add artifact with a valid tagged reference",
args: args{
ctx: ctx,
reference: "hauler/random:v1",
},
wantErr: false,
},
// {
// name: "should fail when an invalid reference is provided",
// args: args{
// ctx: ctx,
// reference: "n0tV@l!d:v1",
// },
// wantErr: true,
// },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
oci, want := genArtifact(t, tt.args.reference)
got, err := s.AddArtifact(tt.args.ctx, oci, tt.args.reference)
if (err != nil) != tt.wantErr {
t.Errorf("AddArtifact() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, want) {
t.Errorf("AddArtifact() got = %v, want %v", got, want)
}
})
}
}
type testImage struct {
gv1.Image
}
func (i *testImage) MediaType() string {
mt, err := i.Image.MediaType()
if err != nil {
return ""
}
return string(mt)
}
func (i *testImage) RawConfig() ([]byte, error) {
return i.RawConfigFile()
}
func genArtifact(t *testing.T, ref string) (artifact.OCI, ocispec.Descriptor) {
img, err := random.Image(1024, 3)
if err != nil {
t.Fatal(err)
}
desc, err := partial.Descriptor(img)
if err != nil {
t.Fatal(err)
}
desc.Annotations = make(map[string]string)
desc.Annotations[ocispec.AnnotationRefName] = ref
data, err := json.Marshal(desc)
if err != nil {
t.Fatal(err)
}
var m ocispec.Descriptor
if err := json.NewDecoder(bytes.NewBuffer(data)).Decode(&m); err != nil {
t.Fatal(err)
}
return &testImage{Image: img}, m
}
func TestStore_CopyAll(t *testing.T) {
ctx := context.Background()
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
tmpdirRegistry := filepath.Join(tmpdir, "registry")
if err := os.Mkdir(tmpdirRegistry, os.ModePerm); err != nil {
t.Fatal()
}
tmpdirStore := filepath.Join(tmpdir, "store")
if err := os.Mkdir(tmpdirStore, os.ModePerm); err != nil {
t.Fatal()
}
r := server.NewTempRegistry(ctx, tmpdirRegistry)
if err := r.Start(); err != nil {
t.Error(err)
}
defer r.Stop()
rc, err := content.NewRegistry(content.RegistryOptions{Insecure: true})
if err != nil {
t.Fatal(err)
}
type args struct {
ctx context.Context
to target.Target
toMapper func(string) (string, error)
refs []string
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "should copy an artifact to a registry",
args: args{
ctx: ctx,
to: rc,
toMapper: nil,
refs: []string{"tester:tester"},
},
wantErr: false,
},
{
name: "should copy a lot of artifacts to a registry (test concurrency)",
args: args{
ctx: ctx,
to: rc,
toMapper: nil,
refs: []string{"a/b:c", "a/c:b", "b/c:d", "z/y:w", "u/q:a", "y/y:y", "z/z:z", "b/b:b", "c/c:c"},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, err := store.NewStore(tmpdirStore)
if err != nil {
t.Fatal(err)
}
for _, ref := range tt.args.refs {
locRef, _ := name.ParseReference(ref, name.WithDefaultRegistry(r.Registry()))
a, _ := genArtifact(t, locRef.Name())
if _, err := s.AddArtifact(ctx, a, locRef.Name()); err != nil {
t.Errorf("Failed to generate store contents for CopyAll(): %v", err)
}
}
if descs, err := s.CopyAll(tt.args.ctx, tt.args.to, tt.args.toMapper); (err != nil) != tt.wantErr {
t.Errorf("CopyAll() error = %v, wantErr %v", err, tt.wantErr)
} else if len(descs) != len(tt.args.refs) {
t.Errorf("CopyAll() expected to push %d descriptors, but only pushed %d", len(descs), len(tt.args.refs))
}
if err := s.Flush(ctx); err != nil {
t.Fatal(err)
}
})
}
}