From d55e7572e66cf7db6965c9f76cf609db1706283f Mon Sep 17 00:00:00 2001 From: Josh Wolf Date: Fri, 3 Dec 2021 14:01:06 -0700 Subject: [PATCH] remove custom file store in favor of less hacky IoContentWriter extended on top of existing file store --- .goreleaser.yaml | 2 +- cmd/hauler/cli/download/download.go | 54 ++-------- cmd/hauler/cli/serve/files.go | 1 - cmd/hauler/cli/serve/registry.go | 3 - cmd/hauler/cli/store/copy.go | 3 - cmd/hauler/cli/store/info.go | 6 +- cmd/hauler/cli/version.go | 1 - go.mod | 8 +- go.sum | 18 ++-- internal/getter/getter.go | 8 +- internal/mapper/filestore.go | 151 +++++----------------------- internal/mapper/mappers.go | 40 ++------ pkg/content/chart/chart.go | 18 ---- pkg/content/file/file.go | 18 ---- pkg/content/image/image.go | 37 ------- pkg/store/oci.go | 6 +- pkg/store/store.go | 2 - pkg/store/store_test.go | 5 - 18 files changed, 64 insertions(+), 317 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7e6460e..e4def44 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -22,7 +22,7 @@ builds: - CGO_ENABLED=0 universal_binaries: - - replace: true + - replace: false changelog: skip: false diff --git a/cmd/hauler/cli/download/download.go b/cmd/hauler/cli/download/download.go index fa9de87..7b39f49 100644 --- a/cmd/hauler/cli/download/download.go +++ b/cmd/hauler/cli/download/download.go @@ -16,9 +16,6 @@ import ( "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/consts" - "github.com/rancherfederal/hauler/pkg/content/chart" - "github.com/rancherfederal/hauler/pkg/content/file" - "github.com/rancherfederal/hauler/pkg/content/image" "github.com/rancherfederal/hauler/pkg/log" ) @@ -37,9 +34,6 @@ func (o *Opts) AddArgs(cmd *cobra.Command) { func Cmd(ctx context.Context, o *Opts, reference string) error { l := log.FromContext(ctx) - fs := content.NewFile(o.DestinationDir) - defer fs.Close() - rs, err := content.NewRegistry(content.RegistryOptions{}) if err != nil { return err @@ -76,53 +70,23 @@ func Cmd(ctx context.Context, o *Opts, reference string) error { outputFile = fmt.Sprintf("%s:%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier()) } - is := mapper.NewStore(o.DestinationDir, image.Mapper()) - defer is.Close() - - ms = is - - // l.Infof("downloaded image [%s] to [%s]", ref.Name(), outputFile) + s := mapper.NewMapperFileStore(o.DestinationDir, mapper.Images()) + defer s.Close() + ms = s case consts.FileLocalConfigMediaType: l.Debugf("identified [file] (%s) content", manifest.Config.MediaType) - fs := mapper.NewStore(o.DestinationDir, file.Mapper()) - defer fs.Close() - - ms = fs - - // _, err := oras.Copy(ctx, rs, ref.Name(), fs, "", - // oras.WithLayerDescriptors(func(descriptors []ocispec.Descriptor) { - // for _, desc := range descriptors { - // if _, ok := desc.Annotations[ocispec.AnnotationTitle]; !ok { - // continue - // } - // descs = append(descs, desc) - // } - // })) - // if err != nil { - // return err - // } - // - // ldescs := len(descs) - // for i, desc := range descs { - // // NOTE: This is safe without a map key check b/c we're not allowing unnamed content from oras.Pull - // l.Infof("downloaded (%d/%d) files to [%s]", i+1, ldescs, desc.Annotations[ocispec.AnnotationTitle]) - // } + s := mapper.NewMapperFileStore(o.DestinationDir, nil) + defer s.Close() + ms = s case consts.ChartLayerMediaType, consts.ChartConfigMediaType: l.Debugf("identified [chart] (%s) content", manifest.Config.MediaType) - cs := mapper.NewStore(o.DestinationDir, chart.Mapper()) - defer cs.Close() - - ms = cs - // desc, err := oras.Copy(ctx, rs, ref.Name(), fs, "") - // if err != nil { - // return err - // } - // - // l.Infof("downloaded chart [%s] to [%s]", ref.String(), desc.Annotations[ocispec.AnnotationTitle]) + s := mapper.NewMapperFileStore(o.DestinationDir, mapper.Chart()) + defer s.Close() + ms = s default: return fmt.Errorf("unrecognized content type: %s", manifest.Config.MediaType) diff --git a/cmd/hauler/cli/serve/files.go b/cmd/hauler/cli/serve/files.go index 986d25d..7ecd788 100644 --- a/cmd/hauler/cli/serve/files.go +++ b/cmd/hauler/cli/serve/files.go @@ -28,6 +28,5 @@ func FilesCmd(ctx context.Context, o *FilesOpts) error { if err := s.ListenAndServe(); err != nil { return err } - return nil } diff --git a/cmd/hauler/cli/serve/registry.go b/cmd/hauler/cli/serve/registry.go index 778f3e8..9336314 100644 --- a/cmd/hauler/cli/serve/registry.go +++ b/cmd/hauler/cli/serve/registry.go @@ -47,9 +47,6 @@ func RegistryCmd(ctx context.Context, o *RegistryOpts) error { if err := s.ListenAndServe(); err != nil { return err } - - // TODO: Graceful cancelling - return nil } diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index a921f32..4dcacb5 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -2,7 +2,6 @@ package store import ( "context" - "fmt" "strings" "github.com/pkg/errors" @@ -41,8 +40,6 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, targetRef string) return err } - fmt.Println(components[1]) - mapperFn := func(reference string) (string, error) { ref, err := store.RelocateReference(reference, components[1]) if err != nil { diff --git a/cmd/hauler/cli/store/info.go b/cmd/hauler/cli/store/info.go index 4dfce8a..4521593 100644 --- a/cmd/hauler/cli/store/info.go +++ b/cmd/hauler/cli/store/info.go @@ -15,10 +15,8 @@ import ( type InfoOpts struct{} func (o *InfoOpts) AddFlags(cmd *cobra.Command) { - f := cmd.Flags() - _ = f - - // TODO: Regex matching + _ = cmd.Flags() + // TODO: Regex/globbing } func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Store) error { diff --git a/cmd/hauler/cli/version.go b/cmd/hauler/cli/version.go index bae31a1..4914843 100644 --- a/cmd/hauler/cli/version.go +++ b/cmd/hauler/cli/version.go @@ -29,7 +29,6 @@ func addVersion(parent *cobra.Command) { return nil }, } - cmd.Flags().BoolVar(&json, "json", false, "toggle output in JSON") parent.AddCommand(cmd) diff --git a/go.mod b/go.mod index 408bf0d..cb766cf 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( github.com/dsnet/compress v0.0.1 // indirect github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/fatih/color v1.9.0 // indirect + github.com/fatih/color v1.12.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.0.1 // indirect @@ -98,7 +98,7 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -115,8 +115,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/prometheus/common v0.29.0 // indirect + github.com/prometheus/procfs v0.7.1 // indirect github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect diff --git a/go.sum b/go.sum index a41f461..0884adb 100644 --- a/go.sum +++ b/go.sum @@ -408,8 +408,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -782,13 +782,10 @@ github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -819,8 +816,9 @@ github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HK github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -968,8 +966,9 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= +github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -981,8 +980,9 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.1 h1:TlEtJq5GvGqMykEwWzbZWjjztF86swFhsPix1i0bkgA= +github.com/prometheus/procfs v0.7.1/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA= github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 h1:NxR8Fh0eE7/5/5Zvlog9B5NVjWKqBSb1WYMUF7/IE5c= @@ -1314,6 +1314,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= @@ -1360,7 +1361,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/getter/getter.go b/internal/getter/getter.go index b345edf..029d167 100644 --- a/internal/getter/getter.go +++ b/internal/getter/getter.go @@ -12,6 +12,7 @@ import ( "github.com/rancherfederal/hauler/internal/layer" "github.com/rancherfederal/hauler/pkg/artifact" + "github.com/rancherfederal/hauler/pkg/consts" ) type Client struct { @@ -60,11 +61,6 @@ func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) opener := func() (io.ReadCloser, error) { return g.Open(ctx, u) } - cfg := g.Config(u) - mt, err := cfg.MediaType() - if err != nil { - return nil, err - } annotations := make(map[string]string) annotations[ocispec.AnnotationTitle] = g.Name(u) @@ -75,7 +71,7 @@ func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) } l, err := layer.FromOpener(opener, - layer.WithMediaType(string(mt)), + layer.WithMediaType(consts.FileLayerMediaType), layer.WithAnnotations(annotations)) if err != nil { return nil, err diff --git a/internal/mapper/filestore.go b/internal/mapper/filestore.go index 5a8fab0..119108b 100644 --- a/internal/mapper/filestore.go +++ b/internal/mapper/filestore.go @@ -2,17 +2,13 @@ package mapper import ( "context" - "fmt" - "io" "io/ioutil" "os" + "path/filepath" "strings" - "time" ccontent "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/remotes" - "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "oras.land/oras-go/pkg/content" @@ -38,6 +34,7 @@ func (s *store) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) hash = parts[1] } return &pusher{ + store: s.File, tag: tag, ref: hash, mapper: s.mapper, @@ -49,142 +46,40 @@ type store struct { mapper map[string]Fn } -type pusher struct { - tag string - ref string - mapper map[string]Fn -} - func (s *pusher) Push(ctx context.Context, desc ocispec.Descriptor) (ccontent.Writer, error) { - now := time.Now() + // TODO: This is suuuuuper ugly... redo this when oras v2 is out + if _, ok := content.ResolveName(desc); ok { + p, err := s.store.Pusher(ctx, s.ref) + if err != nil { + return nil, err + } + return p.Push(ctx, desc) + } + // If no custom mapper found, fall back to content.File mapper if _, ok := s.mapper[desc.MediaType]; !ok { return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(desc.Digest)), nil } - f, err := s.mapper[desc.MediaType](desc) + filename, err := s.mapper[desc.MediaType](desc) if err != nil { return nil, err } - return &fileWriter{ - file: f, - digester: digest.Canonical.Digester(), - status: ccontent.Status{ - Ref: f.Name(), - Total: desc.Size, - StartedAt: now, - UpdatedAt: now, - }, - aftercommit: nil, - }, nil -} - -type fileWriter struct { - // store *content.File - file *os.File - digester digest.Digester - status ccontent.Status - aftercommit func() error -} - -// NewFileWriter will open a new file for writing, existing file will be truncated -func NewFileWriter(path string, perm os.FileMode, size int64) (*fileWriter, error) { - now := time.Now() - - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, perm) + fullFileName := filepath.Join(s.store.ResolvePath(""), filename) + // TODO: Don't rewrite everytime, we can check the digest + f, err := os.OpenFile(fullFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { - return nil, err + return nil, errors.Wrap(err, "pushing file") } - return &fileWriter{ - file: f, - digester: digest.Canonical.Digester(), - status: ccontent.Status{ - Ref: f.Name(), - Total: size, - StartedAt: now, - UpdatedAt: now, - }, - }, nil + w := content.NewIoContentWriter(f, content.WithInputHash(desc.Digest), content.WithOutputHash(desc.Digest)) + return w, nil } -func (w *fileWriter) Write(p []byte) (n int, err error) { - n, err = w.file.Write(p) - w.digester.Hash().Write(p[:n]) - w.status.Offset += int64(len(p)) - w.status.UpdatedAt = time.Now() - return n, err -} - -func (w *fileWriter) Close() error { - if w.file == nil { - return nil - } - w.file.Sync() - err := w.file.Close() - w.file = nil - return err -} - -func (w *fileWriter) Digest() digest.Digest { - return w.digester.Digest() -} - -func (w *fileWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...ccontent.Opt) error { - var base ccontent.Info - for _, opt := range opts { - if err := opt(&base); err != nil { - return err - } - } - - if w.file == nil { - return errors.Wrap(errdefs.ErrFailedPrecondition, "cannot commit on closed writer") - } - file := w.file - w.file = nil - - if err := file.Sync(); err != nil { - file.Close() - return errors.Wrap(err, "sync failed") - } - - fileInfo, err := file.Stat() - if err != nil { - file.Close() - return errors.Wrap(err, "stat failed") - } - if err := file.Close(); err != nil { - return errors.Wrap(err, "failed to close file") - } - - if size > 0 && size != fileInfo.Size() { - return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit size %d, expected %d", fileInfo.Size(), size) - } - if dgst := w.digester.Digest(); expected != "" && expected != dgst { - return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit digest %s, expected %s", dgst, expected) - } - - // w.store.set(w.desc) - if w.aftercommit != nil { - return w.aftercommit() - } - return nil -} - -func (w *fileWriter) Status() (ccontent.Status, error) { - return w.status, nil -} - -func (w *fileWriter) Truncate(size int64) error { - if size != 0 { - return content.ErrUnsupportedSize - } - w.status.Offset = 0 - w.digester.Hash().Reset() - if _, err := w.file.Seek(0, io.SeekStart); err != nil { - return err - } - return w.file.Truncate(0) +type pusher struct { + store *content.File + tag string + ref string + mapper map[string]Fn } diff --git a/internal/mapper/mappers.go b/internal/mapper/mappers.go index 1f28cbc..7cc72d4 100644 --- a/internal/mapper/mappers.go +++ b/internal/mapper/mappers.go @@ -2,38 +2,35 @@ package mapper import ( "fmt" - "os" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" "github.com/rancherfederal/hauler/pkg/consts" ) -type Fn func(desc ocispec.Descriptor) (*os.File, error) +type Fn func(desc ocispec.Descriptor) (string, error) func Images() map[string]Fn { m := make(map[string]Fn) - manifestMapperFn := Fn(func(desc ocispec.Descriptor) (*os.File, error) { - return os.Create("manifest.json") + manifestMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) { + return "manifest.json", nil }) for _, l := range []string{consts.DockerManifestSchema2, consts.OCIManifestSchema1} { m[l] = manifestMapperFn } - layerMapperFn := Fn(func(desc ocispec.Descriptor) (*os.File, error) { - n := fmt.Sprintf("%s.tar.gz", desc.Digest.String()) - return os.Create(n) + layerMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) { + return fmt.Sprintf("%s.tar.gz", desc.Digest.String()), nil }) for _, l := range []string{consts.OCILayer, consts.DockerLayer} { m[l] = layerMapperFn } - configMapperFn := Fn(func(desc ocispec.Descriptor) (*os.File, error) { - return os.Create("config.json") + configMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) { + return "config.json", nil }) for _, l := range []string{consts.DockerConfigJSON} { @@ -43,34 +40,19 @@ func Images() map[string]Fn { return m } -func Files() map[string]Fn { - m := make(map[string]Fn) - - blobMapperFn := Fn(func(desc ocispec.Descriptor) (*os.File, error) { - fmt.Println(desc.Annotations) - if _, ok := desc.Annotations[ocispec.AnnotationTitle]; !ok { - return nil, errors.Errorf("unkown file name") - } - return os.Create(desc.Annotations[ocispec.AnnotationTitle]) - }) - - m[consts.FileLayerMediaType] = blobMapperFn - return m -} - func Chart() map[string]Fn { m := make(map[string]Fn) - chartMapperFn := Fn(func(desc ocispec.Descriptor) (*os.File, error) { + chartMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) { f := "chart.tar.gz" if _, ok := desc.Annotations[ocispec.AnnotationTitle]; ok { f = desc.Annotations[ocispec.AnnotationTitle] } - return os.Create(f) + return f, nil }) - provMapperFn := Fn(func(desc ocispec.Descriptor) (*os.File, error) { - return os.Create("prov.json") + provMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) { + return "prov.json", nil }) m[consts.ChartLayerMediaType] = chartMapperFn diff --git a/pkg/content/chart/chart.go b/pkg/content/chart/chart.go index 42906c4..ae4c9a8 100644 --- a/pkg/content/chart/chart.go +++ b/pkg/content/chart/chart.go @@ -17,7 +17,6 @@ import ( "helm.sh/helm/v3/pkg/cli" "github.com/rancherfederal/hauler/internal/layer" - "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/artifact" "github.com/rancherfederal/hauler/pkg/consts" ) @@ -140,20 +139,3 @@ func chartOpener(path string) layer.Opener { return os.Open(path) } } - -func Mapper() map[string]mapper.Fn { - m := make(map[string]mapper.Fn) - - chartMapperFn := mapper.Fn(func(desc ocispec.Descriptor) (*os.File, error) { - // TODO: Fix this - return os.Create("chart.tar.gz") - }) - - provMapperFn := mapper.Fn(func(desc ocispec.Descriptor) (*os.File, error) { - return os.Create("prov.json") - }) - - m[consts.ChartLayerMediaType] = chartMapperFn - m[consts.ProvLayerMediaType] = provMapperFn - return m -} diff --git a/pkg/content/file/file.go b/pkg/content/file/file.go index b7a8043..db8715d 100644 --- a/pkg/content/file/file.go +++ b/pkg/content/file/file.go @@ -2,16 +2,12 @@ package file import ( "context" - "os" gv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" gtypes "github.com/google/go-containerregistry/pkg/v1/types" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" "github.com/rancherfederal/hauler/internal/getter" - "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/artifact" "github.com/rancherfederal/hauler/pkg/consts" ) @@ -111,17 +107,3 @@ func (f *file) compute() error { f.computed = true return nil } - -func Mapper() map[string]mapper.Fn { - m := make(map[string]mapper.Fn) - - blobMapperFn := mapper.Fn(func(desc ocispec.Descriptor) (*os.File, error) { - if _, ok := desc.Annotations[ocispec.AnnotationTitle]; !ok { - return nil, errors.Errorf("unkown file name") - } - return os.Create(desc.Annotations[ocispec.AnnotationTitle]) - }) - - m[consts.FileLayerMediaType] = blobMapperFn - return m -} diff --git a/pkg/content/image/image.go b/pkg/content/image/image.go index 23782b4..4ac8ab8 100644 --- a/pkg/content/image/image.go +++ b/pkg/content/image/image.go @@ -1,17 +1,11 @@ package image import ( - "fmt" - "os" - "github.com/google/go-containerregistry/pkg/name" gv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" ) var _ artifact.OCI = (*image)(nil) @@ -47,34 +41,3 @@ func NewImage(ref string) (*image, error) { Image: img, }, nil } - -func Mapper() map[string]mapper.Fn { - m := make(map[string]mapper.Fn) - - manifestMapperFn := mapper.Fn(func(desc ocispec.Descriptor) (*os.File, error) { - return os.Create("manifest.json") - }) - - for _, l := range []string{consts.DockerManifestSchema2, consts.OCIManifestSchema1} { - m[l] = manifestMapperFn - } - - layerMapperFn := mapper.Fn(func(desc ocispec.Descriptor) (*os.File, error) { - n := fmt.Sprintf("%s.tar.gz", desc.Digest.String()) - return os.Create(n) - }) - - for _, l := range []string{consts.OCILayer, consts.DockerLayer} { - m[l] = layerMapperFn - } - - configMapperFn := mapper.Fn(func(desc ocispec.Descriptor) (*os.File, error) { - return os.Create("config.json") - }) - - for _, l := range []string{consts.DockerConfigJSON} { - m[l] = configMapperFn - } - - return m -} diff --git a/pkg/store/oci.go b/pkg/store/oci.go index 53e8dda..b6c6d58 100644 --- a/pkg/store/oci.go +++ b/pkg/store/oci.go @@ -17,7 +17,6 @@ import ( "oras.land/oras-go/pkg/content" "oras.land/oras-go/pkg/target" - "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/consts" ) @@ -214,10 +213,11 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil } - writer, err := mapper.NewFileWriter(blobPath, 0644, d.Size) + f, err := os.Create(blobPath) if err != nil { return nil, err } - return writer, nil + w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest)) + return w, nil } diff --git a/pkg/store/store.go b/pkg/store/store.go index 366a1c5..a3b1678 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -3,7 +3,6 @@ package store import ( "context" "encoding/json" - "fmt" "os" "path/filepath" @@ -146,7 +145,6 @@ func (s *Store) Copy(ctx context.Context, to target.Target, toMapper func(string toRef = tr } - fmt.Println("copying to: ", toRef) _, err := oras.Copy(ctx, s.store, ref, to, toRef, oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2)) if err != nil { diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 8f7546f..ab691bc 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -102,11 +102,6 @@ func genArtifact(t *testing.T, ref string) (artifact.OCI, ocispec.Descriptor) { desc.Annotations = make(map[string]string) desc.Annotations[ocispec.AnnotationRefName] = ref - // gm, err := img.Manifest() - // if err != nil { - // t.Fatal(err) - // } - // data, err := json.Marshal(desc) if err != nil { t.Fatal(err)