From 08f566fb28e86d29e50274f358ded837d32797a7 Mon Sep 17 00:00:00 2001 From: Zack Brady Date: Wed, 5 Feb 2025 11:25:11 -0500 Subject: [PATCH] fixed remote code (#407) --- cmd/hauler/cli/store/add.go | 2 +- cmd/hauler/cli/store/load.go | 61 ++++++--------- cmd/hauler/cli/store/sync.go | 76 +++++++++---------- pkg/artifacts/file/file.go | 2 +- pkg/artifacts/file/file_test.go | 2 +- pkg/artifacts/file/options.go | 2 +- pkg/collection/imagetxt/imagetxt.go | 2 +- pkg/{artifacts/file => }/getter/directory.go | 0 pkg/{artifacts/file => }/getter/file.go | 0 pkg/{artifacts/file => }/getter/getter.go | 0 .../file => }/getter/getter_test.go | 2 +- pkg/{artifacts/file => }/getter/https.go | 0 12 files changed, 65 insertions(+), 84 deletions(-) rename pkg/{artifacts/file => }/getter/directory.go (100%) rename pkg/{artifacts/file => }/getter/file.go (100%) rename pkg/{artifacts/file => }/getter/getter.go (100%) rename pkg/{artifacts/file => }/getter/getter_test.go (97%) rename pkg/{artifacts/file => }/getter/https.go (100%) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index c1edca3..44a17ce 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -10,10 +10,10 @@ import ( "hauler.dev/go/hauler/internal/flags" v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1" "hauler.dev/go/hauler/pkg/artifacts/file" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" "hauler.dev/go/hauler/pkg/consts" "hauler.dev/go/hauler/pkg/content/chart" "hauler.dev/go/hauler/pkg/cosign" + "hauler.dev/go/hauler/pkg/getter" "hauler.dev/go/hauler/pkg/log" "hauler.dev/go/hauler/pkg/reference" "hauler.dev/go/hauler/pkg/store" diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index 98b8710..029c1a3 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -10,9 +10,9 @@ import ( "hauler.dev/go/hauler/internal/flags" "hauler.dev/go/hauler/pkg/archives" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" "hauler.dev/go/hauler/pkg/consts" "hauler.dev/go/hauler/pkg/content" + "hauler.dev/go/hauler/pkg/getter" "hauler.dev/go/hauler/pkg/log" "hauler.dev/go/hauler/pkg/store" ) @@ -50,14 +50,35 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDir string) error { l := log.FromContext(ctx) - // if archivePath detects a remote URL... download it if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") { l.Debugf("detected remote archive... starting download... [%s]", haulPath) - var err error - haulPath, err = downloadRemote(ctx, haulPath, tempDir) + + h := getter.NewHttp() + parsedURL, err := url.Parse(haulPath) if err != nil { return err } + rc, err := h.Open(ctx, parsedURL) + if err != nil { + return err + } + defer rc.Close() + + fileName := h.Name(parsedURL) + if fileName == "" { + fileName = filepath.Base(parsedURL.Path) + } + haulPath = filepath.Join(tempDir, fileName) + + out, err := os.Create(haulPath) + if err != nil { + return err + } + defer out.Close() + + if _, err = io.Copy(out, rc); err != nil { + return err + } } if err := archives.Unarchive(ctx, haulPath, tempDir); err != nil { @@ -77,35 +98,3 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi _, err = s.CopyAll(ctx, ts, nil) return err } - -// downloadRemote downloads the remote file using the existing getter -func downloadRemote(ctx context.Context, remoteURL, tempDirDest string) (string, error) { - parsedURL, err := url.Parse(remoteURL) - if err != nil { - return "", err - } - h := getter.NewHttp() - rc, err := h.Open(ctx, parsedURL) - if err != nil { - return "", err - } - defer rc.Close() - - fileName := h.Name(parsedURL) - if fileName == "" { - fileName = filepath.Base(parsedURL.Path) - } - - localPath := filepath.Join(tempDirDest, fileName) - out, err := os.Create(localPath) - if err != nil { - return "", err - } - defer out.Close() - - if _, err = io.Copy(out, rc); err != nil { - return "", err - } - - return localPath, nil -} diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 184418c..d2b07e1 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -18,12 +18,12 @@ import ( convert "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/convert" v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1" v1alpha1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1alpha1" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" tchart "hauler.dev/go/hauler/pkg/collection/chart" "hauler.dev/go/hauler/pkg/collection/imagetxt" "hauler.dev/go/hauler/pkg/consts" "hauler.dev/go/hauler/pkg/content" "hauler.dev/go/hauler/pkg/cosign" + "hauler.dev/go/hauler/pkg/getter" "hauler.dev/go/hauler/pkg/log" "hauler.dev/go/hauler/pkg/reference" "hauler.dev/go/hauler/pkg/store" @@ -84,29 +84,53 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags l.Infof("processing completed successfully") } - // if passed a local manifest, process it + // If passed a local manifest, process it for _, fileName := range o.FileName { l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir) - var localFileName string - if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") { - l.Debugf("detected remote manifest... starting download... [%s]", fileName) - var err error - localFileName, err = downloadRemote(ctx, fileName, tempDir) + haulPath := fileName + if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") { + l.Debugf("detected remote manifest... starting download... [%s]", haulPath) + + h := getter.NewHttp() + parsedURL, err := url.Parse(haulPath) if err != nil { return err } - } else { - localFileName = fileName + rc, err := h.Open(ctx, parsedURL) + if err != nil { + return err + } + defer rc.Close() + + fileName := h.Name(parsedURL) + if fileName == "" { + fileName = filepath.Base(parsedURL.Path) + } + haulPath = filepath.Join(tempDir, fileName) + + out, err := os.Create(haulPath) + if err != nil { + return err + } + defer out.Close() + + if _, err = io.Copy(out, rc); err != nil { + return err + } } - fi, err := os.Open(localFileName) + + fi, err := os.Open(haulPath) if err != nil { return err } + defer fi.Close() + err = processContent(ctx, fi, o, s, rso, ro) if err != nil { return err } + l.Infof("processing completed successfully") } @@ -446,35 +470,3 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor } return nil } - -// downloadRemote downloads the remote file using the existing getter -func downloadRemote(ctx context.Context, remoteURL, tempDirDest string) (string, error) { - parsedURL, err := url.Parse(remoteURL) - if err != nil { - return "", err - } - h := getter.NewHttp() - rc, err := h.Open(ctx, parsedURL) - if err != nil { - return "", err - } - defer rc.Close() - - fileName := h.Name(parsedURL) - if fileName == "" { - fileName = filepath.Base(parsedURL.Path) - } - - localPath := filepath.Join(tempDirDest, fileName) - out, err := os.Create(localPath) - if err != nil { - return "", err - } - defer out.Close() - - if _, err = io.Copy(out, rc); err != nil { - return "", err - } - - return localPath, nil -} diff --git a/pkg/artifacts/file/file.go b/pkg/artifacts/file/file.go index 5fca2ba..214c909 100644 --- a/pkg/artifacts/file/file.go +++ b/pkg/artifacts/file/file.go @@ -8,8 +8,8 @@ import ( gtypes "github.com/google/go-containerregistry/pkg/v1/types" "hauler.dev/go/hauler/pkg/artifacts" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" "hauler.dev/go/hauler/pkg/consts" + "hauler.dev/go/hauler/pkg/getter" ) // interface guard diff --git a/pkg/artifacts/file/file_test.go b/pkg/artifacts/file/file_test.go index c471639..062b80b 100644 --- a/pkg/artifacts/file/file_test.go +++ b/pkg/artifacts/file/file_test.go @@ -14,8 +14,8 @@ import ( "github.com/spf13/afero" "hauler.dev/go/hauler/pkg/artifacts/file" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" "hauler.dev/go/hauler/pkg/consts" + "hauler.dev/go/hauler/pkg/getter" ) var ( diff --git a/pkg/artifacts/file/options.go b/pkg/artifacts/file/options.go index ebbcdef..4bc7770 100644 --- a/pkg/artifacts/file/options.go +++ b/pkg/artifacts/file/options.go @@ -2,7 +2,7 @@ package file import ( "hauler.dev/go/hauler/pkg/artifacts" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" + "hauler.dev/go/hauler/pkg/getter" ) type Option func(*File) diff --git a/pkg/collection/imagetxt/imagetxt.go b/pkg/collection/imagetxt/imagetxt.go index 1f297c7..60f25dc 100644 --- a/pkg/collection/imagetxt/imagetxt.go +++ b/pkg/collection/imagetxt/imagetxt.go @@ -12,8 +12,8 @@ import ( "github.com/google/go-containerregistry/pkg/name" artifact "hauler.dev/go/hauler/pkg/artifacts" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" "hauler.dev/go/hauler/pkg/artifacts/image" + "hauler.dev/go/hauler/pkg/getter" "hauler.dev/go/hauler/pkg/log" ) diff --git a/pkg/artifacts/file/getter/directory.go b/pkg/getter/directory.go similarity index 100% rename from pkg/artifacts/file/getter/directory.go rename to pkg/getter/directory.go diff --git a/pkg/artifacts/file/getter/file.go b/pkg/getter/file.go similarity index 100% rename from pkg/artifacts/file/getter/file.go rename to pkg/getter/file.go diff --git a/pkg/artifacts/file/getter/getter.go b/pkg/getter/getter.go similarity index 100% rename from pkg/artifacts/file/getter/getter.go rename to pkg/getter/getter.go diff --git a/pkg/artifacts/file/getter/getter_test.go b/pkg/getter/getter_test.go similarity index 97% rename from pkg/artifacts/file/getter/getter_test.go rename to pkg/getter/getter_test.go index ce74ce6..7c69576 100644 --- a/pkg/artifacts/file/getter/getter_test.go +++ b/pkg/getter/getter_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "hauler.dev/go/hauler/pkg/artifacts/file/getter" + "hauler.dev/go/hauler/pkg/getter" ) func TestClient_Detect(t *testing.T) { diff --git a/pkg/artifacts/file/getter/https.go b/pkg/getter/https.go similarity index 100% rename from pkg/artifacts/file/getter/https.go rename to pkg/getter/https.go