diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ac50546..46896f4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -261,6 +261,8 @@ jobs: curl -sfOL https://github.com/rancherfederal/rancher-cluster-templates/releases/download/rancher-cluster-templates-0.5.2/rancher-cluster-templates-0.5.2.tgz # verify via sync hauler store sync --filename testdata/hauler-manifest-pipeline.yaml + # verify via sync with multiple files + hauler store sync --filename testdata/hauler-manifest-pipeline.yaml --filename testdata/hauler-manifest.yaml # need more tests here - name: Verify - hauler store serve diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index f885615..184418c 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -5,7 +5,9 @@ import ( "context" "fmt" "io" + "net/url" "os" + "path/filepath" "strings" "github.com/mitchellh/go-homedir" @@ -16,6 +18,7 @@ import ( convert "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/convert" v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1" v1alpha1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1alpha1" + "hauler.dev/go/hauler/pkg/artifacts/file/getter" tchart "hauler.dev/go/hauler/pkg/collection/chart" "hauler.dev/go/hauler/pkg/collection/imagetxt" "hauler.dev/go/hauler/pkg/consts" @@ -29,7 +32,21 @@ import ( func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) - // if passed products, check for a remote manifest to retrieve and use. + tempOverride := o.TempOverride + + if tempOverride == "" { + tempOverride = os.Getenv(consts.HaulerTempDir) + } + + tempDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName) + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + l.Debugf("using temporary directory at [%s]", tempDir) + + // if passed products, check for a remote manifest to retrieve and use for _, productName := range o.Products { l.Infof("processing product manifest for [%s] to store [%s]", productName, o.StoreDir) parts := strings.Split(productName, "=") @@ -54,9 +71,9 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags if err != nil { return err } - filename := fmt.Sprintf("%s-manifest.yaml", parts[0]) + fileName := fmt.Sprintf("%s-manifest.yaml", parts[0]) - fi, err := os.Open(filename) + fi, err := os.Open(fileName) if err != nil { return err } @@ -69,8 +86,20 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags // if passed a local manifest, process it for _, fileName := range o.FileName { - l.Infof("processing manifest for [%s] to store [%s]", fileName, o.StoreDir) - fi, err := os.Open(fileName) + l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir) + var localFileName string + + if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") { + l.Debugf("detected remote manifest... starting download... [%s]", fileName) + var err error + localFileName, err = downloadRemote(ctx, fileName, tempDir) + if err != nil { + return err + } + } else { + localFileName = fileName + } + fi, err := os.Open(localFileName) if err != nil { return err } @@ -417,3 +446,35 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor } return nil } + +// downloadRemote downloads the remote file using the existing getter +func downloadRemote(ctx context.Context, remoteURL, tempDirDest string) (string, error) { + parsedURL, err := url.Parse(remoteURL) + if err != nil { + return "", err + } + h := getter.NewHttp() + rc, err := h.Open(ctx, parsedURL) + if err != nil { + return "", err + } + defer rc.Close() + + fileName := h.Name(parsedURL) + if fileName == "" { + fileName = filepath.Base(parsedURL.Path) + } + + localPath := filepath.Join(tempDirDest, fileName) + out, err := os.Create(localPath) + if err != nil { + return "", err + } + defer out.Close() + + if _, err = io.Copy(out, rc); err != nil { + return "", err + } + + return localPath, nil +} diff --git a/internal/flags/sync.go b/internal/flags/sync.go index 1b9c7bc..f0fcf96 100644 --- a/internal/flags/sync.go +++ b/internal/flags/sync.go @@ -13,6 +13,7 @@ type SyncOpts struct { Platform string Registry string ProductRegistry string + TempOverride string } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { @@ -24,4 +25,5 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e linux/amd64 (defaults to all)") f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one") f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)") + f.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directiory determined by the OS") }