added remote and multiple file fetch to sync (#405)

* updated sync flag and related logs [breaking change]
* fixed typo in sync flag name
* and the last typo fix...
* added remote and multiple file fetch to sync

---------

Signed-off-by: Zack Brady <zackbrady123@gmail.com>

.github/workflows/tests.yaml

@@ -261,6 +261,8 @@ jobs:
           curl -sfOL https://github.com/rancherfederal/rancher-cluster-templates/releases/download/rancher-cluster-templates-0.5.2/rancher-cluster-templates-0.5.2.tgz
           # verify via sync
           hauler store sync --filename testdata/hauler-manifest-pipeline.yaml
+          # verify via sync with multiple files
+          hauler store sync --filename testdata/hauler-manifest-pipeline.yaml --filename testdata/hauler-manifest.yaml
           # need more tests here
 
       - name: Verify - hauler store serve

cmd/hauler/cli/store/sync.go

@@ -5,7 +5,9 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/mitchellh/go-homedir"
@@ -16,6 +18,7 @@ import (
 	convert "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/convert"
 	v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1"
 	v1alpha1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
+	"hauler.dev/go/hauler/pkg/artifacts/file/getter"
 	tchart "hauler.dev/go/hauler/pkg/collection/chart"
 	"hauler.dev/go/hauler/pkg/collection/imagetxt"
 	"hauler.dev/go/hauler/pkg/consts"
@@ -29,7 +32,21 @@ import (
 func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
 	l := log.FromContext(ctx)
 
-	// if passed products, check for a remote manifest to retrieve and use.
+	tempOverride := o.TempOverride
+
+	if tempOverride == "" {
+		tempOverride = os.Getenv(consts.HaulerTempDir)
+	}
+
+	tempDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	l.Debugf("using temporary directory at [%s]", tempDir)
+
+	// if passed products, check for a remote manifest to retrieve and use
 	for _, productName := range o.Products {
 		l.Infof("processing product manifest for [%s] to store [%s]", productName, o.StoreDir)
 		parts := strings.Split(productName, "=")
@@ -54,9 +71,9 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
 		if err != nil {
 			return err
 		}
-		filename := fmt.Sprintf("%s-manifest.yaml", parts[0])
+		fileName := fmt.Sprintf("%s-manifest.yaml", parts[0])
 
-		fi, err := os.Open(filename)
+		fi, err := os.Open(fileName)
 		if err != nil {
 			return err
 		}
@@ -69,8 +86,20 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
 
 	// if passed a local manifest, process it
 	for _, fileName := range o.FileName {
-		l.Infof("processing manifest for [%s] to store [%s]", fileName, o.StoreDir)
-		fi, err := os.Open(fileName)
+		l.Infof("processing manifest [%s] to store [%s]", fileName, o.StoreDir)
+		var localFileName string
+
+		if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") {
+			l.Debugf("detected remote manifest... starting download... [%s]", fileName)
+			var err error
+			localFileName, err = downloadRemote(ctx, fileName, tempDir)
+			if err != nil {
+				return err
+			}
+		} else {
+			localFileName = fileName
+		}
+		fi, err := os.Open(localFileName)
 		if err != nil {
 			return err
 		}
@@ -417,3 +446,35 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 	}
 	return nil
 }
+
+// downloadRemote downloads the remote file using the existing getter
+func downloadRemote(ctx context.Context, remoteURL, tempDirDest string) (string, error) {
+	parsedURL, err := url.Parse(remoteURL)
+	if err != nil {
+		return "", err
+	}
+	h := getter.NewHttp()
+	rc, err := h.Open(ctx, parsedURL)
+	if err != nil {
+		return "", err
+	}
+	defer rc.Close()
+
+	fileName := h.Name(parsedURL)
+	if fileName == "" {
+		fileName = filepath.Base(parsedURL.Path)
+	}
+
+	localPath := filepath.Join(tempDirDest, fileName)
+	out, err := os.Create(localPath)
+	if err != nil {
+		return "", err
+	}
+	defer out.Close()
+
+	if _, err = io.Copy(out, rc); err != nil {
+		return "", err
+	}
+
+	return localPath, nil
+}

internal/flags/sync.go

@@ -13,6 +13,7 @@ type SyncOpts struct {
 	Platform        string
 	Registry        string
 	ProductRegistry string
+	TempOverride    string
 }
 
 func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
@@ -24,4 +25,5 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
 	f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e linux/amd64 (defaults to all)")
 	f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one")
 	f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)")
+	f.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directiory determined by the OS")
 }