path rewrites (#475)

* image reference rewrite
* remove need for rewrite-provider
* handle charts
* handle leading slash and missing tags
* tests
* tool cleanup
* removed intermediate store cleanup
* fix?
* test cleanup
* clean up tools folder again
* debug test
* clear tempdir after each filename provided by load

Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com>

* update tar command in load integration test

Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com>

* clean up debug prints
* clean up debug prints

* fix typo

---------

Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com>
Signed-off-by: Zack Brady <zackbrady123@gmail.com>
Co-authored-by: Adam Martin <adam.martin@ranchergovernment.com>
Co-authored-by: Zack Brady <zackbrady123@gmail.com>

.github/workflows/tests.yaml

@@ -154,6 +154,21 @@ jobs:
           # verify via the hauler store contents
           hauler store info
 
+      - name: Verify - hauler store add chart --rewrite
+        run: |
+          # add chart with rewrite flag
+          hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.4 --rewrite custom-path/rancher:2.8.4
+          # verify new ref in store
+          hauler store info | grep 'custom-path/rancher:2.8.4'
+          # confrim leading slash trimmed from rewrite
+          hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.4 --rewrite /custom-path/rancher:2.8.4
+          # verify no leading slash
+          ! hauler store info | grep '/custom-path/rancher:2.8.4'
+          # confirm old tag used if not specified
+          hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.4 --rewrite /custom-path/rancher
+          # confirm tag 
+          hauler store info | grep '2.8.4'
+
       - name: Verify - hauler store add file
         run: |
           hauler store add file --help
@@ -178,6 +193,21 @@ jobs:
           # verify via the hauler store contents
           hauler store info
 
+      - name: Verify - hauler store add image --rewrite
+        run: |
+          # add image with rewrite flag
+          hauler store add image ghcr.io/hauler-dev/library/busybox --rewrite custom-registry.io/custom-path/busybox:latest
+          # verify new ref in store
+          hauler store info | grep 'custom-registry.io/custom-path/busybox:latest'
+          # confrim leading slash trimmed from rewrite
+          hauler store add image ghcr.io/hauler-dev/library/busybox --rewrite /custom-path/busybox:latest
+          # verify no leading slash
+          ! hauler store info | grep '/custom-path/busybox:latest'
+          # confirm old tag used if not specified
+          hauler store add image ghcr.io/hauler-dev/library/busybox:stable --rewrite /custom-path/busybox
+          # confirm tag 
+          hauler store info | grep ':stable'
+
       - name: Verify - hauler store copy
         run: |
           hauler store copy --help
@@ -229,6 +259,8 @@ jobs:
           hauler store load
           # verify via load with multiple files
           hauler store load --filename haul.tar.zst --filename store.tar.zst
+          # confirm store contents
+          tar -xOf store.tar.zst index.json
           # verify via load with filename and temp directory
           hauler store load --filename store.tar.zst --tempdir /opt
           # verify via load with filename and platform (amd64)

cmd/hauler/cli/store/add.go

@@ -2,11 +2,12 @@ package store
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"strings"
 
 	"github.com/google/go-containerregistry/pkg/name"
-	"helm.sh/helm/v3/pkg/action"
-
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"hauler.dev/go/hauler/internal/flags"
 	v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1"
 	"hauler.dev/go/hauler/pkg/artifacts/file"
@@ -17,6 +18,7 @@ import (
 	"hauler.dev/go/hauler/pkg/log"
 	"hauler.dev/go/hauler/pkg/reference"
 	"hauler.dev/go/hauler/pkg/store"
+	"helm.sh/helm/v3/pkg/action"
 )
 
 func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string) error {
@@ -57,7 +59,8 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
 	l := log.FromContext(ctx)
 
 	cfg := v1.Image{
-		Name: reference,
+		Name:    reference,
+		Rewrite: o.Rewrite,
 	}
 
 	// Check if the user provided a key.
@@ -78,10 +81,10 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
 		l.Infof("keyless signature verified for image [%s]", cfg.Name)
 	}
 
-	return storeImage(ctx, s, cfg, o.Platform, rso, ro)
+	return storeImage(ctx, s, cfg, o.Platform, rso, ro, o.Rewrite)
 }
 
-func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
+func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error {
 	l := log.FromContext(ctx)
 
 	if !ro.IgnoreErrors {
@@ -104,6 +107,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
 		}
 	}
 
+	// copy and sig verification
 	err = cosign.SaveImage(ctx, s, r.Name(), platform, rso, ro)
 	if err != nil {
 		if ro.IgnoreErrors {
@@ -115,10 +119,67 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
 		}
 	}
 
+	if rewrite != "" {
+		rewrite = strings.TrimPrefix(rewrite, "/")
+		if !strings.Contains(rewrite, ":") {
+			rewrite = strings.Join([]string{rewrite, r.(name.Tag).TagStr()}, ":")
+		}
+		// rename image name in store
+		newRef, err := name.ParseReference(rewrite)
+		if err != nil {
+			l.Errorf("unable to parse rewrite name: %w", err)
+		}
+		rewriteReference(ctx, s, r, newRef)
+	}
+
 	l.Infof("successfully added image [%s]", r.Name())
 	return nil
 }
 
+func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference) error {
+	l := log.FromContext(ctx)
+
+	l.Infof("rewriting [%s] to [%s]", oldRef.Name(), newRef.Name())
+
+	s.OCI.LoadIndex()
+
+	//TODO: improve string manipulation
+	oldRefContext := oldRef.Context()
+	newRefContext := newRef.Context()
+
+	oldRepo := oldRefContext.RepositoryStr()
+	newRepo := newRefContext.RepositoryStr()
+	oldTag := oldRef.(name.Tag).TagStr()
+	newTag := newRef.(name.Tag).TagStr()
+	oldRegistry := strings.TrimPrefix(oldRefContext.RegistryStr(), "index.")
+	newRegistry := strings.TrimPrefix(newRefContext.RegistryStr(), "index.")
+
+	oldTotal := oldRepo + ":" + oldTag
+	newTotal := newRepo + ":" + newTag
+	oldTotalReg := oldRegistry + "/" + oldTotal
+	newTotalReg := newRegistry + "/" + newTotal
+
+	//find and update reference
+	found := false
+	if err := s.OCI.Walk(func(k string, d ocispec.Descriptor) error {
+		if d.Annotations[ocispec.AnnotationRefName] == oldTotal && d.Annotations[consts.ContainerdImageNameKey] == oldTotalReg {
+			d.Annotations[ocispec.AnnotationRefName] = newTotal
+			d.Annotations[consts.ContainerdImageNameKey] = newTotalReg
+			found = true
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	if !found {
+		return fmt.Errorf("could not find image [%s] in store", oldRef.Name())
+	}
+
+	return s.OCI.SaveIndex()
+
+}
+
 func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, chartName string) error {
 	cfg := v1.Chart{
 		Name:    chartName,
@@ -126,10 +187,14 @@ func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, ch
 		Version: o.ChartOpts.Version,
 	}
 
-	return storeChart(ctx, s, cfg, o.ChartOpts)
+	rewrite := ""
+	if o.Rewrite != "" {
+		rewrite = o.Rewrite
+	}
+	return storeChart(ctx, s, cfg, o.ChartOpts, rewrite)
 }
 
-func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action.ChartPathOptions) error {
+func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action.ChartPathOptions, rewrite string) error {
 	l := log.FromContext(ctx)
 
 	l.Infof("adding chart [%s] to the store", cfg.Name)
@@ -152,11 +217,60 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action
 	if err != nil {
 		return err
 	}
+
 	_, err = s.AddOCI(ctx, chrt, ref.Name())
 	if err != nil {
 		return err
+	} else {
+		s.OCI.SaveIndex()
 	}
 
+	if rewrite != "" {
+		rewrite = strings.TrimPrefix(rewrite, "/")
+		newRef, err := name.ParseReference(rewrite)
+		if err != nil {
+			l.Errorf("unable to parse rewrite name: %w", err)
+		}
+
+		s.OCI.LoadIndex()
+
+		oldRefContext := ref.Context()
+		newRefContext := newRef.Context()
+
+		oldRepo := oldRefContext.RepositoryStr()
+		newRepo := newRefContext.RepositoryStr()
+		oldTag := ref.(name.Tag).TagStr()
+
+		var newTag string
+		if strings.Contains(rewrite, ":") {
+			newTag = newRef.(name.Tag).TagStr()
+		} else {
+			newTag = oldTag
+		}
+
+		oldTotal := oldRepo + ":" + oldTag
+		newTotal := newRepo + ":" + newTag
+
+		found := false
+		if err := s.OCI.Walk(func(k string, d ocispec.Descriptor) error {
+			if d.Annotations[ocispec.AnnotationRefName] == oldTotal {
+				d.Annotations[ocispec.AnnotationRefName] = newTotal
+				found = true
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		if !found {
+			return fmt.Errorf("could not find chart [%s] in store", ref.Name())
+		}
+
+		cfg.Name = newRef.Name()
+		fmt.Println("chart name (new): ", cfg.Name)
+
+		s.OCI.SaveIndex()
+	}
 	l.Infof("successfully added chart [%s]", ref.Name())
 	return nil
 }

cmd/hauler/cli/store/load.go

@@ -44,6 +44,7 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r
 		if err != nil {
 			return err
 		}
+		clearDir(tempDir)
 	}
 
 	return nil
@@ -137,3 +138,19 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
 	_, err = s.CopyAll(ctx, ts, nil)
 	return err
 }
+
+func clearDir(path string) error {
+	entries, err := os.ReadDir(path)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		err = os.RemoveAll(filepath.Join(path, entry.Name()))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/hauler/cli/store/sync.go

@@ -63,7 +63,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
 		img := v1.Image{
 			Name: manifestLoc,
 		}
-		err := storeImage(ctx, s, img, o.Platform, rso, ro)
+		err := storeImage(ctx, s, img, o.Platform, rso, ro, "")
 		if err != nil {
 			return err
 		}
@@ -338,7 +338,12 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 						platform = i.Platform
 					}
 
-					if err := storeImage(ctx, s, i, platform, rso, ro); err != nil {
+					rewrite := ""
+					if i.Rewrite != "" {
+						rewrite = i.Rewrite
+					}
+
+					if err := storeImage(ctx, s, i, platform, rso, ro, rewrite); err != nil {
 						return err
 					}
 				}
@@ -473,7 +478,12 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 						platform = i.Platform
 					}
 
-					if err := storeImage(ctx, s, i, platform, rso, ro); err != nil {
+					rewrite := ""
+					if i.Rewrite != "" {
+						rewrite = i.Rewrite
+					}
+
+					if err := storeImage(ctx, s, i, platform, rso, ro, rewrite); err != nil {
 						return err
 					}
 				}
@@ -496,8 +506,8 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 				if err := convert.ConvertCharts(&alphaCfg, &v1Cfg); err != nil {
 					return err
 				}
-				for _, ch := range v1Cfg.Spec.Charts {
-					if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}); err != nil {
+				for i, ch := range v1Cfg.Spec.Charts {
+					if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}, v1Cfg.Spec.Charts[i].Rewrite); err != nil {
 						return err
 					}
 				}
@@ -507,8 +517,8 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 				if err := yaml.Unmarshal(doc, &cfg); err != nil {
 					return err
 				}
-				for _, ch := range cfg.Spec.Charts {
-					if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}); err != nil {
+				for i, ch := range cfg.Spec.Charts {
+					if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}, cfg.Spec.Charts[i].Rewrite); err != nil {
 						return err
 					}
 				}

internal/flags/add.go

@@ -16,6 +16,7 @@ type AddImageOpts struct {
 	CertGithubWorkflowRepository string
 	Tlog                         bool
 	Platform                     string
+	Rewrite                      string
 }
 
 func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
@@ -28,8 +29,11 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
 	f.StringVar(&o.CertGithubWorkflowRepository, "certificate-github-workflow-repository", "", "(Optional) Cosign certificate-github-workflow-repository option")
 	f.BoolVarP(&o.Tlog, "use-tlog-verify", "v", false, "(Optional) Allow transparency log verification. (defaults to false)")
 	f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specifiy the platform of the image... i.e. linux/amd64 (defaults to all)")
+	f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string")
 }
 
+//func (o *AddImageOpts) RewriteValue() string { return o.Rewrite }
+
 type AddFileOpts struct {
 	*StoreRootOpts
 	Name string
@@ -44,6 +48,7 @@ type AddChartOpts struct {
 	*StoreRootOpts
 
 	ChartOpts *action.ChartPathOptions
+	Rewrite   string
 }
 
 func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
@@ -58,4 +63,5 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
 	f.StringVar(&o.ChartOpts.KeyFile, "key-file", "", "(Optional) Location of the TLS Key to use for client authenication")
 	f.BoolVar(&o.ChartOpts.InsecureSkipTLSverify, "insecure-skip-tls-verify", false, "(Optional) Skip TLS certificate verification")
 	f.StringVar(&o.ChartOpts.CaFile, "ca-file", "", "(Optional) Location of CA Bundle to enable certification verification")
+	f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string")
 }

internal/flags/sync.go

@@ -20,6 +20,7 @@ type SyncOpts struct {
 	ProductRegistry              string
 	TempOverride                 string
 	Tlog                         bool
+	Rewrite                      string
 }
 
 func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
@@ -38,4 +39,7 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
 	f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)")
 	f.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directiory determined by the OS")
 	f.BoolVarP(&o.Tlog, "use-tlog-verify", "v", false, "(Optional) Allow transparency log verification. (defaults to false)")
+	f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string")
 }
+
+//func (o *SyncOpts) RewriteValue() string { return o.Rewrite }

pkg/apis/hauler.cattle.io/v1/chart.go

@@ -19,6 +19,7 @@ type Chart struct {
 	Name    string `json:"name,omitempty"`
 	RepoURL string `json:"repoURL,omitempty"`
 	Version string `json:"version,omitempty"`
+	Rewrite string `json:"rewrite,omitempty"`
 }
 
 type ThickCharts struct {

pkg/apis/hauler.cattle.io/v1/image.go

@@ -37,4 +37,5 @@ type Image struct {
 	// Platform of the image to be pulled.  If not specified, all platforms will be pulled.
 	//Platform string `json:"key,omitempty"`
 	Platform string `json:"platform"`
+	Rewrite  string `json:"rewrite"`
 }

pkg/apis/hauler.cattle.io/v1alpha1/chart.go

@@ -19,6 +19,7 @@ type Chart struct {
 	Name    string `json:"name,omitempty"`
 	RepoURL string `json:"repoURL,omitempty"`
 	Version string `json:"version,omitempty"`
+	Rewrite string `json:"rewrite,omitempty"`
 }
 
 type ThickCharts struct {

pkg/apis/hauler.cattle.io/v1alpha1/image.go

@@ -37,4 +37,5 @@ type Image struct {
 	// Platform of the image to be pulled.  If not specified, all platforms will be pulled.
 	//Platform string `json:"key,omitempty"`
 	Platform string `json:"platform"`
+	Rewrite  string `json:"rewrite"`
 }

pkg/consts/consts.go

@@ -50,6 +50,7 @@ const (
 	ImageAnnotationPlatform = "hauler.dev/platform"
 	ImageAnnotationRegistry = "hauler.dev/registry"
 	ImageAnnotationTlog     = "hauler.dev/use-tlog-verify"
+	ImageAnnotationRewrite  = "hauler.dev/rewrite"
 	ImageRefKey             = "org.opencontainers.image.ref.name"
 
 	// cosign keyless validation options