From f059a135da3df60ab96ac9bd86dc60b157185a7d Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Tue, 7 Apr 2026 12:48:05 -0400 Subject: [PATCH] add optional flag for excluding extra artifacts when pulling from a registry (#541) * add optional flag for excluding extra artifacts when pulling from a registry Signed-off-by: Adam Martin * add optional flag to charts for excluding extra artifacts when pulling from a registry Signed-off-by: Adam Martin --------- Signed-off-by: Adam Martin --- cmd/hauler/cli/store/add.go | 8 +- cmd/hauler/cli/store/add_test.go | 232 +++++++++++++++++++++++-- cmd/hauler/cli/store/copy_test.go | 10 +- cmd/hauler/cli/store/extract_test.go | 10 +- cmd/hauler/cli/store/lifecycle_test.go | 2 +- cmd/hauler/cli/store/save_test.go | 10 +- cmd/hauler/cli/store/sync.go | 26 ++- internal/flags/add.go | 4 + internal/flags/sync.go | 2 + pkg/apis/hauler.cattle.io/v1/chart.go | 1 + pkg/apis/hauler.cattle.io/v1/image.go | 5 +- pkg/consts/consts.go | 13 +- pkg/store/store.go | 14 +- pkg/store/store_test.go | 4 +- 14 files changed, 292 insertions(+), 49 deletions(-) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index f7373a4..9f92288 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -92,10 +92,10 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re l.Infof("keyless signature verified for image [%s]", cfg.Name) } - return storeImage(ctx, s, cfg, o.Platform, rso, ro, o.Rewrite) + return storeImage(ctx, s, cfg, o.Platform, o.ExcludeExtras, rso, ro, o.Rewrite) } -func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error { +func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, excludeExtras bool, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error { l := log.FromContext(ctx) if !ro.IgnoreErrors { @@ -120,7 +120,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin // fetch image along with any associated signatures and attestations err = retry.Operation(ctx, rso, ro, func() error { - return s.AddImage(ctx, r.Name(), platform) + return s.AddImage(ctx, r.Name(), platform, excludeExtras) }) if err != nil { if ro.IgnoreErrors { @@ -522,7 +522,7 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. } imgCfg := v1.Image{Name: image} - if err := storeImage(ctx, s, imgCfg, opts.Platform, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, imgCfg, opts.Platform, opts.ExcludeExtras, rso, ro, ""); err != nil { if ro.IgnoreErrors { l.Warnf("%s ↳ failed to store image [%s]: %v... skipping...", prefix, image, err) continue diff --git a/cmd/hauler/cli/store/add_test.go b/cmd/hauler/cli/store/add_test.go index 82799f0..14b2a82 100644 --- a/cmd/hauler/cli/store/add_test.go +++ b/cmd/hauler/cli/store/add_test.go @@ -14,6 +14,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" ocispec "github.com/opencontainers/image-spec/specs-go/v1" helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" "hauler.dev/go/hauler/internal/flags" v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1" @@ -236,7 +237,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/src/repo:v1", "", rOpts...); err != nil { + if err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -338,7 +339,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/src/repo:v1", "", rOpts...); err != nil { + if err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -467,7 +468,7 @@ func TestStoreImage(t *testing.T) { ro := defaultCliOpts() ro.IgnoreErrors = tc.ignoreErrors - err := storeImage(ctx, s, v1.Image{Name: tc.imageName}, "", rso, ro, "") + err := storeImage(ctx, s, v1.Image{Name: tc.imageName}, "", false, rso, ro, "") if (err != nil) != tc.wantErr { t.Fatalf("error = %v, wantErr %v", err, tc.wantErr) } @@ -488,7 +489,7 @@ func TestStoreImage_Rewrite(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v1"}, "", rso, ro, "newrepo/img:v2") + err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v1"}, "", false, rso, ro, "newrepo/img:v2") if err != nil { t.Fatalf("storeImage with rewrite: %v", err) } @@ -501,7 +502,7 @@ func TestStoreImage_Rewrite(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v3"}, "", rso, ro, "newrepo/img") + err := storeImage(ctx, s, v1.Image{Name: host + "/src/repo:v3"}, "", false, rso, ro, "newrepo/img") if err != nil { t.Fatalf("storeImage with tagless rewrite: %v", err) } @@ -521,7 +522,7 @@ func TestStoreImage_Rewrite(t *testing.T) { ro := defaultCliOpts() digestRef := host + "/src/repo@" + h.String() - err = storeImage(ctx, s, v1.Image{Name: digestRef}, "", rso, ro, "newrepo/img") + err = storeImage(ctx, s, v1.Image{Name: digestRef}, "", false, rso, ro, "newrepo/img") if err == nil { t.Fatal("expected error for digest ref rewrite without explicit tag, got nil") } @@ -540,7 +541,7 @@ func TestStoreImage_MultiArch(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v1"}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v1"}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage multi-arch index: %v", err) } // Full index (both platforms) must be stored as an index, not a single image. @@ -556,7 +557,7 @@ func TestStoreImage_PlatformFilter(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v2"}, "linux/amd64", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/multiarch:v2"}, "linux/amd64", false, rso, ro, ""); err != nil { t.Fatalf("storeImage with platform filter: %v", err) } // Platform filter resolves a single manifest from the index → stored as a single image. @@ -574,7 +575,7 @@ func TestStoreImage_CosignV2Artifacts(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v1"}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v1"}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage: %v", err) } assertArtifactKindInStore(t, s, "test/signed:v1", consts.KindAnnotationSigs) @@ -593,12 +594,109 @@ func TestStoreImage_CosignV3Referrer(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: host + "/test/image:v1"}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/image:v1"}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage: %v", err) } assertReferrerInStore(t, s, "test/image:v1") } +func TestStoreImage_ExcludeExtras(t *testing.T) { + ctx := newTestContext(t) + + t.Run("cosign v2 artifacts excluded when excludeExtras=true", func(t *testing.T) { + host, rOpts := newLocalhostRegistry(t) + + img := seedImage(t, host, "test/signed", "v1", rOpts...) + seedCosignV2Artifacts(t, host, "test/signed", img, rOpts...) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v1"}, "", true, rso, ro, ""); err != nil { + t.Fatalf("storeImage with excludeExtras: %v", err) + } + + // Only the primary image must be present — no sigs, atts, or sboms. + count := countArtifactsInStore(t, s) + if count != 1 { + t.Errorf("expected 1 artifact in store, got %d", count) + } + assertArtifactKindInStore(t, s, "test/signed:v1", consts.KindAnnotationImage) + + // Verify no sig/att/sbom kind annotations are present. + for _, kind := range []string{consts.KindAnnotationSigs, consts.KindAnnotationAtts, consts.KindAnnotationSboms} { + found := false + if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error { + if desc.Annotations[consts.KindAnnotationName] == kind { + found = true + } + return nil + }); err != nil { + t.Fatalf("walk: %v", err) + } + if found { + t.Errorf("unexpected artifact with kind %q found in store", kind) + } + } + }) + + t.Run("OCI 1.1 referrers excluded when excludeExtras=true", func(t *testing.T) { + host, rOpts := newLocalhostRegistry(t) + + img := seedImage(t, host, "test/image", "v1", rOpts...) + seedOCI11Referrer(t, host, "test/image", img, rOpts...) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/image:v1"}, "", true, rso, ro, ""); err != nil { + t.Fatalf("storeImage with excludeExtras: %v", err) + } + + // Only the primary image must be present — no referrers. + count := countArtifactsInStore(t, s) + if count != 1 { + t.Errorf("expected 1 artifact in store, got %d", count) + } + + // Verify no referrer kind annotations are present. + found := false + if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error { + if strings.HasPrefix(desc.Annotations[consts.KindAnnotationName], consts.KindAnnotationReferrers) { + found = true + } + return nil + }); err != nil { + t.Fatalf("walk: %v", err) + } + if found { + t.Errorf("unexpected OCI referrer found in store when excludeExtras=true") + } + }) + + t.Run("cosign v2 artifacts included when excludeExtras=false", func(t *testing.T) { + host, rOpts := newLocalhostRegistry(t) + + img := seedImage(t, host, "test/signed", "v2", rOpts...) + seedCosignV2Artifacts(t, host, "test/signed", img, rOpts...) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + if err := storeImage(ctx, s, v1.Image{Name: host + "/test/signed:v2"}, "", false, rso, ro, ""); err != nil { + t.Fatalf("storeImage without excludeExtras: %v", err) + } + + // All four artifacts (image + sig + att + sbom) must be present. + assertArtifactKindInStore(t, s, "test/signed:v2", consts.KindAnnotationSigs) + assertArtifactKindInStore(t, s, "test/signed:v2", consts.KindAnnotationAtts) + assertArtifactKindInStore(t, s, "test/signed:v2", consts.KindAnnotationSboms) + }) +} + func TestAddChartCmd_LocalTgz(t *testing.T) { ctx := newTestContext(t) s := newTestStore(t) @@ -643,3 +741,117 @@ func TestStoreChart_Rewrite(t *testing.T) { } assertArtifactInStore(t, s, "myorg/custom-chart") } + +// seedChartWithImages builds a minimal Helm chart whose helm.sh/images +// annotation lists the given image refs and saves it as a .tgz into dir. +// Returns the path to the saved .tgz file. +func seedChartWithImages(t *testing.T, dir string, images []string) string { + t.Helper() + + // Build a helm.sh/images YAML list from the image refs. + var sb strings.Builder + for _, img := range images { + sb.WriteString("- image: ") + sb.WriteString(img) + sb.WriteString("\n") + } + + c := &helmchart.Chart{ + Metadata: &helmchart.Metadata{ + APIVersion: "v2", + Name: "test-chart", + Version: "0.1.0", + Annotations: map[string]string{ + "helm.sh/images": sb.String(), + }, + }, + } + + saved, err := chartutil.Save(c, dir) + if err != nil { + t.Fatalf("seedChartWithImages: chartutil.Save: %v", err) + } + return saved +} + +func TestStoreChart_AddImages_ExcludeExtras(t *testing.T) { + ctx := newTestContext(t) + host, rOpts := newLocalhostRegistry(t) + + // Seed an image with cosign v2 artifacts (sig + att + sbom). + img := seedImage(t, host, "test/chart-image", "v1", rOpts...) + seedCosignV2Artifacts(t, host, "test/chart-image", img, rOpts...) + + // Build a minimal chart whose helm.sh/images annotation references the image. + chartDir := t.TempDir() + imageRef := host + "/test/chart-image:v1" + tgzPath := seedChartWithImages(t, chartDir, []string{imageRef}) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + t.Run("excludeExtras=true suppresses sigs/atts/sboms for chart-discovered images", func(t *testing.T) { + o := &flags.AddChartOpts{ + ChartOpts: newAddChartOpts("", "").ChartOpts, + AddImages: true, + ExcludeExtras: true, + } + if err := storeChart(ctx, s, v1.Chart{Name: tgzPath}, o, rso, ro, ""); err != nil { + t.Fatalf("storeChart with ExcludeExtras: %v", err) + } + + // The chart itself is stored as an OCI image artifact. + assertArtifactInStore(t, s, "test-chart") + // The discovered image is stored (bare, no extras). + assertArtifactInStore(t, s, "test/chart-image:v1") + + // No sig / att / sbom entries must be present. + for _, kind := range []string{consts.KindAnnotationSigs, consts.KindAnnotationAtts, consts.KindAnnotationSboms} { + found := false + if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error { + if desc.Annotations[consts.KindAnnotationName] == kind { + found = true + } + return nil + }); err != nil { + t.Fatalf("walk: %v", err) + } + if found { + t.Errorf("unexpected artifact with kind %q found in store when ExcludeExtras=true", kind) + } + } + }) +} + +func TestStoreChart_AddImages_IncludeExtras(t *testing.T) { + ctx := newTestContext(t) + host, rOpts := newLocalhostRegistry(t) + + // Seed an image with cosign v2 artifacts. + img := seedImage(t, host, "test/chart-image", "v2", rOpts...) + seedCosignV2Artifacts(t, host, "test/chart-image", img, rOpts...) + + chartDir := t.TempDir() + imageRef := host + "/test/chart-image:v2" + tgzPath := seedChartWithImages(t, chartDir, []string{imageRef}) + + s := newTestStore(t) + rso := defaultRootOpts(s.Root) + ro := defaultCliOpts() + + t.Run("excludeExtras=false includes sigs/atts/sboms for chart-discovered images", func(t *testing.T) { + o := &flags.AddChartOpts{ + ChartOpts: newAddChartOpts("", "").ChartOpts, + AddImages: true, + ExcludeExtras: false, + } + if err := storeChart(ctx, s, v1.Chart{Name: tgzPath}, o, rso, ro, ""); err != nil { + t.Fatalf("storeChart without ExcludeExtras: %v", err) + } + + assertArtifactKindInStore(t, s, "test/chart-image:v2", consts.KindAnnotationSigs) + assertArtifactKindInStore(t, s, "test/chart-image:v2", consts.KindAnnotationAtts) + assertArtifactKindInStore(t, s, "test/chart-image:v2", consts.KindAnnotationSboms) + }) +} diff --git a/cmd/hauler/cli/store/copy_test.go b/cmd/hauler/cli/store/copy_test.go index 32620cd..28b41f5 100644 --- a/cmd/hauler/cli/store/copy_test.go +++ b/cmd/hauler/cli/store/copy_test.go @@ -90,7 +90,7 @@ func TestCopyCmd_Registry(t *testing.T) { s := newTestStore(t) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/copy:v1"}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/copy:v1"}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage: %v", err) } @@ -126,7 +126,7 @@ func TestCopyCmd_Registry_OnlyFilter(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() for _, repo := range []string{"myorg/repo1:v1", "myorg/repo2:v1"} { - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/" + repo}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/" + repo}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage %s: %v", repo, err) } } @@ -173,7 +173,7 @@ func TestCopyCmd_Registry_SigTagDerivation(t *testing.T) { // AddImage discovers and stores the .sig/.att/.sbom tags automatically. s := newTestStore(t) - if err := s.AddImage(ctx, srcHost+"/test/signed:v1", ""); err != nil { + if err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -213,7 +213,7 @@ func TestCopyCmd_Registry_IgnoreErrors(t *testing.T) { s := newTestStore(t) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/ignore:v1"}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/ignore:v1"}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage: %v", err) } @@ -273,7 +273,7 @@ func TestCopyCmd_Dir_SkipsImages(t *testing.T) { s := newTestStore(t) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/imgskip:v1"}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, s, v1.Image{Name: srcHost + "/test/imgskip:v1"}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage: %v", err) } diff --git a/cmd/hauler/cli/store/extract_test.go b/cmd/hauler/cli/store/extract_test.go index 5349c20..6d951e0 100644 --- a/cmd/hauler/cli/store/extract_test.go +++ b/cmd/hauler/cli/store/extract_test.go @@ -164,7 +164,7 @@ func TestExtractCmd_OciArtifactKindImage(t *testing.T) { // Pull into a fresh store — AddImage sets kind=KindAnnotationImage on all manifests. s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", rOpts...); err != nil { + if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -246,7 +246,7 @@ func TestExtractCmd_OciImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", rOpts...); err != nil { + if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -356,7 +356,7 @@ func TestExtractCmd_NestedImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", rOpts...); err != nil { + if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -424,7 +424,7 @@ func TestExtractCmd_ContainerImage_Skipped(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", rOpts...); err != nil { + if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -494,7 +494,7 @@ func TestExtractCmd_ContainerImageIndex_Skipped(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", rOpts...); err != nil { + if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/lifecycle_test.go b/cmd/hauler/cli/store/lifecycle_test.go index a7c89a7..f1f7a9e 100644 --- a/cmd/hauler/cli/store/lifecycle_test.go +++ b/cmd/hauler/cli/store/lifecycle_test.go @@ -109,7 +109,7 @@ func TestLifecycle_Image_AddSaveLoadCopyRegistry(t *testing.T) { storeA := newTestStore(t) rso := defaultRootOpts(storeA.Root) ro := defaultCliOpts() - if err := storeImage(ctx, storeA, v1.Image{Name: srcHost + "/lifecycle/app:v1"}, "", rso, ro, ""); err != nil { + if err := storeImage(ctx, storeA, v1.Image{Name: srcHost + "/lifecycle/app:v1"}, "", false, rso, ro, ""); err != nil { t.Fatalf("storeImage: %v", err) } assertArtifactInStore(t, storeA, "lifecycle/app:v1") diff --git a/cmd/hauler/cli/store/save_test.go b/cmd/hauler/cli/store/save_test.go index b1a58bc..5902519 100644 --- a/cmd/hauler/cli/store/save_test.go +++ b/cmd/hauler/cli/store/save_test.go @@ -59,7 +59,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v1", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/multiarch:v1", ""); err != nil { + if err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -78,7 +78,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v2", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/multiarch:v2", ""); err != nil { + if err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -143,7 +143,7 @@ func TestSaveCmd(t *testing.T) { seedImage(t, host, "test/save", "v1") s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/save:v1", ""); err != nil { + if err := s.AddImage(ctx, host+"/test/save:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -176,7 +176,7 @@ func TestSaveCmd_ContainerdCompatibility(t *testing.T) { seedImage(t, host, "test/containerd-compat", "v1") s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/containerd-compat:v1", ""); err != nil { + if err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -275,7 +275,7 @@ func TestSaveCmd_ChunkSize(t *testing.T) { seedImage(t, host, "test/chunksave", "v1") s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/chunksave:v1", ""); err != nil { + if err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index dbbebf0..72e57ea 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -59,7 +59,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags img := v1.Image{ Name: manifestLoc, } - err := storeImage(ctx, s, img, o.Platform, rso, ro, "") + err := storeImage(ctx, s, img, o.Platform, o.ExcludeExtras, rso, ro, "") if err != nil { return err } @@ -363,7 +363,15 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor rewrite = i.Rewrite } - if err := storeImage(ctx, s, i, platform, rso, ro, rewrite); err != nil { + excludeExtras := o.ExcludeExtras + if !o.ExcludeExtras && a[consts.ImageAnnotationExcludeExtras] == "true" { + excludeExtras = true + } + if i.ExcludeExtras { + excludeExtras = i.ExcludeExtras + } + + if err := storeImage(ctx, s, i, platform, excludeExtras, rso, ro, rewrite); err != nil { return err } } @@ -381,14 +389,23 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor return err } registry := o.Registry + annotation := cfg.GetAnnotations() if registry == "" { - annotation := cfg.GetAnnotations() if annotation != nil { registry = annotation[consts.ImageAnnotationRegistry] } } for i, ch := range cfg.Spec.Charts { + // Resolve excludeExtras: per-chart field > chart manifest annotation > CLI flag. + excludeExtras := o.ExcludeExtras + if !o.ExcludeExtras && annotation != nil && annotation[consts.ImageAnnotationExcludeExtras] == "true" { + excludeExtras = true + } + if ch.ExcludeExtras { + excludeExtras = ch.ExcludeExtras + } + if err := storeChart(ctx, s, ch, &flags.AddChartOpts{ ChartOpts: &action.ChartPathOptions{ @@ -397,6 +414,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor }, AddImages: ch.AddImages, AddDependencies: ch.AddDependencies, + ExcludeExtras: excludeExtras, Registry: registry, Platform: o.Platform, }, @@ -429,7 +447,7 @@ func processImageTxt(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *sto } img := v1.Image{Name: line} l.Infof("adding image [%s] to the store [%s]", line, o.StoreDir) - if err := storeImage(ctx, s, img, o.Platform, rso, ro, ""); err != nil { + if err := storeImage(ctx, s, img, o.Platform, o.ExcludeExtras, rso, ro, ""); err != nil { return err } } diff --git a/internal/flags/add.go b/internal/flags/add.go index e2bb725..b927866 100644 --- a/internal/flags/add.go +++ b/internal/flags/add.go @@ -17,6 +17,7 @@ type AddImageOpts struct { Tlog bool Platform string Rewrite string + ExcludeExtras bool } func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { @@ -30,6 +31,7 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { f.BoolVar(&o.Tlog, "use-tlog-verify", false, "(Optional) Enable transparency log verification for key-based signature verification (keyless/OIDC verification always uses the tlog)") f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e. linux/amd64 (defaults to all)") f.StringVar(&o.Rewrite, "rewrite", "", "(EXPERIMENTAL & Optional) Rewrite artifact path to specified string") + f.BoolVar(&o.ExcludeExtras, "exclude-extras", false, "(Optional) Exclude cosign signatures, attestations, SBOMs, and OCI referrers when pulling the image") } type AddFileOpts struct { @@ -49,6 +51,7 @@ type AddChartOpts struct { Rewrite string AddDependencies bool AddImages bool + ExcludeExtras bool HelmValues string Platform string Registry string @@ -74,6 +77,7 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) { cmd.Flags().BoolVar(&o.AddDependencies, "add-dependencies", false, "(EXPERIMENTAL & Optional) Fetch dependent helm charts") f.BoolVar(&o.AddImages, "add-images", false, "(EXPERIMENTAL & Optional) Fetch images referenced in helm charts") + f.BoolVar(&o.ExcludeExtras, "exclude-extras", false, "(Optional) Exclude cosign signatures, attestations, SBOMs, and OCI referrers when pulling images discovered via --add-images") f.StringVar(&o.HelmValues, "values", "", "(EXPERIMENTAL & Optional) Specify helm chart values when fetching images") f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image, e.g. linux/amd64") f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one") diff --git a/internal/flags/sync.go b/internal/flags/sync.go index ff0aaae..455808e 100644 --- a/internal/flags/sync.go +++ b/internal/flags/sync.go @@ -20,6 +20,7 @@ type SyncOpts struct { ProductRegistry string Tlog bool Rewrite string + ExcludeExtras bool } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { @@ -39,4 +40,5 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)") f.BoolVar(&o.Tlog, "use-tlog-verify", false, "(Optional) Allow transparency log verification (defaults to false)") f.StringVar(&o.Rewrite, "rewrite", "", "(EXPERIMENTAL & Optional) Rewrite artifact path to specified string") + f.BoolVar(&o.ExcludeExtras, "exclude-extras", false, "(Optional) Exclude cosign signatures, attestations, SBOMs, and OCI referrers when pulling images") } diff --git a/pkg/apis/hauler.cattle.io/v1/chart.go b/pkg/apis/hauler.cattle.io/v1/chart.go index b80de7b..c87897a 100644 --- a/pkg/apis/hauler.cattle.io/v1/chart.go +++ b/pkg/apis/hauler.cattle.io/v1/chart.go @@ -23,4 +23,5 @@ type Chart struct { AddImages bool `json:"add-images,omitempty"` AddDependencies bool `json:"add-dependencies,omitempty"` + ExcludeExtras bool `json:"exclude-extras,omitempty"` } diff --git a/pkg/apis/hauler.cattle.io/v1/image.go b/pkg/apis/hauler.cattle.io/v1/image.go index b213156..b5cc5b6 100644 --- a/pkg/apis/hauler.cattle.io/v1/image.go +++ b/pkg/apis/hauler.cattle.io/v1/image.go @@ -36,6 +36,7 @@ type Image struct { // Platform of the image to be pulled. If not specified, all platforms will be pulled. //Platform string `json:"key,omitempty"` - Platform string `json:"platform"` - Rewrite string `json:"rewrite"` + Platform string `json:"platform"` + Rewrite string `json:"rewrite"` + ExcludeExtras bool `json:"exclude-extras"` } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 1a41854..6fd8d6d 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -59,12 +59,13 @@ const ( SigstoreBundleMediaType = "application/vnd.dev.sigstore.bundle.v0.3+json" OCIEmptyConfigMediaType = "application/vnd.oci.empty.v1+json" - ImageAnnotationKey = "hauler.dev/key" - ImageAnnotationPlatform = "hauler.dev/platform" - ImageAnnotationRegistry = "hauler.dev/registry" - ImageAnnotationTlog = "hauler.dev/use-tlog-verify" - ImageAnnotationRewrite = "hauler.dev/rewrite" - ImageRefKey = "org.opencontainers.image.ref.name" + ImageAnnotationKey = "hauler.dev/key" + ImageAnnotationPlatform = "hauler.dev/platform" + ImageAnnotationRegistry = "hauler.dev/registry" + ImageAnnotationTlog = "hauler.dev/use-tlog-verify" + ImageAnnotationRewrite = "hauler.dev/rewrite" + ImageAnnotationExcludeExtras = "hauler.dev/exclude-extras" + ImageRefKey = "org.opencontainers.image.ref.name" // cosign keyless validation options ImageAnnotationCertIdentity = "hauler.dev/certificate-identity" diff --git a/pkg/store/store.go b/pkg/store/store.go index 9e6993c..103b7d8 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -155,7 +155,8 @@ func (l *Layout) AddArtifactCollection(ctx context.Context, collection artifacts // and saves it to the store along with any associated signatures, attestations, and SBOMs // discovered via cosign's tag convention (.sig, .att, .sbom). // When platform is non-empty and the ref is a multi-arch index, only that platform is fetched. -func (l *Layout) AddImage(ctx context.Context, ref string, platform string, opts ...remote.Option) error { +// When excludeExtras is true, cosign signatures, attestations, SBOMs, and OCI referrers are skipped. +func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, opts ...remote.Option) error { allOpts := append([]remote.Option{ remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithContext(ctx), @@ -205,11 +206,14 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, opts } } - savedDigests, err := l.saveRelatedArtifacts(ctx, parsedRef, imageDigest, allOpts...) - if err != nil { - return err + if !excludeExtras { + savedDigests, err := l.saveRelatedArtifacts(ctx, parsedRef, imageDigest, allOpts...) + if err != nil { + return err + } + return l.saveReferrers(ctx, parsedRef, imageDigest, savedDigests, allOpts...) } - return l.saveReferrers(ctx, parsedRef, imageDigest, savedDigests, allOpts...) + return nil } // writeImageBlobs writes all blobs for a single image (layers, config, manifest) to the store's diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 8ae8cbd..eb0ec32 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -464,7 +464,7 @@ func TestCopyDescriptorGraph_Index(t *testing.T) { if err != nil { t.Fatal(err) } - if err := src.AddImage(ctx, idxTag.Name(), "", remoteOpts...); err != nil { + if err := src.AddImage(ctx, idxTag.Name(), "", false, remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } if err := src.OCI.SaveIndex(); err != nil { @@ -767,7 +767,7 @@ func TestAddImage_OCI11Referrers(t *testing.T) { if err != nil { t.Fatalf("new layout: %v", err) } - if err := s.AddImage(context.Background(), baseTag.Name(), "", remoteOpts...); err != nil { + if err := s.AddImage(context.Background(), baseTag.Name(), "", false, remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) }