diff --git a/.gitignore b/.gitignore index bf8ee18..d312a41 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ vulncheck.out trivy.out CLAUDE.md **/CLAUDE.* -.claude** \ No newline at end of file +.claude** +__debug_bin* \ No newline at end of file diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 3482999..6630661 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -2,12 +2,15 @@ package cli import ( "fmt" + "os" + "strconv" "github.com/spf13/cobra" "helm.sh/helm/v4/pkg/action" "hauler.dev/go/hauler/v2/cmd/hauler/cli/store" "hauler.dev/go/hauler/v2/internal/flags" + "hauler.dev/go/hauler/v2/pkg/consts" "hauler.dev/go/hauler/v2/pkg/log" ) @@ -71,6 +74,17 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman Short: "Sync content to the content store", Args: cobra.ExactArgs(0), PreRunE: func(cmd *cobra.Command, args []string) error { + // Check for ca-file & insecure-skip-tls-verify env variables + if o.CaFile == "" { + o.CaFile = os.Getenv(consts.CaFile) + } + if o.InsecureSkipTLSVerify == nil { + if v := os.Getenv(consts.InsecureSkipTLSVerify); v != "" { + b, _ := strconv.ParseBool(v) + o.InsecureSkipTLSVerify = &b + } + } + // --dry-run requires --products if o.DryRun && len(o.Products) == 0 { return fmt.Errorf("--dry-run requires --products") @@ -110,6 +124,12 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman } rso.BlobConcurrency = bc + // resolve *bool: nil unless the user explicitly passed the flag + if cmd.Flags().Changed("insecure-skip-tls-verify") { + v, _ := cmd.Flags().GetBool("insecure-skip-tls-verify") + o.InsecureSkipTLSVerify = &v + } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -388,6 +408,19 @@ func addStoreAddImage(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co # add image from local Docker daemon hauler store add image my-local-app:latest --local`, Args: cobra.ExactArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + // Check for ca-file & insecure-skip-tls-verify env variables + if o.CaFile == "" { + o.CaFile = os.Getenv(consts.CaFile) + } + if o.InsecureSkipTLSVerify == nil { + if v := os.Getenv(consts.InsecureSkipTLSVerify); v != "" { + b, _ := strconv.ParseBool(v) + o.InsecureSkipTLSVerify = &b + } + } + return nil + }, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index e18cda8..b8a0794 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -57,7 +57,9 @@ func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, refe }() cfg := v1.File{ - Path: reference, + Path: reference, + CaFile: o.CaFile, + InsecureSkipTLSVerify: &o.InsecureSkipTLSVerify, } if len(o.Name) > 0 { cfg.Name = o.Name @@ -80,7 +82,9 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1.File, ro *flags.CliRo } copts := getter.ClientOptions{ - NameOverride: fi.Name, + NameOverride: fi.Name, + InsecureSkipTLSVerify: derefInsecure(fi.InsecureSkipTLSVerify), + CAFile: fi.CaFile, } f := file.NewFile(fi.Path, file.WithClient(getter.NewClient(copts)), file.WithContext(ctx)) @@ -198,6 +202,8 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re Rewrite: o.Rewrite, ExcludeExtras: o.ExcludeExtras, Local: o.Local, + CaFile: o.CaFile, + InsecureSkipTLSVerify: o.InsecureSkipTLSVerify, } if o.Local { @@ -242,7 +248,7 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re func addImageVerifyConfig(o *flags.AddImageOpts) cosign.Config { switch { case o.Key != "": - return cosign.Config{Key: o.Key, Tlog: o.Tlog} + return cosign.Config{Key: o.Key, Tlog: o.Tlog, InsecureSkipTLSVerify: derefInsecure(o.InsecureSkipTLSVerify), CaFile: o.CaFile} case o.CertIdentityRegexp != "" || o.CertIdentity != "": return cosign.Config{ CertIdentity: o.CertIdentity, @@ -250,6 +256,8 @@ func addImageVerifyConfig(o *flags.AddImageOpts) cosign.Config { CertOidcIssuer: o.CertOidcIssuer, CertOidcIssuerRegexp: o.CertOidcIssuerRegexp, CertGithubWorkflowRepository: o.CertGithubWorkflowRepository, + InsecureSkipTLSVerify: derefInsecure(o.InsecureSkipTLSVerify), + CaFile: o.CaFile, } default: return cosign.Config{} @@ -320,6 +328,7 @@ func verifyAddImage(ctx context.Context, o *flags.AddImageOpts, ref string, rso } func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, chartName string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { + l := log.FromContext(ctx) // Nothing in the chart path forces an fsync: every descriptor it adds -- @@ -484,6 +493,9 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin } } + insecureSkipTLSVerify := derefInsecure(i.InsecureSkipTLSVerify) + caFile := i.CaFile + // fetch image along with any associated signatures and attestations. // A fresh store.ImageStats is built inside the closure on every attempt, // not once outside it, so a failed attempt's partial layer/byte counts @@ -495,7 +507,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin err = retry.Operation(ctx, rso, ro, func() error { attemptStats := &store.ImageStats{} var addErr error - imageDigest, addErr = s.AddImage(store.WithImageStats(ctx, attemptStats), r.Name(), platform, excludeExtras, pinnedDigest) + imageDigest, addErr = s.AddImage(store.WithImageStats(ctx, attemptStats), r.Name(), platform, excludeExtras, pinnedDigest, insecureSkipTLSVerify, caFile) if addErr == nil { stats = attemptStats } @@ -564,6 +576,8 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin "certificate-oidc-issuer": i.CertOidcIssuer, "certificate-oidc-issuer-regexp": i.CertOidcIssuerRegexp, "certificate-github-workflow-repository": i.CertGithubWorkflowRepository, + "ca-file": i.CaFile, + "insecure-skip-tls-verify": i.InsecureSkipTLSVerify, "rewrite": rewrite, "exclude-extras": excludeExtras, } @@ -807,6 +821,22 @@ func resolveChartJobs(o *flags.SyncOpts, annotations map[string]string, manifest return nil, err } + // caFile precedence: cli > per-chart > annotation. + caFile := o.CaFile + if caFile == "" { + if ch.CaFile != "" { + caFile = ch.CaFile + } else if annotations[consts.ImageAnnotationCaFile] == "true" { + caFile = annotations[consts.ImageAnnotationCaFile] + } + } + + insecureSkipTLSVerify := false + if o.CaFile == "" { + insecureSkipTLSVerify = resolveInsecure(ch.InsecureSkipTLSVerify, annotations, o.InsecureSkipTLSVerify) + } else { + } + jobs = append(jobs, chartJob{ cfg: ch, opts: flags.AddChartOpts{ @@ -820,8 +850,8 @@ func resolveChartJobs(o *flags.SyncOpts, annotations map[string]string, manifest PassCredentialsAll: ch.PassCredentialsAll, CertFile: ch.CertFile, KeyFile: ch.KeyFile, - CaFile: ch.CaFile, - InsecureSkipTLSVerify: ch.InsecureSkipTLSVerify, + CaFile: caFile, + InsecureSkipTLSVerify: insecureSkipTLSVerify, PlainHTTP: ch.PlainHTTP, }, AddImages: ch.AddImages, @@ -1350,8 +1380,17 @@ func fetchChart(ctx context.Context, s *store.Layout, j chartJob, tempRoot strin return nil, nil, fmt.Errorf("unable to apply registry to image [%s]: %w", image, err) } + // Chart-discovered images inherit the chart's own TLS settings -- + // there is no separate per-discovered-image TLS knob in a chart + // manifest, so the registry a chart's images live in is assumed + // to share the chart repo's trust configuration. + chartInsecure := j.opts.ChartOpts.InsecureSkipTLSVerify imageJobs = append(imageJobs, imageJob{ - img: v1.Image{Name: relocated}, + img: v1.Image{ + Name: relocated, + CaFile: j.opts.ChartOpts.CaFile, + InsecureSkipTLSVerify: &chartInsecure, + }, platform: j.opts.Platform, excludeExtras: j.opts.ExcludeExtras, }) diff --git a/cmd/hauler/cli/store/add_test.go b/cmd/hauler/cli/store/add_test.go index b7c2770..13e5309 100644 --- a/cmd/hauler/cli/store/add_test.go +++ b/cmd/hauler/cli/store/add_test.go @@ -3,9 +3,16 @@ package store import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" "io" + "math/big" "net" "net/http" "net/http/httptest" @@ -252,7 +259,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, "", rOpts...); err != nil { + if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -354,7 +361,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, "", rOpts...); err != nil { + if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -2148,6 +2155,7 @@ func TestResolveChartJobs_NoCharts(t *testing.T) { // TestResolveChartJobs_CredentialFields pins that every TLS/verification // field on v1.Chart reaches the job's ChartOpts unchanged. func TestResolveChartJobs_CredentialFields(t *testing.T) { + insecure := true ch := v1.Chart{ Name: "rancher", Verify: true, @@ -2156,7 +2164,7 @@ func TestResolveChartJobs_CredentialFields(t *testing.T) { CertFile: "/certs/client.crt", KeyFile: "/certs/client.key", CaFile: "/certs/ca.crt", - InsecureSkipTLSVerify: true, + InsecureSkipTLSVerify: &insecure, PlainHTTP: true, } @@ -2187,8 +2195,8 @@ func TestResolveChartJobs_CredentialFields(t *testing.T) { if opts.CaFile != ch.CaFile { t.Errorf("CaFile = %q, want %q", opts.CaFile, ch.CaFile) } - if opts.InsecureSkipTLSVerify != ch.InsecureSkipTLSVerify { - t.Errorf("InsecureSkipTLSVerify = %v, want %v", opts.InsecureSkipTLSVerify, ch.InsecureSkipTLSVerify) + if opts.InsecureSkipTLSVerify != derefInsecure(ch.InsecureSkipTLSVerify) { + t.Errorf("InsecureSkipTLSVerify = %v, want %v", opts.InsecureSkipTLSVerify, derefInsecure(ch.InsecureSkipTLSVerify)) } if opts.PlainHTTP != ch.PlainHTTP { t.Errorf("PlainHTTP = %v, want %v", opts.PlainHTTP, ch.PlainHTTP) @@ -2972,3 +2980,100 @@ func TestFormatAddedLine_WithStats(t *testing.T) { }) } } + +// TestStoreImage_CAFileAndInsecure exercises the insecureSkipTLSVerify / caFile +// plumbing through storeImage -> AddImage. Note: the in-memory registry runs on +// localhost, which go-containerregistry forces to http, so these cases do NOT +// perform a real TLS handshake — the actual CA trust/reject behavior is covered +// by buildTransport's handshake test in pkg/store. What's verified here is caFile +// error propagation, insecure-over-caFile precedence, and that a valid caFile +// doesn't break the pull. +func TestStoreImage_CAFileAndInsecure(t *testing.T) { + ctx := newTestContext(t) + host, rOpts := newLocalhostRegistry(t) + seedImage(t, host, "tls/repo", "v1", rOpts...) + ref := host + "/tls/repo:v1" + + const missingCA = "/nonexistent/ca.pem" + + t.Run("bad caFile without insecure returns error and stores nothing", func(t *testing.T) { + s := newTestStore(t) + insecure := false + img := v1.Image{Name: ref, CaFile: missingCA, InsecureSkipTLSVerify: &insecure} + err := storeImage(ctx, s, img, "", false, + defaultRootOpts(s.Root), defaultCliOpts(), "", "", false) + if err == nil { + t.Fatal("expected error from unreadable caFile, got nil") + } + if n := countArtifactsInStore(t, s); n != 0 { + t.Errorf("expected nothing stored on caFile error, got %d", n) + } + }) + + t.Run("non-PEM caFile without insecure returns error", func(t *testing.T) { + s := newTestStore(t) + junk := filepath.Join(t.TempDir(), "junk.pem") + if err := os.WriteFile(junk, []byte("not a certificate"), 0o600); err != nil { + t.Fatal(err) + } + insecure := false + img := v1.Image{Name: ref, CaFile: junk, InsecureSkipTLSVerify: &insecure} + err := storeImage(ctx, s, img, "", false, + defaultRootOpts(s.Root), defaultCliOpts(), "", "", false) + if err == nil { + t.Fatal("expected error from non-PEM caFile, got nil") + } + }) + + t.Run("insecure takes precedence over bad caFile", func(t *testing.T) { + s := newTestStore(t) + // insecure=true short-circuits before caFile is read; the bogus path is + // ignored and the pull still succeeds. If caFile were read first, the + // pull would error and nothing would be stored. + insecure := true + img := v1.Image{Name: ref, CaFile: missingCA, InsecureSkipTLSVerify: &insecure} + err := storeImage(ctx, s, img, "", false, + defaultRootOpts(s.Root), defaultCliOpts(), "", "", false) + if err != nil { + t.Fatalf("insecure should ignore caFile, got: %v", err) + } + assertArtifactInStore(t, s, "tls/repo:v1") + }) + + t.Run("valid caFile without insecure is accepted", func(t *testing.T) { + s := newTestStore(t) + insecure := false + img := v1.Image{Name: ref, CaFile: writeCAFile(t), InsecureSkipTLSVerify: &insecure} + err := storeImage(ctx, s, img, "", false, + defaultRootOpts(s.Root), defaultCliOpts(), "", "", false) + if err != nil { + t.Fatalf("valid caFile should be accepted, got: %v", err) + } + assertArtifactInStore(t, s, "tls/repo:v1") + }) +} + +// writeCAFile writes a valid self-signed cert PEM to a temp file and returns its +// path. Its only job is to be a parseable CA file (AppendCertsFromPEM succeeds). +func writeCAFile(t *testing.T) string { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + p := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(p, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + return p +} diff --git a/cmd/hauler/cli/store/copy_test.go b/cmd/hauler/cli/store/copy_test.go index e5d417e..e6a66c3 100644 --- a/cmd/hauler/cli/store/copy_test.go +++ b/cmd/hauler/cli/store/copy_test.go @@ -222,7 +222,7 @@ func TestCopyCmd_Registry_SigTagDerivation(t *testing.T) { // AddImage discovers and stores the .sig/.att/.sbom tags automatically. s := newTestStore(t) - if _, err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false, ""); err != nil { + if _, err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/extract_test.go b/cmd/hauler/cli/store/extract_test.go index 61a69af..cfc39e2 100644 --- a/cmd/hauler/cli/store/extract_test.go +++ b/cmd/hauler/cli/store/extract_test.go @@ -167,7 +167,7 @@ func TestExtractCmd_OciArtifactKindImage(t *testing.T) { // Pull into a fresh store — AddImage sets kind=KindAnnotationImage on all manifests. s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -249,7 +249,7 @@ func TestExtractCmd_OciImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -359,7 +359,7 @@ func TestExtractCmd_NestedImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -427,7 +427,7 @@ func TestExtractCmd_ContainerImage_Skipped(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -497,7 +497,7 @@ func TestExtractCmd_ContainerImageIndex_Skipped(t *testing.T) { } s := newTestStore(t) - if _, err := s.AddImage(ctx, ref, "", false, "", rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, "", false, "", rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/info_test.go b/cmd/hauler/cli/store/info_test.go index 4050d70..a1912f6 100644 --- a/cmd/hauler/cli/store/info_test.go +++ b/cmd/hauler/cli/store/info_test.go @@ -352,7 +352,7 @@ func TestInfoCmd_CheckHealthyStore(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/healthy", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/healthy:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/healthy:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -403,11 +403,11 @@ func TestInfoCmd_CheckCorruptBlob_HealthySiblingOmitted(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/corrupt-a", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/corrupt-a:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/corrupt-a:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage corrupt-a: %v", err) } seedImage(t, host, "test/healthy-b", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/healthy-b:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/healthy-b:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage healthy-b: %v", err) } @@ -468,7 +468,7 @@ func TestInfoCmd_CheckCorruptManifest_RegressionGuard(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/badmanifest", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/badmanifest:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/badmanifest:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -511,7 +511,7 @@ func TestInfoCmd_NoCheck_OutputUnchanged(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/plain", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -567,7 +567,7 @@ func TestInfoCmd_CheckWithTypeFilter_SkipsFilteredCorruption(t *testing.T) { host, opts := newTestRegistry(t) seedImage(t, host, "test/filterhealthy", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/filterhealthy:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/filterhealthy:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -621,7 +621,7 @@ func TestInfoCmd_CheckHealthyStore_TableFormat_NoTableRendered(t *testing.T) { var buf bytes.Buffer ctx := newCapturingContext(&buf) - if _, err := s.AddImage(ctx, host+"/test/healthytable:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/healthytable:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -668,7 +668,7 @@ func TestInfoCmd_CheckCorruptBlob_RemediationHintLogged(t *testing.T) { ctx := newCapturingContext(&buf) seedImage(t, host, "test/remediation", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/remediation:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/remediation:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -711,7 +711,7 @@ func TestInfoCmd_RemediationHint_DedupedForMultiPlatformImage(t *testing.T) { logCtx := newCapturingContext(&buf) idx := seedIndex(t, host, "test/remediation-multiarch", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/remediation-multiarch:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/remediation-multiarch:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -770,7 +770,7 @@ func TestInfoCmd_CheckMultipleBadBlobs_OneRowPerProblem(t *testing.T) { host, opts := newTestRegistry(t) img := seedImage(t, host, "test/doublebad", "v1", opts...) - if _, err := s.AddImage(ctx, host+"/test/doublebad:v1", "", true, "", opts...); err != nil { + if _, err := s.AddImage(ctx, host+"/test/doublebad:v1", "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index 13ba594..490b1fd 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -122,7 +122,7 @@ func stageRemoteChunks(ctx context.Context, fileNames []string, stageDir string) // downloadHaul fetches urlStr into destDir, using the server-provided // filename when available, and returns the local path it was saved to. func downloadHaul(ctx context.Context, urlStr, destDir string) (string, error) { - h := getter.NewHttp() + h := getter.NewHttp(false, "") parsedURL, err := url.Parse(urlStr) if err != nil { return "", err diff --git a/cmd/hauler/cli/store/save_test.go b/cmd/hauler/cli/store/save_test.go index b9d138c..8f856bf 100644 --- a/cmd/hauler/cli/store/save_test.go +++ b/cmd/hauler/cli/store/save_test.go @@ -59,7 +59,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v1", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -78,7 +78,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v2", rOpts...) s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -126,7 +126,7 @@ func TestWriteExportsManifest_DigestOnlyImageHasRepoTag(t *testing.T) { // Add the image BY DIGEST s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false, "", false, ""); err != nil { t.Fatalf("AddImage by digest: %v", err) } @@ -174,7 +174,7 @@ func TestSaveCmd(t *testing.T) { seedImage(t, host, "test/save", "v1") s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/save:v1", "", false, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/save:v1", "", false, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -207,7 +207,7 @@ func TestSaveCmd_ContainerdCompatibility(t *testing.T) { seedImage(t, host, "test/containerd-compat", "v1") s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -306,7 +306,7 @@ func TestSaveCmd_ChunkSize(t *testing.T) { seedImage(t, host, "test/chunksave", "v1") s := newTestStore(t) - if _, err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 9ccdb3b..58e2801 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -148,7 +148,9 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags l.Infof("fetching product manifest from [%s]", manifestLoc) img := v1.Image{ - Name: manifestLoc, + Name: manifestLoc, + InsecureSkipTLSVerify: o.InsecureSkipTLSVerify, + CaFile: o.CaFile, } err := storeImage(ctx, s, img, o.Platform, o.ExcludeExtras, rso, ro, "", "", false) if err != nil { @@ -180,7 +182,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") { l.Debugf("detected remote manifest... starting download... [%s]", haulPath) - h := getter.NewHttp() + h := getter.NewHttp(derefInsecure(o.InsecureSkipTLSVerify), o.CaFile) parsedURL, err := url.Parse(haulPath) if err != nil { return err @@ -232,7 +234,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags if strings.HasPrefix(haulPath, "http://") || strings.HasPrefix(haulPath, "https://") { l.Debugf("detected remote image.txt... starting download... [%s]", haulPath) - h := getter.NewHttp() + h := getter.NewHttp(derefInsecure(o.InsecureSkipTLSVerify), o.CaFile) parsedURL, err := url.Parse(haulPath) if err != nil { return err @@ -278,6 +280,30 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags return nil } +// resolveInsecure applies precedence: cli > per-item > annotation. +// A non-nil per-item pointer wins outright — including an explicit false — so an +// individual file/image/chart can opt out of an insecure annotation or the global +// --insecure-skip-tls-verify flag. nil means "not set on the item", which falls +// through to the annotation, then the global flag. +func resolveInsecure(item *bool, ann map[string]string, global *bool) bool { + if global != nil { + return *global + } + if item != nil { + return *item + } + if ann != nil && ann[consts.ImageAnnotationInsecureSkipTLSVerify] == "true" { + return true + } + return false +} + +// derefInsecure is a nil-safe read of a *bool for logging/plumbing where a plain +// bool is needed. nil reads as false. +func derefInsecure(p *bool) bool { + return p != nil && *p +} + func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) @@ -314,7 +340,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor if err := yaml.Unmarshal(doc, &cfg); err != nil { return err } - jobs := resolveFileJobs(cfg.Spec.Files) + jobs := resolveFileJobs(o, cfg.GetAnnotations(), cfg.Spec.Files) if err := runFileJobs(ctx, s, jobs, o.Concurrency, rso, ro, newSyncProgress(o, ro)); err != nil { return err } @@ -401,7 +427,11 @@ func processImageTxt(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *sto } l.Debugf("adding image [%s] to the store [%s]", line, o.StoreDir) jobs = append(jobs, imageJob{ - img: v1.Image{Name: line}, + img: v1.Image{ + Name: line, + CaFile: o.CaFile, + InsecureSkipTLSVerify: o.InsecureSkipTLSVerify, + }, platform: o.Platform, excludeExtras: o.ExcludeExtras, }) @@ -505,6 +535,19 @@ func resolveImageJobs(o *flags.SyncOpts, a map[string]string, images []v1.Image) i.Name = newRef.Name() } + // caFile precedence: cli >per-image > annotation. + if o.CaFile == "" && i.CaFile == "" && a[consts.ImageAnnotationCaFile] != "" { + i.CaFile = a[consts.ImageAnnotationCaFile] + } else if o.CaFile != "" { + i.CaFile = o.CaFile + } + + insecureSkipTLSVerify := false + if o.CaFile == "" { + insecureSkipTLSVerify = resolveInsecure(i.InsecureSkipTLSVerify, a, o.InsecureSkipTLSVerify) + } + i.InsecureSkipTLSVerify = &insecureSkipTLSVerify + if i.Local { needsPubKeyVerification := a[consts.ImageAnnotationKey] != "" || o.Key != "" || i.Key != "" needsKeylessVerification := a[consts.ImageAnnotationCertIdentityRegexp] != "" || a[consts.ImageAnnotationCertIdentity] != "" || @@ -656,7 +699,12 @@ func resolveImageJobs(o *flags.SyncOpts, a map[string]string, images []v1.Image) func (j imageJob) verifyConfig() cosign.Config { switch { case j.needsPubKey: - return cosign.Config{Key: j.key, Tlog: j.tlog} + return cosign.Config{ + Key: j.key, + Tlog: j.tlog, + InsecureSkipTLSVerify: derefInsecure(j.img.InsecureSkipTLSVerify), + CaFile: j.img.CaFile, + } case j.needsKeyless: return cosign.Config{ CertIdentity: j.certIdentity, @@ -664,6 +712,8 @@ func (j imageJob) verifyConfig() cosign.Config { CertOidcIssuer: j.certOidcIssuer, CertOidcIssuerRegexp: j.certOidcIssuerRegexp, CertGithubWorkflowRepository: j.certGithubWorkflowRepository, + InsecureSkipTLSVerify: derefInsecure(j.img.InsecureSkipTLSVerify), + CaFile: j.img.CaFile, } default: return cosign.Config{} @@ -1009,11 +1059,26 @@ type fileJob struct { file v1.File } -// resolveFileJobs converts every v1.File in files into a fileJob. It is -// pure. -func resolveFileJobs(files []v1.File) []fileJob { +// resolveFileJobs converts every v1.File in files into a fileJob, applying +// the caFile/insecure precedence rules (per-file > annotation > global) -- +// see resolveInsecure. It is pure. +func resolveFileJobs(o *flags.SyncOpts, a map[string]string, files []v1.File) []fileJob { jobs := make([]fileJob, 0, len(files)) for _, f := range files { + + // caFile precedence: cli >per-image > annotation. + if o.CaFile == "" && f.CaFile == "" && a[consts.ImageAnnotationCaFile] != "" { + f.CaFile = a[consts.ImageAnnotationCaFile] + } else if o.CaFile != "" { + f.CaFile = o.CaFile + } + + insecure := false + if o.CaFile == "" { + insecure = resolveInsecure(f.InsecureSkipTLSVerify, a, o.InsecureSkipTLSVerify) + } + f.InsecureSkipTLSVerify = &insecure + jobs = append(jobs, fileJob{file: f}) } return jobs diff --git a/cmd/hauler/cli/store/sync_test.go b/cmd/hauler/cli/store/sync_test.go index 9f2d98b..5d253f3 100644 --- a/cmd/hauler/cli/store/sync_test.go +++ b/cmd/hauler/cli/store/sync_test.go @@ -1314,7 +1314,7 @@ func TestResolveFileJobs_OneJobPerFile(t *testing.T) { {Path: "https://example.com/b.sh", Name: "renamed-b.sh"}, } - jobs := resolveFileJobs(files) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, files) if len(jobs) != 2 { t.Fatalf("resolveFileJobs: got %d jobs, want 2", len(jobs)) } @@ -1327,7 +1327,7 @@ func TestResolveFileJobs_OneJobPerFile(t *testing.T) { } func TestResolveFileJobs_EmptyInput(t *testing.T) { - jobs := resolveFileJobs(nil) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, nil) if len(jobs) != 0 { t.Errorf("resolveFileJobs(nil): got %d jobs, want 0", len(jobs)) } @@ -1344,7 +1344,7 @@ func TestRunFileJobs_AllSucceed(t *testing.T) { url1 := seedFileInHTTPServer(t, "one.sh", "#!/bin/sh\necho one") url2 := seedFileInHTTPServer(t, "two.sh", "#!/bin/sh\necho two") - jobs := resolveFileJobs([]v1.File{{Path: url1}, {Path: url2}}) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, []v1.File{{Path: url1}, {Path: url2}}) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() @@ -1374,7 +1374,7 @@ func TestRunFileJobs_ConcurrencyOneVsFour_ProduceEquivalentStores(t *testing.T) run := func(concurrency int) *storeSnapshot { s := newTestStore(t) - jobs := resolveFileJobs(files) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, files) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() if err := runFileJobs(ctx, s, jobs, concurrency, rso, ro, nil); err != nil { @@ -1457,7 +1457,7 @@ func TestRunFileJobs_DedupesDuplicateSourceAcrossEntries(t *testing.T) { {Path: url, Name: "rke2-install.sh"}, } - jobs := resolveFileJobs(files) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, files) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() @@ -1515,7 +1515,7 @@ func TestRunFileJobs_ErrorPropagation(t *testing.T) { {Path: goodURL}, } - jobs := resolveFileJobs(files) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, files) rso := defaultRootOpts(s.Root) rso.Retries = 1 // avoid RetriesInterval sleeps in this table ro := defaultCliOpts() @@ -1569,7 +1569,7 @@ func TestRunFileJobs_RetryEventuallySucceeds(t *testing.T) { srv := httptest.NewServer(mux) t.Cleanup(srv.Close) - jobs := resolveFileJobs([]v1.File{{Path: srv.URL + "/eventual.sh"}}) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, []v1.File{{Path: srv.URL + "/eventual.sh"}}) rso := defaultRootOpts(s.Root) rso.Retries = 2 ro := defaultCliOpts() @@ -1619,7 +1619,7 @@ func TestRunFileJobs_CancellationAbortsPromptly(t *testing.T) { }() s := newTestStore(t) - jobs := resolveFileJobs([]v1.File{{Path: srv.URL + "/slow.sh"}}) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, []v1.File{{Path: srv.URL + "/slow.sh"}}) rso := defaultRootOpts(s.Root) ro := defaultCliOpts() @@ -1657,7 +1657,7 @@ func TestRunFileJobs_WithProgress_RendersEscapeCodesAndCompletionLines(t *testin ro := defaultCliOpts() progress := log.NewRenderer(&buf) - jobs := resolveFileJobs(files) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, files) if err := runFileJobs(ctx, s, jobs, 2, rso, ro, progress); err != nil { t.Fatalf("runFileJobs: %v", err) @@ -1682,7 +1682,7 @@ func TestRunFileJobs_NoProgress_CompletionLineRefAppearsOnce(t *testing.T) { rso := defaultRootOpts(s.Root) ro := defaultCliOpts() - jobs := resolveFileJobs([]v1.File{{Path: url}}) + jobs := resolveFileJobs(&flags.SyncOpts{}, nil, []v1.File{{Path: url}}) if err := runFileJobs(ctx, s, jobs, 1, rso, ro, nil); err != nil { t.Fatalf("runFileJobs: %v", err) } diff --git a/cmd/hauler/cli/store_info_check_test.go b/cmd/hauler/cli/store_info_check_test.go index d87ca50..56e7049 100644 --- a/cmd/hauler/cli/store_info_check_test.go +++ b/cmd/hauler/cli/store_info_check_test.go @@ -158,7 +158,7 @@ func TestStoreInfoCheck_JSON_CorruptArtifactInPayload(t *testing.T) { ctx := context.Background() seedInfoCheckImage(t, host, "test/corrupt", "v1") - if _, err := s.AddImage(ctx, host+"/test/corrupt:v1", "", true, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/corrupt:v1", "", true, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } @@ -213,7 +213,7 @@ func TestStoreInfoCheck_JSON_NoCheckStillWorks(t *testing.T) { ctx := context.Background() seedInfoCheckImage(t, host, "test/plain", "v1") - if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true, ""); err != nil { + if _, err := s.AddImage(ctx, host+"/test/plain:v1", "", true, "", false, ""); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/internal/flags/add.go b/internal/flags/add.go index 75270b3..a219f20 100644 --- a/internal/flags/add.go +++ b/internal/flags/add.go @@ -21,6 +21,8 @@ type AddImageOpts struct { Rewrite string ExcludeExtras bool Local bool + CaFile string + InsecureSkipTLSVerify *bool } func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { @@ -36,16 +38,22 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string") f.BoolVar(&o.ExcludeExtras, "exclude-extras", false, "(Optional) Exclude cosign signatures, attestations, SBOMs, and OCI referrers when pulling the image") f.BoolVar(&o.Local, "local", false, "(Optional) Add image from the local Docker daemon instead of a remote registry") + f.StringVar(&o.CaFile, "ca-file", "", "(Optional) Location of CA Bundle to enable certification verification") + f.Bool("insecure-skip-tls-verify", false, "(Optional) Skip TLS certificate verification") } type AddFileOpts struct { *StoreRootOpts - Name string + Name string + CaFile string + InsecureSkipTLSVerify bool } func (o *AddFileOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() f.StringVarP(&o.Name, "name", "n", "", "(Optional) Rewrite the name of the file") + f.StringVar(&o.CaFile, "ca-file", "", "(Optional) Location of CA Bundle to enable certification verification for remote files") + f.BoolVar(&o.InsecureSkipTLSVerify, "insecure-skip-tls-verify", false, "(Optional) Skip TLS certificate verification for remote files") } type AddChartOpts struct { @@ -80,7 +88,7 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) { f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string") cmd.MarkFlagsRequiredTogether("username", "password") - cmd.MarkFlagsRequiredTogether("cert-file", "key-file", "ca-file") + cmd.MarkFlagsRequiredTogether("cert-file", "key-file") cmd.Flags().BoolVar(&o.AddDependencies, "add-dependencies", false, "(Optional) Fetch dependent helm charts") f.BoolVar(&o.AddImages, "add-images", false, "(Optional) Fetch images referenced in helm charts") diff --git a/internal/flags/sync.go b/internal/flags/sync.go index 1e483c4..6f067fe 100644 --- a/internal/flags/sync.go +++ b/internal/flags/sync.go @@ -25,6 +25,8 @@ type SyncOpts struct { DryRun bool Concurrency int NoProgress bool + CaFile string + InsecureSkipTLSVerify *bool } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { @@ -47,4 +49,6 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.BoolVar(&o.DryRun, "dry-run", false, "(Optional) Output product manifest content to stdout instead of processing it (requires --products)") f.IntVarP(&o.Concurrency, "concurrency", "j", consts.DefaultConcurrency, "(Optional) Maximum number of artifacts to fetch and store concurrently (1 = serial; also via HAULER_CONCURRENCY, explicit flag wins)") f.BoolVar(&o.NoProgress, "no-progress", false, "(Optional) Disable the live progress display") + f.StringVar(&o.CaFile, "ca-file", "", "(Optional) Location of CA Bundle to enable certification verification") + f.Bool("insecure-skip-tls-verify", false, "(Optional) Skip TLS certificate verification") } diff --git a/pkg/apis/hauler.cattle.io/v1/chart.go b/pkg/apis/hauler.cattle.io/v1/chart.go index cbd73a0..1098dcb 100644 --- a/pkg/apis/hauler.cattle.io/v1/chart.go +++ b/pkg/apis/hauler.cattle.io/v1/chart.go @@ -41,6 +41,6 @@ type Chart struct { CertFile string `json:"certFile,omitempty"` KeyFile string `json:"keyFile,omitempty"` CaFile string `json:"caFile,omitempty"` - InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` + InsecureSkipTLSVerify *bool `json:"insecureSkipTLSVerify,omitempty"` PlainHTTP bool `json:"plainHTTP,omitempty"` } diff --git a/pkg/apis/hauler.cattle.io/v1/file.go b/pkg/apis/hauler.cattle.io/v1/file.go index 6abc47f..b1c9cfe 100644 --- a/pkg/apis/hauler.cattle.io/v1/file.go +++ b/pkg/apis/hauler.cattle.io/v1/file.go @@ -22,4 +22,9 @@ type File struct { // Name is an optional field specifying the name of the file when specified, // it will override any dynamic name discovery from Path Name string `json:"name,omitempty"` + + // TLS options for verifying the file contents for remote files. + // If not specified, the default system CA bundle will be used. + CaFile string `json:"ca-file"` + InsecureSkipTLSVerify *bool `json:"insecure-skip-tls-verify,omitempty"` } diff --git a/pkg/apis/hauler.cattle.io/v1/image.go b/pkg/apis/hauler.cattle.io/v1/image.go index b257e79..45e765d 100644 --- a/pkg/apis/hauler.cattle.io/v1/image.go +++ b/pkg/apis/hauler.cattle.io/v1/image.go @@ -40,4 +40,8 @@ type Image struct { Rewrite string `json:"rewrite"` ExcludeExtras bool `json:"exclude-extras"` Local bool `json:"local"` + + // TLS options for verifying the image signature. If not specified, the default system CA bundle will be used. + CaFile string `json:"ca-file"` + InsecureSkipTLSVerify *bool `json:"insecure-skip-tls-verify,omitempty"` } diff --git a/pkg/artifacts/file/file_test.go b/pkg/artifacts/file/file_test.go index b40e4f6..cce2c28 100644 --- a/pkg/artifacts/file/file_test.go +++ b/pkg/artifacts/file/file_test.go @@ -127,7 +127,7 @@ func setup() func() { mf := &mockFile{File: getter.NewFile(), fs: tfs} - mockHttp := getter.NewHttp() + mockHttp := getter.NewHttp(false, "") mhttp := afero.NewHttpFs(tfs) fileserver := http.FileServer(mhttp.Dir(".")) http.Handle("/", fileserver) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index b5f20e6..93fb6e1 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -76,6 +76,10 @@ const ( ImageAnnotationCertOidcIssuerRegexp = "hauler.dev/certificate-oidc-issuer-regexp" ImageAnnotationCertGithubWorkflowRepository = "hauler.dev/certificate-github-workflow-repository" + // TLS options for verifying the image signature. If not specified, the default system CA bundle will be used. + ImageAnnotationCaFile = "hauler.dev/ca-file" + ImageAnnotationInsecureSkipTLSVerify = "hauler.dev/insecure-skip-tls-verify" + // content kinds ImagesContentKind = "Images" ChartsContentKind = "Charts" @@ -96,6 +100,9 @@ const ( HaulerLogLevel = "HAULER_LOG_LEVEL" HaulerAuditLevel = "HAULER_AUDIT_LEVEL" + CaFile = "CA_FILE" + InsecureSkipTLSVerify = "INSECURE_SKIP_TLS_VERIFY" + // container files and directories ImageManifestFile = "manifest.json" ImageConfigFile = "config.json" diff --git a/pkg/content/transport.go b/pkg/content/transport.go new file mode 100644 index 0000000..040353e --- /dev/null +++ b/pkg/content/transport.go @@ -0,0 +1,79 @@ +package content + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "os" + "strings" + "sync" + + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/rs/zerolog" +) + +type transport struct { + base http.RoundTripper + warnOnce sync.Once +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + if strings.EqualFold(req.URL.Scheme, "http") { + t.warnOnce.Do(func() { + zerolog.Ctx(req.Context()).Warn().Msgf("pulling content over plain HTTP [%s]", req.URL.Host) + }) + } + + return t.base.RoundTrip(req) +} + +// BuildTransport returns a RoundTripper configured with the requested TLS +// settings. It also warns if the image pull makes a request over plain HTTP. +// +// insecureSkipTLSVerify takes precedence over caFile. When enabled, caFile is +// ignored. +func BuildTransport(insecureSkipTLSVerify bool, caFile string) (http.RoundTripper, error) { + base := remote.DefaultTransport + + if insecureSkipTLSVerify || caFile != "" { + defaultTransport, ok := remote.DefaultTransport.(*http.Transport) + if !ok { + return nil, fmt.Errorf("unexpected default transport type %T", remote.DefaultTransport) + } + + tr := defaultTransport.Clone() + + if tr.TLSClientConfig == nil { + tr.TLSClientConfig = &tls.Config{} + } else { + tr.TLSClientConfig = tr.TLSClientConfig.Clone() + } + + if insecureSkipTLSVerify { + tr.TLSClientConfig.InsecureSkipVerify = true //nolint:gosec + } else { + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + + pem, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("reading CA file %q: %w", caFile, err) + } + + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("no valid certificates found in CA file %q", caFile) + } + + tr.TLSClientConfig.RootCAs = pool + } + + base = tr + } + + return &transport{ + base: base, + }, nil +} diff --git a/pkg/cosign/verifier.go b/pkg/cosign/verifier.go index d3c7e3c..5daddf2 100644 --- a/pkg/cosign/verifier.go +++ b/pkg/cosign/verifier.go @@ -34,6 +34,12 @@ type Config struct { CertOidcIssuer string CertOidcIssuerRegexp string CertGithubWorkflowRepository string + + // TLS options for reaching the registry (signatures/attestations/SBOMs) + // and the transparency log. InsecureSkipTLSVerify takes precedence over + // CaFile -- see NewVerifier. + InsecureSkipTLSVerify bool + CaFile string } // Empty reports whether cfg requests no verification at all. @@ -119,7 +125,14 @@ func NewVerifier(ctx context.Context, cfg Config, rso *flags.StoreRootOpts, ro * } } + // insecureSkipTLSVerify takes precedence: when set, caFile is ignored -- + // mirrors content.BuildTransport's precedence for the plain registry pull. regOpts := options.RegistryOptions{} + if cfg.InsecureSkipTLSVerify { + regOpts.AllowInsecure = true + } else { + regOpts.RegistryCACert = cfg.CaFile + } ociremoteOpts, err := regOpts.ClientOpts(ctx) if err != nil { return nil, fmt.Errorf("constructing registry client options: %w", err) diff --git a/pkg/getter/getter.go b/pkg/getter/getter.go index fcad862..d8789d5 100644 --- a/pkg/getter/getter.go +++ b/pkg/getter/getter.go @@ -23,7 +23,9 @@ type Client struct { // ClientOptions provides options for the client type ClientOptions struct { - NameOverride string + NameOverride string + InsecureSkipTLSVerify bool + CAFile string } var ( @@ -44,7 +46,7 @@ func NewClient(opts ClientOptions) *Client { defaults := map[string]Getter{ "file": NewFile(), "directory": NewDirectory(), - "http": NewHttp(), + "http": NewHttp(opts.InsecureSkipTLSVerify, opts.CAFile), } c := &Client{ diff --git a/pkg/getter/https.go b/pkg/getter/https.go index 943426a..592cd0e 100644 --- a/pkg/getter/https.go +++ b/pkg/getter/https.go @@ -4,44 +4,32 @@ import ( "context" "fmt" "io" - "mime" "net/http" "net/url" "path/filepath" - "strings" "hauler.dev/go/hauler/v2/pkg/artifacts" "hauler.dev/go/hauler/v2/pkg/consts" + "hauler.dev/go/hauler/v2/pkg/content" ) -type Http struct{} +type Http struct { + client *http.Client +} -func NewHttp() *Http { - return &Http{} +func NewHttp(insecureSkipTLSVerify bool, caFile string) *Http { + tr, err := content.BuildTransport(insecureSkipTLSVerify, caFile) + if err != nil { + return &Http{client: http.DefaultClient} + } + return &Http{client: &http.Client{Transport: tr}} } func (h Http) Name(u *url.URL) string { - resp, err := http.Head(u.String()) - if err != nil { - return "" - } - defer resp.Body.Close() - unescaped, err := url.PathUnescape(u.String()) if err != nil { return "" } - - contentType := resp.Header.Get("Content-Type") - for _, v := range strings.Split(contentType, ",") { - t, _, err := mime.ParseMediaType(v) - if err != nil { - break - } - // TODO: Identify known mimetypes for hints at a filename - _ = t - } - return filepath.Base(unescaped) } @@ -50,7 +38,7 @@ func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { if err != nil { return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := h.client.Do(req) if err != nil { return nil, err } diff --git a/pkg/getter/https_test.go b/pkg/getter/https_test.go index f0b3c3d..07bbcb1 100644 --- a/pkg/getter/https_test.go +++ b/pkg/getter/https_test.go @@ -43,7 +43,7 @@ func TestHttp_Open_HonorsContextCancellation(t *testing.T) { cancel() }() - h := getter.NewHttp() + h := getter.NewHttp(false, "") done := make(chan error, 1) go func() { diff --git a/pkg/store/check_test.go b/pkg/store/check_test.go index 7734472..24e0cff 100644 --- a/pkg/store/check_test.go +++ b/pkg/store/check_test.go @@ -67,7 +67,7 @@ func pushAndAddExistingImage(t *testing.T, s *store.Layout, host, repo, tag stri if err := remote.Write(ref, img, opts...); err != nil { t.Fatalf("remote.Write: %v", err) } - if _, err := s.AddImage(context.Background(), ref.Name(), "", true, "", opts...); err != nil { + if _, err := s.AddImage(context.Background(), ref.Name(), "", true, "", false, "", opts...); err != nil { t.Fatalf("AddImage: %v", err) } return findManifestDescForRef(t, s, repo+":"+tag) diff --git a/pkg/store/stats_test.go b/pkg/store/stats_test.go index f22f005..de2a5c3 100644 --- a/pkg/store/stats_test.go +++ b/pkg/store/stats_test.go @@ -62,7 +62,7 @@ func TestAddImage_ImageStatsAccumulation(t *testing.T) { stats := &store.ImageStats{} ctx := store.WithImageStats(context.Background(), stats) - if _, err := s.AddImage(ctx, tag.Name(), "", false, "", remoteOpts...); err != nil { + if _, err := s.AddImage(ctx, tag.Name(), "", false, "", false, "", remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -103,7 +103,7 @@ func TestAddImage_NoImageStatsInContext(t *testing.T) { t.Fatalf("new layout: %v", err) } - if _, err := s.AddImage(context.Background(), tag.Name(), "", false, "", remoteOpts...); err != nil { + if _, err := s.AddImage(context.Background(), tag.Name(), "", false, "", false, "", remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } } diff --git a/pkg/store/store.go b/pkg/store/store.go index f3231e9..7804186 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -238,10 +238,21 @@ func (l *Layout) AddArtifactCollection(ctx context.Context, collection artifacts // signature pass the digest they verified, so the bytes stored are provably // the bytes checked even if the tag moves mid-run. An empty pinnedDigest // resolves ref normally. -func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, pinnedDigest string, opts ...remote.Option) (string, error) { +// +// insecureSkipTLSVerify and caFile configure the transport used for every +// registry round trip this call makes (the image itself plus any related +// signatures/attestations/SBOMs/referrers); insecureSkipTLSVerify takes +// precedence over caFile -- see content.BuildTransport. +func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, pinnedDigest string, insecureSkipTLSVerify bool, caFile string, opts ...remote.Option) (string, error) { + tr, err := content.BuildTransport(insecureSkipTLSVerify, caFile) + if err != nil { + return "", err + } + allOpts := append([]remote.Option{ remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithContext(ctx), + remote.WithTransport(tr), }, opts...) parsedRef, err := gname.ParseReference(ref) diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 2ea23fc..75cd17c 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -464,7 +464,7 @@ func TestCopyDescriptorGraph_Index(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := src.AddImage(ctx, idxTag.Name(), "", false, "", remoteOpts...); err != nil { + if _, err := src.AddImage(ctx, idxTag.Name(), "", false, "", false, "", remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } if err := src.OCI.SaveIndex(); err != nil { @@ -767,7 +767,7 @@ func TestAddImage_OCI11Referrers(t *testing.T) { if err != nil { t.Fatalf("new layout: %v", err) } - if _, err := s.AddImage(context.Background(), baseTag.Name(), "", false, "", remoteOpts...); err != nil { + if _, err := s.AddImage(context.Background(), baseTag.Name(), "", false, "", false, "", remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -853,7 +853,7 @@ func TestAddImagePinnedDigestIgnoresMovedTag(t *testing.T) { s := newTestStore(t) got, err := s.AddImage(context.Background(), host+"/test/pinned:v1", "", true, - originalDigest.String(), remoteOpts...) + originalDigest.String(), false, "", remoteOpts...) if err != nil { t.Fatalf("AddImage: %v", err) } @@ -873,7 +873,7 @@ func TestAddImageEmptyPinResolvesTag(t *testing.T) { } s := newTestStore(t) - got, err := s.AddImage(context.Background(), host+"/test/unpinned:v1", "", true, "", remoteOpts...) + got, err := s.AddImage(context.Background(), host+"/test/unpinned:v1", "", true, "", false, "", remoteOpts...) if err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/pkg/store/store_tls_test.go b/pkg/store/store_tls_test.go new file mode 100644 index 0000000..f17b3a6 --- /dev/null +++ b/pkg/store/store_tls_test.go @@ -0,0 +1,113 @@ +package store + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "hauler.dev/go/hauler/v2/pkg/content" +) + +func transport(t *testing.T, rt http.RoundTripper) *http.Transport { + t.Helper() + tr, ok := rt.(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", rt) + } + return tr +} + +// insecureSkipTLSVerify must win over caFile: a bogus caFile is never read. +func TestBuildTransport_InsecurePrecedence(t *testing.T) { + // insecure must short-circuit before caFile is read: a bogus path is ignored. + rt, err := content.BuildTransport(true, "/definitely/not/here.pem") + if err != nil { + t.Fatalf("want no error (caFile must be ignored when insecure), got %v", err) + } + if rt == nil { + t.Fatal("want a transport, got nil") + } +} + +func TestBuildTransport_CAFileErrors(t *testing.T) { + if _, err := content.BuildTransport(false, "/definitely/not/here.pem"); err == nil { + t.Fatal("missing caFile: want error, got nil") + } + junk := filepath.Join(t.TempDir(), "junk.pem") + if err := os.WriteFile(junk, []byte("not a certificate"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := content.BuildTransport(false, junk); err == nil { + t.Fatal("junk caFile: want error, got nil") + } +} + +func TestBuildTransport_Noop(t *testing.T) { + if _, err := content.BuildTransport(false, ""); err != nil { + t.Fatalf("want no error, got %v", err) + } +} + +// Real TLS handshakes: caFile trusts a matching server, rejects an unrelated CA, +// and insecure trusts anything. +func TestBuildTransport_TLSHandshake(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + defer srv.Close() + + caFile := writeCertPEM(t, srv.Certificate().Raw) + rt, err := content.BuildTransport(false, caFile) + if err != nil { + t.Fatal(err) + } + if _, err := (&http.Client{Transport: rt}).Get(srv.URL); err != nil { + t.Fatalf("matching caFile: want success, got %v", err) + } + + rt, _ = content.BuildTransport(false, unrelatedCAFile(t)) + if _, err := (&http.Client{Transport: rt}).Get(srv.URL); err == nil { + t.Fatal("unrelated caFile: want TLS error, got success") + } + + rt, _ = content.BuildTransport(true, "") + if _, err := (&http.Client{Transport: rt}).Get(srv.URL); err != nil { + t.Fatalf("insecure: want success, got %v", err) + } +} + +func writeCertPEM(t *testing.T, der []byte) string { + t.Helper() + p := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(p, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func unrelatedCAFile(t *testing.T) string { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "unrelated"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return writeCertPEM(t, der) +}