diff --git a/.gitignore b/.gitignore index a8e7c0c..bf8ee18 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ vulncheck.out trivy.out CLAUDE.md **/CLAUDE.* +.claude** \ No newline at end of file diff --git a/cmd/hauler/cli/cli.go b/cmd/hauler/cli/cli.go index a32950c..8435fbe 100644 --- a/cmd/hauler/cli/cli.go +++ b/cmd/hauler/cli/cli.go @@ -2,6 +2,8 @@ package cli import ( "context" + "fmt" + "os" cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd" "github.com/sirupsen/logrus" @@ -15,14 +17,35 @@ func New(ctx context.Context, ro *flags.CliRootOpts) *cobra.Command { cmd := &cobra.Command{ Use: "hauler", Short: "Airgap Swiss Army Knife", - Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + "\n Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.", + Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + " | " + consts.HaulerLogLevel + " | " + consts.HaulerAuditLevel + "\n Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + // check for log level env variable or flag + if ro.LogLevel == "" { + ro.LogLevel = os.Getenv(consts.HaulerLogLevel) + } + // default to info log level + if ro.LogLevel == "" { + ro.LogLevel = "info" + } + + // check for audit level env variable or flag + if ro.AuditLevel == "" { + ro.AuditLevel = os.Getenv(consts.HaulerAuditLevel) + } + // default to standard audit level + if ro.AuditLevel == "" { + ro.AuditLevel = "standard" + } + switch ro.AuditLevel { + case "none", "standard", "verbose": + default: + return fmt.Errorf("invalid --audit-level %q: must be one of none, standard, verbose", ro.AuditLevel) + } + l := log.FromContext(ctx) l.SetLevel(ro.LogLevel) l.Debugf("running cli command [%s]", cmd.CommandPath()) - // Suppress WARN-level messages from containerd and other - // libraries that use the global logrus logger. if ro.LogLevel == "debug" { logrus.SetLevel(logrus.DebugLevel) } else { diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 17cccfd..174f72c 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -50,7 +50,7 @@ func addStoreExtract(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Com RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -103,7 +103,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman return store.SyncCmd(ctx, o, nil, rso, ro) } - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -126,7 +126,7 @@ func addStoreLoad(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -165,7 +165,7 @@ func addStoreServeRegistry(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cob RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -188,7 +188,7 @@ func addStoreServeFiles(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra. RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -212,7 +212,7 @@ func addStoreSave(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -239,7 +239,7 @@ func addStoreInfo(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -270,7 +270,7 @@ func addStoreCopy(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -319,12 +319,12 @@ func addStoreAddFile(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Com RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } - return store.AddFileCmd(ctx, o, s, args[0]) + return store.AddFileCmd(ctx, o, s, args[0], ro) }, } o.AddFlags(cmd) @@ -363,7 +363,7 @@ func addStoreAddImage(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -406,7 +406,7 @@ func addStoreAddChart(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := o.Store(ctx) + s, err := o.Store(ctx, ro) if err != nil { return err } @@ -448,12 +448,12 @@ func addStoreRemove(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comm RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - s, err := rso.Store(ctx) + s, err := rso.Store(ctx, ro) if err != nil { return err } - return store.RemoveCmd(ctx, o, s, args[0]) + return store.RemoveCmd(ctx, o, s, args[0], ro, rso) }, } o.AddFlags(cmd) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 87b6bb1..11cb286 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -23,6 +23,7 @@ import ( "hauler.dev/go/hauler/v2/internal/flags" v1 "hauler.dev/go/hauler/v2/pkg/apis/hauler.cattle.io/v1" "hauler.dev/go/hauler/v2/pkg/artifacts/file" + "hauler.dev/go/hauler/v2/pkg/audit" "hauler.dev/go/hauler/v2/pkg/consts" "hauler.dev/go/hauler/v2/pkg/content/chart" "hauler.dev/go/hauler/v2/pkg/cosign" @@ -33,17 +34,17 @@ import ( "hauler.dev/go/hauler/v2/pkg/store" ) -func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string) error { +func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string, ro *flags.CliRootOpts) error { cfg := v1.File{ Path: reference, } if len(o.Name) > 0 { cfg.Name = o.Name } - return storeFile(ctx, s, cfg) + return storeFile(ctx, s, cfg, ro, o.StoreRootOpts) } -func storeFile(ctx context.Context, s *store.Layout, fi v1.File) error { +func storeFile(ctx context.Context, s *store.Layout, fi v1.File, ro *flags.CliRootOpts, rso *flags.StoreRootOpts) error { l := log.FromContext(ctx) copts := getter.ClientOptions{ @@ -57,11 +58,45 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1.File) error { } l.Infof("adding file [%s] to the store as [%s]", fi.Path, ref.Name()) - _, err = s.AddArtifact(ctx, f, ref.Name()) + desc, err := s.AddArtifact(ctx, f, ref.Name()) if err != nil { return err } + resolvedPath := fi.Path + if !strings.HasPrefix(fi.Path, "http://") && !strings.HasPrefix(fi.Path, "https://") { + if abs, err := filepath.Abs(fi.Path); err == nil { + resolvedPath = abs + } + } + if auditLevel(ro) != "none" { + e := audit.Entry{ + StoreID: s.StoreID, + Store: s.Root, + Type: "file", + Command: "store add file", + Args: []string{audit.SanitizeURL(fi.Path)}, + Reference: audit.SanitizeURL(resolvedPath), + PortableReference: audit.ShortFileRef(fi.Path), + Digest: desc.Digest.String(), + } + if auditLevel(ro) == "verbose" { + sys := audit.BuildSystem() + g := audit.BuildGlobal(ro, rso) + e.System = &sys + e.Global = &g + e.Flags = map[string]any{ + "name": fi.Name, + } + } + if err := audit.Append(ro.HaulerDir, e); err != nil { + l.Warnf("failed to write audit entry: %v", err) + } + l.Debugf("generated audit id of [%s]", audit.ID()) + } else { + l.Debugf("generated audit id of [none]") + } + l.Infof("successfully added file [%s]", ref.Name()) return nil @@ -71,9 +106,18 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re l := log.FromContext(ctx) cfg := v1.Image{ - Name: reference, - Rewrite: o.Rewrite, - Local: o.Local, + Name: reference, + Key: o.Key, + Tlog: o.Tlog, + CertIdentity: o.CertIdentity, + CertIdentityRegexp: o.CertIdentityRegexp, + CertOidcIssuer: o.CertOidcIssuer, + CertOidcIssuerRegexp: o.CertOidcIssuerRegexp, + CertGithubWorkflowRepository: o.CertGithubWorkflowRepository, + Platform: o.Platform, + Rewrite: o.Rewrite, + ExcludeExtras: o.ExcludeExtras, + Local: o.Local, } if o.Local { @@ -132,7 +176,8 @@ func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags. return err } - if err := s.AddLocalImage(ctx, r.Name()); err != nil { + localDigest, err := s.AddLocalImage(ctx, r.Name()) + if err != nil { if ro.IgnoreErrors { l.Warnf("unable to add image [%s] from Docker daemon to store: %v... skipping...", r.Name(), err) return nil @@ -160,6 +205,35 @@ func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags. } } + if auditLevel(ro) != "none" { + e := audit.Entry{ + StoreID: s.StoreID, + Store: s.Root, + Type: "image", + Command: "store add image", + Args: []string{i.Name}, + Reference: r.Name(), + Digest: localDigest, + } + if auditLevel(ro) == "verbose" { + sys := audit.BuildSystem() + g := audit.BuildGlobal(ro, nil) + e.System = &sys + e.Global = &g + e.Flags = map[string]any{ + "verified": false, + "local": true, + "rewrite": rewrite, + } + } + if err := audit.Append(ro.HaulerDir, e); err != nil { + l.Warnf("failed to write audit entry: %v", err) + } + l.Debugf("generated audit id of [%s]", audit.ID()) + } else { + l.Debugf("generated audit id of [none]") + } + l.Infof("successfully added image [%s] from local Docker daemon", r.Name()) return nil } @@ -188,8 +262,11 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin } // fetch image along with any associated signatures and attestations + var imageDigest string err = retry.Operation(ctx, rso, ro, func() error { - return s.AddImage(ctx, r.Name(), platform, excludeExtras) + var addErr error + imageDigest, addErr = s.AddImage(ctx, r.Name(), platform, excludeExtras) + return addErr }) if err != nil { if ro.IgnoreErrors { @@ -221,6 +298,44 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin } } + verified := i.Key != "" || i.CertIdentity != "" || i.CertIdentityRegexp != "" + if auditLevel(ro) != "none" { + e := audit.Entry{ + StoreID: s.StoreID, + Store: s.Root, + Type: "image", + Command: "store add image", + Args: []string{i.Name}, + Reference: r.Name(), + Digest: imageDigest, + } + if auditLevel(ro) == "verbose" { + sys := audit.BuildSystem() + g := audit.BuildGlobal(ro, rso) + e.System = &sys + e.Global = &g + e.Flags = map[string]any{ + "verified": verified, + "platform": platform, + "key": i.Key, + "use-tlog-verify": i.Tlog, + "certificate-identity": i.CertIdentity, + "certificate-identity-regexp": i.CertIdentityRegexp, + "certificate-oidc-issuer": i.CertOidcIssuer, + "certificate-oidc-issuer-regexp": i.CertOidcIssuerRegexp, + "certificate-github-workflow-repository": i.CertGithubWorkflowRepository, + "rewrite": rewrite, + "exclude-extras": excludeExtras, + } + } + if err := audit.Append(ro.HaulerDir, e); err != nil { + l.Warnf("failed to write audit entry: %v", err) + } + l.Debugf("generated audit id of [%s]", audit.ID()) + } else { + l.Debugf("generated audit id of [none]") + } + l.Infof("successfully added image [%s]", r.Name()) return nil } @@ -446,13 +561,55 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags. return err } - if _, err := s.AddArtifact(ctx, chrt, ref.Name()); err != nil { + chartDesc, err := s.AddArtifact(ctx, chrt, ref.Name()) + if err != nil { return err } if err := s.OCI.SaveIndex(); err != nil { return err } + if auditLevel(ro) != "none" { + e := audit.Entry{ + StoreID: s.StoreID, + Store: s.Root, + Type: "chart", + Command: "store add chart", + Args: []string{c.Name()}, + Reference: c.Name() + ":" + c.Metadata.Version, + Digest: chartDesc.Digest.String(), + } + if auditLevel(ro) == "verbose" { + sys := audit.BuildSystem() + g := audit.BuildGlobal(ro, rso) + e.System = &sys + e.Global = &g + e.Flags = map[string]any{ + "repo": audit.SanitizeURL(cfg.RepoURL), + "version": c.Metadata.Version, + "rewrite": rewrite, + "add-images": opts.AddImages, + "add-dependencies": opts.AddDependencies, + "exclude-extras": opts.ExcludeExtras, + "values": opts.ValuesFiles, + "platform": opts.Platform, + "registry": opts.Registry, + "kube-version": opts.KubeVersion, + "verify": opts.ChartOpts.Verify, + "insecure-skip-tls-verify": opts.ChartOpts.InsecureSkipTLSVerify, + "ca-file": opts.ChartOpts.CaFile, + "cert-file": opts.ChartOpts.CertFile, + "key-file": opts.ChartOpts.KeyFile, + } + } + if err := audit.Append(ro.HaulerDir, e); err != nil { + l.Warnf("failed to write audit entry: %v", err) + } + l.Debugf("generated audit id of [%s]", audit.ID()) + } else { + l.Debugf("generated audit id of [none]") + } + l.Infof("%ssuccessfully added chart [%s:%s]", prefix, c.Name(), c.Metadata.Version) tempOverride := rso.TempOverride diff --git a/cmd/hauler/cli/store/add_test.go b/cmd/hauler/cli/store/add_test.go index f7a98f0..1917703 100644 --- a/cmd/hauler/cli/store/add_test.go +++ b/cmd/hauler/cli/store/add_test.go @@ -237,7 +237,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -294,7 +294,7 @@ func TestRewriteReference(t *testing.T) { if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil { t.Fatalf("rewriteReference: %v", err) } - // library/ must be stripped; registry stays index.docker.io + // library/ must be stripped... registry stays index.docker.io assertAnnotationsInStore(t, s, "nginx:v2", "index.docker.io/nginx:v2") }) @@ -339,7 +339,7 @@ func TestRewriteReference(t *testing.T) { seedImage(t, host, "src/repo", "v1", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -371,7 +371,7 @@ func TestStoreFile(t *testing.T) { tmp.Close() s := newTestStore(t) - if err := storeFile(ctx, s, v1.File{Path: tmp.Name()}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: tmp.Name()}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } assertArtifactInStore(t, s, filepath.Base(tmp.Name())) @@ -380,7 +380,7 @@ func TestStoreFile(t *testing.T) { t.Run("HTTP URL stored under basename", func(t *testing.T) { url := seedFileInHTTPServer(t, "script.sh", "#!/bin/sh\necho ok") s := newTestStore(t) - if err := storeFile(ctx, s, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } assertArtifactInStore(t, s, "script.sh") @@ -394,7 +394,7 @@ func TestStoreFile(t *testing.T) { tmp.Close() s := newTestStore(t) - if err := storeFile(ctx, s, v1.File{Path: tmp.Name(), Name: "custom.sh"}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: tmp.Name(), Name: "custom.sh"}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } assertArtifactInStore(t, s, "custom.sh") @@ -402,7 +402,7 @@ func TestStoreFile(t *testing.T) { t.Run("nonexistent local path returns error", func(t *testing.T) { s := newTestStore(t) - err := storeFile(ctx, s, v1.File{Path: "/nonexistent/path/missing-file.txt"}) + err := storeFile(ctx, s, v1.File{Path: "/nonexistent/path/missing-file.txt"}, defaultCliOpts(), defaultRootOpts(s.Root)) if err == nil { t.Fatal("expected error for nonexistent path, got nil") } @@ -421,7 +421,7 @@ func TestAddFileCmd(t *testing.T) { tmp.Close() o := &flags.AddFileOpts{Name: "renamed.txt"} - if err := AddFileCmd(ctx, o, s, tmp.Name()); err != nil { + if err := AddFileCmd(ctx, o, s, tmp.Name(), defaultCliOpts()); err != nil { t.Fatalf("AddFileCmd: %v", err) } assertArtifactInStore(t, s, "renamed.txt") diff --git a/cmd/hauler/cli/store/audit.go b/cmd/hauler/cli/store/audit.go new file mode 100644 index 0000000..c8195b2 --- /dev/null +++ b/cmd/hauler/cli/store/audit.go @@ -0,0 +1,14 @@ +package store + +import "hauler.dev/go/hauler/v2/internal/flags" + +// auditLevel returns the resolved audit level (none, standard, verbose) +func auditLevel(ro *flags.CliRootOpts) string { + if ro == nil { + return "none" + } + if ro.AuditLevel == "" { + return "standard" + } + return ro.AuditLevel +} diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index c2e7338..360357b 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -237,7 +237,7 @@ func CopyCmd(ctx context.Context, o *flags.CopyOpts, s *store.Layout, targetRef // OCI 1.1 referrer (cosign v3 new-bundle-format): push by manifest digest so // the target registry wires it up via the OCI Referrers API (subject field). // For registries that don't support the Referrers API natively, the manifest - // is still pushed intact; the subject linkage depends on registry support. + // is still pushed intact... the subject linkage depends on registry support. repo := baseRef if colon := strings.LastIndex(baseRef, ":"); colon != -1 { repo = baseRef[:colon] diff --git a/cmd/hauler/cli/store/copy_test.go b/cmd/hauler/cli/store/copy_test.go index 17b77a5..2bb80bf 100644 --- a/cmd/hauler/cli/store/copy_test.go +++ b/cmd/hauler/cli/store/copy_test.go @@ -176,7 +176,7 @@ func TestCopyCmd_Registry_SigTagDerivation(t *testing.T) { // AddImage discovers and stores the .sig/.att/.sbom tags automatically. s := newTestStore(t) - if err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -257,7 +257,7 @@ func TestCopyCmd_Registry_InvalidFilenameSkipTest(t *testing.T) { if err := os.WriteFile(p, []byte(pf.content), 0644); err != nil { t.Fatalf("WriteFile %s: %v", pf.name, err) } - if err := storeFile(ctx, s, v1.File{Path: p}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: p}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile %s: %v", pf.name, err) } } @@ -304,7 +304,7 @@ func TestCopyCmd_Dir_Files(t *testing.T) { url := seedFileInHTTPServer(t, "data.txt", content) s := newTestStore(t) - if err := storeFile(ctx, s, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } @@ -395,6 +395,6 @@ func TestCopyCmd_Dir_Charts(t *testing.T) { for i, e := range entries { names[i] = e.Name() } - t.Errorf("no .tgz found in destDir after chart copy; found: %v", names) + t.Errorf("no .tgz found in destDir after chart copy... found: %v", names) } } diff --git a/cmd/hauler/cli/store/extract.go b/cmd/hauler/cli/store/extract.go index 0bb1dbd..4adc072 100644 --- a/cmd/hauler/cli/store/extract.go +++ b/cmd/hauler/cli/store/extract.go @@ -23,7 +23,7 @@ func isIndexMediaType(mt string) bool { // firstLeafManifest walks a (potentially nested) OCI index and returns the // decoded manifest of the first non-index child. It prefers non-index children -// at each level; if all children are indexes it descends into the first one. +// at each level... if all children are indexes it descends into the first one. // Returns an error if any nested index or manifest cannot be decoded. func firstLeafManifest(ctx context.Context, s *store.Layout, idx ocispec.Index) (ocispec.Manifest, error) { for { @@ -31,7 +31,7 @@ func firstLeafManifest(ctx context.Context, s *store.Layout, idx ocispec.Index) return ocispec.Manifest{}, fmt.Errorf("image index has no child manifests") } - // Prefer the first non-index child; fall back to the first child (an index) if all are indexes. + // Prefer the first non-index child... fall back to the first child (an index) if all are indexes. desc := idx.Manifests[0] for _, d := range idx.Manifests { if !isIndexMediaType(d.MediaType) { diff --git a/cmd/hauler/cli/store/extract_test.go b/cmd/hauler/cli/store/extract_test.go index 91ba593..c59e183 100644 --- a/cmd/hauler/cli/store/extract_test.go +++ b/cmd/hauler/cli/store/extract_test.go @@ -31,7 +31,7 @@ func TestExtractCmd_File(t *testing.T) { fileContent := "hello extract test" url := seedFileInHTTPServer(t, "extract-me.txt", fileContent) - if err := storeFile(ctx, s, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } @@ -167,7 +167,7 @@ func TestExtractCmd_OciArtifactKindImage(t *testing.T) { // Pull into a fresh store — AddImage sets kind=KindAnnotationImage on all manifests. s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -249,7 +249,7 @@ func TestExtractCmd_OciImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -288,7 +288,7 @@ func TestExtractCmd_OciImageIndex_NoBinFiles(t *testing.T) { } } if !found { - t.Errorf("expected binary %q not found; got: %v", want, names) + t.Errorf("expected binary %q not found... got: %v", want, names) } } } @@ -359,7 +359,7 @@ func TestExtractCmd_NestedImageIndex_NoBinFiles(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -398,7 +398,7 @@ func TestExtractCmd_NestedImageIndex_NoBinFiles(t *testing.T) { } } if !found { - t.Errorf("expected binary %q not found; got: %v", want, names) + t.Errorf("expected binary %q not found... got: %v", want, names) } } } @@ -427,7 +427,7 @@ func TestExtractCmd_ContainerImage_Skipped(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -497,7 +497,7 @@ func TestExtractCmd_ContainerImageIndex_Skipped(t *testing.T) { } s := newTestStore(t) - if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { + if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil { t.Fatalf("AddImage: %v", err) } @@ -533,7 +533,7 @@ func TestExtractCmd_SubstringMatch(t *testing.T) { fileContent := "substring match content" url := seedFileInHTTPServer(t, "extract-sub.txt", fileContent) - if err := storeFile(ctx, s, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } @@ -581,7 +581,7 @@ func TestExtractCmd_CosignArtifactsProduceNoContainerImageWarning(t *testing.T) // Seed a real file artifact so ExtractCmd finds something to extract. fileContent := "cosign-filter test file content" url := seedFileInHTTPServer(t, "sigtest.txt", fileContent) - if err := storeFile(ctx, s, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } diff --git a/cmd/hauler/cli/store/info.go b/cmd/hauler/cli/store/info.go index 9d5d202..8fed740 100644 --- a/cmd/hauler/cli/store/info.go +++ b/cmd/hauler/cli/store/info.go @@ -18,6 +18,12 @@ import ( "hauler.dev/go/hauler/v2/pkg/store" ) +type infoOutput struct { + StorePath string `json:"store-path"` + StoreID string `json:"store-id"` + Artifacts []item `json:"artifacts"` +} + func InfoCmd(ctx context.Context, o *flags.InfoOpts, s *store.Layout) error { var items []item if err := s.Walk(func(ref string, desc ocispec.Descriptor) error { @@ -124,13 +130,20 @@ func InfoCmd(ctx context.Context, o *flags.InfoOpts, s *store.Layout) error { // sort items by ref and arch sort.Sort(byReferenceAndArch(items)) - var msg string switch o.OutputFormat { case "json": - msg = buildJson(items...) - fmt.Println(msg) + out := infoOutput{ + StorePath: s.Root, + StoreID: s.StoreID, + Artifacts: items, + } + data, err := json.MarshalIndent(out, "", " ") + if err != nil { + return err + } + fmt.Println(string(data)) default: - if err := buildTable(o.ShowDigests, items...); err != nil { + if err := buildTable(s.Root, s.StoreID, o.ShowDigests, items...); err != nil { return err } } @@ -161,10 +174,11 @@ func buildListRepos(items ...item) { } } -func buildTable(showDigests bool, items ...item) error { +func buildTable(storePath, storeID string, showDigests bool, items ...item) error { table := tablewriter.NewTable(os.Stdout) table.Configure(func(cfg *tablewriter.Config) { cfg.Header.Alignment.Global = tw.AlignLeft + cfg.Footer.Alignment.PerColumn = []tw.Align{tw.AlignLeft} cfg.Row.Merging.Mode = tw.MergeVertical cfg.Row.Merging.ByColumnIndex = tw.NewBoolMapper(0) }) @@ -214,11 +228,11 @@ func buildTable(showDigests bool, items ...item) error { } } - // align total column based on digest visibility + footerLabel := "store-path: " + storePath + "\nstore-id: " + storeID if showDigests { - table.Footer("", "", "", "", "Total", byteCountSI(totalSize)) + table.Footer(footerLabel, "", "", "", "Total", byteCountSI(totalSize)) } else { - table.Footer("", "", "", "Total", byteCountSI(totalSize)) + table.Footer(footerLabel, "", "", "Total", byteCountSI(totalSize)) } return table.Render() @@ -237,21 +251,13 @@ func truncateReference(ref string) string { return ref } -func buildJson(item ...item) string { - data, err := json.MarshalIndent(item, "", " ") - if err != nil { - return "" - } - return string(data) -} - type item struct { - Reference string - Type string - Platform string - Digest string - Layers int - Size int64 + Reference string `json:"reference"` + Type string `json:"type"` + Platform string `json:"platform"` + Digest string `json:"digest,omitempty"` + Layers int `json:"layers"` + Size int64 `json:"size"` } type byReferenceAndArch []item diff --git a/cmd/hauler/cli/store/info_test.go b/cmd/hauler/cli/store/info_test.go index 25c9175..092de2b 100644 --- a/cmd/hauler/cli/store/info_test.go +++ b/cmd/hauler/cli/store/info_test.go @@ -53,37 +53,45 @@ func TestTruncateReference(t *testing.T) { } } -func TestBuildJson(t *testing.T) { +func TestInfoOutputJSON(t *testing.T) { items := []item{ {Reference: "myrepo/myimage:v1", Type: "image", Platform: "linux/amd64", Size: 1024, Layers: 2}, {Reference: "myrepo/mychart:v1", Type: "chart", Platform: "-", Size: 512, Layers: 1}, } - out := buildJson(items...) - if out == "" { - t.Fatal("buildJson returned empty string") + out := infoOutput{ + StorePath: "/tmp/test-store", + StoreID: "test-store-id", + Artifacts: items, } - var got []item - if err := json.Unmarshal([]byte(out), &got); err != nil { - t.Fatalf("buildJson output is not valid JSON: %v\noutput: %s", err, out) + data, err := json.Marshal(out) + if err != nil { + t.Fatalf("json.Marshal infoOutput: %v", err) } - if len(got) != len(items) { - t.Fatalf("buildJson: got %d items, want %d", len(got), len(items)) + var got infoOutput + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("json.Unmarshal infoOutput: %v\ndata: %s", err, data) + } + if got.StorePath != out.StorePath { + t.Errorf("StorePath = %q, want %q", got.StorePath, out.StorePath) + } + if len(got.Artifacts) != len(items) { + t.Fatalf("Artifacts len = %d, want %d", len(got.Artifacts), len(items)) } for i, want := range items { - if got[i].Reference != want.Reference { - t.Errorf("item[%d].Reference = %q, want %q", i, got[i].Reference, want.Reference) + if got.Artifacts[i].Reference != want.Reference { + t.Errorf("Artifacts[%d].Reference = %q, want %q", i, got.Artifacts[i].Reference, want.Reference) } - if got[i].Type != want.Type { - t.Errorf("item[%d].Type = %q, want %q", i, got[i].Type, want.Type) + if got.Artifacts[i].Type != want.Type { + t.Errorf("Artifacts[%d].Type = %q, want %q", i, got.Artifacts[i].Type, want.Type) } - if got[i].Size != want.Size { - t.Errorf("item[%d].Size = %d, want %d", i, got[i].Size, want.Size) + if got.Artifacts[i].Size != want.Size { + t.Errorf("Artifacts[%d].Size = %d, want %d", i, got.Artifacts[i].Size, want.Size) } } } func TestNewItem(t *testing.T) { - // newItem uses s only for its signature; it does not dereference s in practice. + // newItem uses s only for its signature... it does not dereference s in practice. // We pass nil to keep tests dependency-free. const validRef = "myrepo/myimage:latest" @@ -199,7 +207,7 @@ func TestInfoCmd(t *testing.T) { t.Fatalf("write tmpFile: %v", err) } fi := v1.File{Path: tmpFile} - if err := storeFile(ctx, s, fi); err != nil { + if err := storeFile(ctx, s, fi, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } @@ -224,7 +232,7 @@ func TestInfoCmd(t *testing.T) { }) t.Run("TypeFilter:image json", func(t *testing.T) { - // Store has only a file artifact; image filter returns no items (no error). + // Store has only a file artifact... image filter returns no items (no error). if err := InfoCmd(ctx, baseOpts("image", "json"), s); err != nil { t.Errorf("InfoCmd(image, json): %v", err) } diff --git a/cmd/hauler/cli/store/lifecycle_test.go b/cmd/hauler/cli/store/lifecycle_test.go index 17f4243..dc9d25b 100644 --- a/cmd/hauler/cli/store/lifecycle_test.go +++ b/cmd/hauler/cli/store/lifecycle_test.go @@ -31,7 +31,7 @@ func TestLifecycle_FileArtifact_AddSaveLoadCopy(t *testing.T) { // Step 2: storeFile into store A. storeA := newTestStore(t) - if err := storeFile(ctx, storeA, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, storeA, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil { t.Fatalf("storeFile: %v", err) } assertArtifactInStore(t, storeA, "lifecycle.txt") @@ -309,10 +309,10 @@ func TestLifecycle_Remove_ThenSave(t *testing.T) { url2 := seedFileInHTTPServer(t, "remove-me.txt", "content to remove") storeA := newTestStore(t) - if err := storeFile(ctx, storeA, v1.File{Path: url1}); err != nil { + if err := storeFile(ctx, storeA, v1.File{Path: url1}, defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil { t.Fatalf("storeFile keep-me: %v", err) } - if err := storeFile(ctx, storeA, v1.File{Path: url2}); err != nil { + if err := storeFile(ctx, storeA, v1.File{Path: url2}, defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil { t.Fatalf("storeFile remove-me: %v", err) } @@ -321,7 +321,7 @@ func TestLifecycle_Remove_ThenSave(t *testing.T) { } // Step 2: RemoveCmd(Force:true) on the "remove-me" artifact. - if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, storeA, "remove-me"); err != nil { + if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, storeA, "remove-me", defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil { t.Fatalf("RemoveCmd: %v", err) } diff --git a/cmd/hauler/cli/store/load_test.go b/cmd/hauler/cli/store/load_test.go index fd6d6ee..0073743 100644 --- a/cmd/hauler/cli/store/load_test.go +++ b/cmd/hauler/cli/store/load_test.go @@ -367,11 +367,11 @@ func TestUnarchiveLayoutTo_LegacyKindMigration(t *testing.T) { if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error { kind := desc.Annotations[consts.KindAnnotationName] if strings.HasPrefix(kind, legacyPrefix) { - t.Errorf("descriptor %s still has legacy kind %q; expected dev.hauler prefix", + t.Errorf("descriptor %s still has legacy kind %q... expected dev.hauler prefix", desc.Digest, kind) } if !strings.HasPrefix(kind, newPrefix) { - t.Errorf("descriptor %s has unexpected kind %q; expected dev.hauler prefix", + t.Errorf("descriptor %s has unexpected kind %q... expected dev.hauler prefix", desc.Digest, kind) } return nil diff --git a/cmd/hauler/cli/store/remove.go b/cmd/hauler/cli/store/remove.go index e72b390..d6cb1c8 100644 --- a/cmd/hauler/cli/store/remove.go +++ b/cmd/hauler/cli/store/remove.go @@ -3,6 +3,7 @@ package store import ( "bufio" "context" + "encoding/json" "errors" "fmt" "io" @@ -12,10 +13,50 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "hauler.dev/go/hauler/v2/internal/flags" + "hauler.dev/go/hauler/v2/pkg/audit" + "hauler.dev/go/hauler/v2/pkg/consts" "hauler.dev/go/hauler/v2/pkg/log" "hauler.dev/go/hauler/v2/pkg/store" ) +// artifactType derives a human-readable content type for an artifact the same way `store info` +// does: from the manifest's config media type, since AddArtifact stores every non-image-command +// artifact (files, charts) under the same "kind" annotation and can't distinguish them +func artifactType(ctx context.Context, s *store.Layout, desc ocispec.Descriptor) string { + switch { + case desc.Annotations[consts.KindAnnotationName] == consts.KindAnnotationSigs: + return "sigs" + case desc.Annotations[consts.KindAnnotationName] == consts.KindAnnotationAtts: + return "atts" + case desc.Annotations[consts.KindAnnotationName] == consts.KindAnnotationSboms: + return "sbom" + case strings.HasPrefix(desc.Annotations[consts.KindAnnotationName], consts.KindAnnotationReferrers): + return "referrer" + case desc.MediaType == consts.OCIImageIndexSchema, desc.MediaType == consts.DockerManifestListSchema2: + return "image" + } + + rc, err := s.Fetch(ctx, desc) + if err != nil { + return "image" + } + defer rc.Close() + + var m ocispec.Manifest + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return "image" + } + + switch m.Config.MediaType { + case consts.ChartConfigMediaType: + return "chart" + case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType, consts.FileDirectoryConfigMediaType: + return "file" + default: + return "image" + } +} + func formatReference(ref string) string { tagIdx := strings.LastIndex(ref, ":") if tagIdx == -1 { @@ -39,7 +80,7 @@ func formatReference(ref string) string { return fmt.Sprintf("%s [%s]", base, suffix) } -func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref string) error { +func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref string, ro *flags.CliRootOpts, rso *flags.StoreRootOpts) error { l := log.FromContext(ctx) // collect matching artifacts @@ -106,6 +147,37 @@ func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref st return fmt.Errorf("failed to remove artifact [%s]: %w", formatReference(m.reference), err) } + if auditLevel(ro) != "none" { + cleanRef := m.desc.Annotations[consts.ContainerdImageNameKey] + if cleanRef == "" { + cleanRef = m.desc.Annotations[ocispec.AnnotationRefName] + } + e := audit.Entry{ + StoreID: s.StoreID, + Store: s.Root, + Type: artifactType(ctx, s, m.desc), + Command: "store remove", + Args: []string{ref}, + Reference: cleanRef, + Digest: m.desc.Digest.String(), + } + if auditLevel(ro) == "verbose" { + sys := audit.BuildSystem() + g := audit.BuildGlobal(ro, rso) + e.System = &sys + e.Global = &g + e.Flags = map[string]any{ + "force": o.Force, + } + } + if err := audit.Append(ro.HaulerDir, e); err != nil { + l.Warnf("failed to write audit entry: %v", err) + } + l.Debugf("generated audit id of [%s]", audit.ID()) + } else { + l.Debugf("generated audit id of [none]") + } + l.Infof("successfully removed [%s] of type [%s] with digest [%s]", formatReference(m.reference), m.desc.MediaType, m.desc.Digest.String()) } diff --git a/cmd/hauler/cli/store/remove_test.go b/cmd/hauler/cli/store/remove_test.go index a06dbda..dbd1b3e 100644 --- a/cmd/hauler/cli/store/remove_test.go +++ b/cmd/hauler/cli/store/remove_test.go @@ -81,7 +81,7 @@ func TestRemoveCmd_Force(t *testing.T) { s := newTestStore(t) url := seedFileInHTTPServer(t, "removeme.txt", "file-to-remove") - if err := storeFile(ctx, s, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } @@ -103,7 +103,7 @@ func TestRemoveCmd_Force(t *testing.T) { t.Fatal("could not find stored artifact reference containing 'removeme'") } - if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "removeme"); err != nil { + if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "removeme", defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("RemoveCmd: %v", err) } @@ -116,7 +116,7 @@ func TestRemoveCmd_NotFound(t *testing.T) { ctx := newTestContext(t) s := newTestStore(t) - err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "nonexistent-ref") + err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "nonexistent-ref", defaultCliOpts(), defaultRootOpts(s.Root)) if err == nil { t.Fatal("expected error for non-existent ref, got nil") } @@ -133,10 +133,10 @@ func TestRemoveCmd_Force_MultipleMatches(t *testing.T) { url1 := seedFileInHTTPServer(t, "testfile-alpha.txt", "content-alpha") url2 := seedFileInHTTPServer(t, "testfile-beta.txt", "content-beta") - if err := storeFile(ctx, s, v1.File{Path: url1}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url1}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile alpha: %v", err) } - if err := storeFile(ctx, s, v1.File{Path: url2}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url2}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile beta: %v", err) } @@ -145,7 +145,7 @@ func TestRemoveCmd_Force_MultipleMatches(t *testing.T) { } // Remove using a substring that matches both. - if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "testfile"); err != nil { + if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "testfile", defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("RemoveCmd: %v", err) } diff --git a/cmd/hauler/cli/store/save_test.go b/cmd/hauler/cli/store/save_test.go index 113fbf2..f73809a 100644 --- a/cmd/hauler/cli/store/save_test.go +++ b/cmd/hauler/cli/store/save_test.go @@ -59,7 +59,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v1", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -78,7 +78,7 @@ func TestWriteExportsManifest(t *testing.T) { seedIndex(t, host, "test/multiarch", "v2", rOpts...) s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -126,7 +126,7 @@ func TestWriteExportsManifest_DigestOnlyImageHasRepoTag(t *testing.T) { // Add the image BY DIGEST s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false); err != nil { t.Fatalf("AddImage by digest: %v", err) } @@ -149,7 +149,7 @@ func TestWriteExportsManifest_SkipsNonImages(t *testing.T) { url := seedFileInHTTPServer(t, "skip.sh", "#!/bin/sh\necho skip") s := newTestStore(t) - if err := storeFile(ctx, s, v1.File{Path: url}); err != nil { + if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil { t.Fatalf("storeFile: %v", err) } @@ -174,7 +174,7 @@ func TestSaveCmd(t *testing.T) { seedImage(t, host, "test/save", "v1") s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/save:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/save:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -207,7 +207,7 @@ func TestSaveCmd_ContainerdCompatibility(t *testing.T) { seedImage(t, host, "test/containerd-compat", "v1") s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } @@ -236,7 +236,7 @@ func TestSaveCmd_EmptyStore(t *testing.T) { s := newTestStore(t) // SaveCmd uses layout.FromPath which stats index.json — it must exist on - // disk. A fresh store holds the index only in memory; SaveIndex flushes it. + // disk. A fresh store holds the index only in memory... SaveIndex flushes it. if err := s.SaveIndex(); err != nil { t.Fatalf("SaveIndex: %v", err) } @@ -306,7 +306,7 @@ func TestSaveCmd_ChunkSize(t *testing.T) { seedImage(t, host, "test/chunksave", "v1") s := newTestStore(t) - if err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false); err != nil { + if _, err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false); err != nil { t.Fatalf("AddImage: %v", err) } diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index d3d5404..fcddcd7 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -296,7 +296,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor return err } for _, f := range cfg.Spec.Files { - if err := storeFile(ctx, s, f); err != nil { + if err := storeFile(ctx, s, f, ro, rso); err != nil { return err } } diff --git a/cmd/hauler/cli/store/sync_test.go b/cmd/hauler/cli/store/sync_test.go index 2f7ad3a..18a35d2 100644 --- a/cmd/hauler/cli/store/sync_test.go +++ b/cmd/hauler/cli/store/sync_test.go @@ -110,7 +110,7 @@ func TestProcessContent_Images_v1(t *testing.T) { s := newTestStore(t) host, _ := newLocalhostRegistry(t) - seedImage(t, host, "myorg/myimage", "v1") // transport not needed; AddImage reads via localhost scheme + seedImage(t, host, "myorg/myimage", "v1") // transport not needed... AddImage reads via localhost scheme manifest := fmt.Sprintf(`apiVersion: content.hauler.cattle.io/v1 kind: Images diff --git a/cmd/hauler/cli/store/testhelpers_test.go b/cmd/hauler/cli/store/testhelpers_test.go index b44f59d..3e840fd 100644 --- a/cmd/hauler/cli/store/testhelpers_test.go +++ b/cmd/hauler/cli/store/testhelpers_test.go @@ -58,7 +58,7 @@ func newTestRegistry(t *testing.T) (host string, remoteOpts []remote.Option) { } // seedImage pushes a random single-platform image to the test registry. -// repo is a bare path like "myorg/myimage"; tag is the image tag string. +// repo is a bare path like "myorg/myimage"... tag is the image tag string. // Pass the remoteOpts from newTestRegistry so writes use the correct transport. func seedImage(t *testing.T, host, repo, tag string, opts ...remote.Option) gcrv1.Image { t.Helper() @@ -138,11 +138,14 @@ func defaultRootOpts(storeDir string) *flags.StoreRootOpts { } } -// defaultCliOpts returns CliRootOpts with error-level logging and IgnoreErrors=false. +// defaultCliOpts returns CliRootOpts with log level error, audit level none, and ignore errors false. +// Audit is disabled here rather than pointed at a temp HaulerDir so tests don't write to the +// developer/CI user's real $HOME/.hauler/audit.log. func defaultCliOpts() *flags.CliRootOpts { return &flags.CliRootOpts{ IgnoreErrors: false, LogLevel: "error", + AuditLevel: "none", } } diff --git a/go.mod b/go.mod index 88d5d8d..e766a11 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/distribution/distribution/v3 v3.1.1 github.com/distribution/reference v0.6.0 github.com/google/go-containerregistry v0.21.7 + github.com/google/uuid v1.6.0 github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 github.com/mholt/archives v0.1.5 @@ -184,7 +185,6 @@ require ( github.com/google/go-github/v88 v88.0.0 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect diff --git a/internal/flags/cli.go b/internal/flags/cli.go index c68cadc..a85e3f6 100644 --- a/internal/flags/cli.go +++ b/internal/flags/cli.go @@ -6,12 +6,14 @@ type CliRootOpts struct { LogLevel string HaulerDir string IgnoreErrors bool + AuditLevel string } func AddRootFlags(cmd *cobra.Command, ro *CliRootOpts) { pf := cmd.PersistentFlags() - pf.StringVarP(&ro.LogLevel, "log-level", "l", "info", "Set the logging level (i.e. info, debug, warn)") + pf.StringVarP(&ro.LogLevel, "log-level", "l", "", "Set the logging level (i.e. info, debug, warn) (defaults info)") pf.StringVarP(&ro.HaulerDir, "haulerdir", "d", "", "Set the location of the hauler directory (default $HOME/.hauler)") pf.BoolVar(&ro.IgnoreErrors, "ignore-errors", false, "Ignore/Bypass errors (i.e. warn on error) (defaults false)") + pf.StringVar(&ro.AuditLevel, "audit-level", "", "Set the audit logging level (none, standard, verbose) (defaults standard)") } diff --git a/internal/flags/store.go b/internal/flags/store.go index 2ef89bf..e17be34 100644 --- a/internal/flags/store.go +++ b/internal/flags/store.go @@ -3,8 +3,10 @@ package flags import ( "context" "errors" + "fmt" "os" "path/filepath" + "regexp" "github.com/spf13/cobra" "hauler.dev/go/hauler/v2/pkg/consts" @@ -12,6 +14,10 @@ import ( "hauler.dev/go/hauler/v2/pkg/store" ) +// storeIDPattern matches the full shape of a StoreID as generated by uuid.New()... e.g. "ec520cf6-e01b-4d6f-93ea-6588de0d5159" +// Short prefixes (e.g. "ec520cf6") are resolved by ResolveStoreID itself and never reach this pattern unless the lookup fails +var storeIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + type StoreRootOpts struct { StoreDir string Retries int @@ -25,9 +31,11 @@ func (o *StoreRootOpts) AddFlags(cmd *cobra.Command) { pf.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directory determined by the OS") } -func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) { +func (o *StoreRootOpts) Store(ctx context.Context, ro *CliRootOpts) (*store.Layout, error) { l := log.FromContext(ctx) + haulerDir := resolveHaulerDir(ro) + storeDir := o.StoreDir if storeDir == "" { @@ -38,6 +46,22 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) { storeDir = consts.DefaultStoreName } + // If storeDir doesn't exist as a path, see if it's a StoreID registered from a previous run elsewhere + if _, err := os.Stat(storeDir); errors.Is(err, os.ErrNotExist) { + id, resolved, rerr := store.ResolveStoreID(haulerDir, storeDir) + switch { + case rerr == nil: + if !store.MatchesStoreID(resolved, id) { + return nil, fmt.Errorf("store id %q was last seen at %s, but that path no longer contains that store", storeDir, resolved) + } + l.Debugf("resolved store id [%s] to [%s]", storeDir, resolved) + storeDir = resolved + case storeIDPattern.MatchString(storeDir): + // looks like an ID, not a directory name to create + return nil, fmt.Errorf("no store found matching id %q (for directories, use the absolute path)", storeDir) + } + } + abs, err := filepath.Abs(storeDir) if err != nil { return nil, err @@ -55,9 +79,22 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) { return nil, err } - s, err := store.NewLayout(abs) + s, err := store.NewLayout(abs, store.WithHaulerDir(haulerDir)) if err != nil { return nil, err } + l.Debugf("generated store id of [%s]", s.StoreID) return s, nil } + +// resolveHaulerDir mirrors other variable detection, but duplicated to avoid an import cycle +func resolveHaulerDir(ro *CliRootOpts) string { + if ro != nil && ro.HaulerDir != "" { + return ro.HaulerDir + } + if d := os.Getenv(consts.HaulerDir); d != "" { + return d + } + home, _ := os.UserHomeDir() + return filepath.Join(home, consts.DefaultHaulerDirName) +} diff --git a/internal/mapper/filestore.go b/internal/mapper/filestore.go index a24950d..e93c62f 100644 --- a/internal/mapper/filestore.go +++ b/internal/mapper/filestore.go @@ -72,7 +72,7 @@ func (s *pusher) Push(ctx context.Context, desc ocispec.Descriptor) (ccontent.Wr // Get the filename from the mapper function. // An empty filename means the mapper explicitly declined this descriptor (e.g. a - // config blob that has no title annotation); treat it the same as no mapper. + // config blob that has no title annotation... treat it the same as no mapper. filename, err := mapperFn(desc) if err != nil { return nil, err diff --git a/internal/mapper/mappers.go b/internal/mapper/mappers.go index a334320..02afac3 100644 --- a/internal/mapper/mappers.go +++ b/internal/mapper/mappers.go @@ -131,7 +131,7 @@ func Files() map[string]Fn { m["application/vnd.oci.image.layer.v1.tar"] = fileMapperFn // And the tar variant // Catch-all for OCI artifacts that use custom layer media types (e.g. rke2-binary). - // Write the blob if it carries an AnnotationTitle; silently discard everything else + // Write the blob if it carries an AnnotationTitle... silently discard everything else // (config blobs, metadata) by returning an empty filename. m[DefaultCatchAll] = Fn(func(desc ocispec.Descriptor) (string, error) { if title, ok := desc.Annotations[ocispec.AnnotationTitle]; ok { diff --git a/pkg/audit/audit.go b/pkg/audit/audit.go new file mode 100644 index 0000000..f33800a --- /dev/null +++ b/pkg/audit/audit.go @@ -0,0 +1,228 @@ +package audit + +import ( + "encoding/json" + "fmt" + "net" + "net/url" + "os" + osuser "os/user" + "path" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "hauler.dev/go/hauler/v2/internal/flags" + "hauler.dev/go/hauler/v2/pkg/consts" +) + +var auditID string + +// auditID is generated once per process to group all entries from a single invocation +func init() { + if id, err := uuid.NewV7(); err == nil { + auditID = id.String() + } else { + auditID = uuid.New().String() + } +} + +// auditID returns the audit ID for the current process invocation +func ID() string { return auditID } + +// SystemEntry captures OS level context and only records at verbose level to the global log +type SystemEntry struct { + User string `json:"user,omitempty"` + Hostname string `json:"hostname,omitempty"` + IPAddress string `json:"ip-address,omitempty"` +} + +// GlobalEntry captures hauler flag values and only records at verbose audit level +type GlobalEntry struct { + HaulerDir string `json:"haulerdir,omitempty"` + IgnoreErrors bool `json:"ignore-errors,omitempty"` + LogLevel string `json:"log-level,omitempty"` + AuditLevel string `json:"audit-level,omitempty"` + Retries int `json:"retries,omitempty"` + StoreDir string `json:"store,omitempty"` + TempDir string `json:"tempdir,omitempty"` + Env map[string]string `json:"env,omitempty"` +} + +// Entry records a single auditable operation on the store +type Entry struct { + AuditID string `json:"audit-id,omitempty"` + StoreID string `json:"store-id,omitempty"` + Timestamp string `json:"timestamp"` + Command string `json:"command"` + Args []string `json:"args,omitempty"` + Type string `json:"type,omitempty"` + Reference string `json:"reference,omitempty"` + Digest string `json:"digest,omitempty"` + Store string `json:"store,omitempty"` + System *SystemEntry `json:"system,omitempty"` + Global *GlobalEntry `json:"global,omitempty"` + Flags map[string]any `json:"flags,omitempty"` + + // PortableReference replaces Reference in the store audit log + PortableReference string `json:"-"` +} + +// portableEntry is the machine-agnostic subset written to the store audit log +type portableEntry struct { + AuditID string `json:"audit-id"` + StoreID string `json:"store-id,omitempty"` + Timestamp string `json:"timestamp"` + Command string `json:"command"` + Type string `json:"type,omitempty"` + Reference string `json:"reference,omitempty"` + Digest string `json:"digest,omitempty"` +} + +// ShortFileRef returns a portable-safe short name for a local path or URL +func ShortFileRef(raw string) string { + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + if u, err := url.Parse(raw); err == nil { + if base := path.Base(u.Path); base != "." && base != "/" { + return base + } + } + } + return filepath.Base(raw) +} + +// SanitizeURL strips userinfo and the query string from an http(s) URL so +// embedded credentials (presigned signatures, tokens, API keys) never reach +// the audit log, at any audit level. Non-URL values (local paths, image or +// chart references) are returned unchanged. +func SanitizeURL(raw string) string { + if !strings.HasPrefix(raw, "http://") && !strings.HasPrefix(raw, "https://") { + return raw + } + u, err := url.Parse(raw) + if err != nil { + return raw + } + u.User = nil + u.RawQuery = "" + u.Fragment = "" + return u.String() +} + +// BuildSystem returns OS level context for verbose audit entries +func BuildSystem() SystemEntry { + s := SystemEntry{} + if u, err := osuser.Current(); err == nil { + s.User = u.Username + } + s.Hostname, _ = os.Hostname() + if addrs, err := net.InterfaceAddrs(); err == nil { + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() != nil { + s.IPAddress = ipnet.IP.String() + break + } + } + } + return s +} + +// BuildGlobal returns hauler flag values for verbose audit entries +func BuildGlobal(ro *flags.CliRootOpts, rso *flags.StoreRootOpts) GlobalEntry { + g := GlobalEntry{} + if ro != nil { + g.HaulerDir = resolveDir(ro.HaulerDir) + g.IgnoreErrors = ro.IgnoreErrors + g.LogLevel = ro.LogLevel + g.AuditLevel = ro.AuditLevel + } + if rso != nil { + g.Retries = rso.Retries + g.StoreDir = rso.StoreDir + g.TempDir = rso.TempOverride + } + env := map[string]string{} + for _, key := range []string{ + consts.HaulerDir, + consts.HaulerTempDir, + consts.HaulerStoreDir, + consts.HaulerIgnoreErrors, + consts.HaulerLogLevel, + consts.HaulerAuditLevel, + } { + if v := os.Getenv(key); v != "" { + env[key] = v + } + } + if len(env) > 0 { + g.Env = env + } + return g +} + +// Append records a full log entry to /audit.log +// When e.Store is set... a portable subset to /audit.log +func Append(haulerDir string, e Entry) error { + e.AuditID = auditID + e.Timestamp = time.Now().UTC().Format(time.RFC3339) + + // global write... full entry including system/global/flags + var globalErr error + if err := appendLine(resolveDir(haulerDir), e); err != nil { + globalErr = fmt.Errorf("audit: global write: %w", err) + } + + // store write... portable subset only, attempted even if the global write above failed + if e.Store != "" { + reference := e.Reference + if e.PortableReference != "" { + reference = e.PortableReference + } + pe := portableEntry{ + AuditID: e.AuditID, + StoreID: e.StoreID, + Timestamp: e.Timestamp, + Command: e.Command, + Type: e.Type, + Reference: reference, + Digest: e.Digest, + } + if err := appendLine(e.Store, pe); err != nil { + if globalErr != nil { + return fmt.Errorf("%v: audit: store write: %w", globalErr, err) + } + return fmt.Errorf("audit: store write: %w", err) + } + } + + return globalErr +} + +func appendLine(dir string, v any) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("audit: ensure dir: %w", err) + } + data, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("audit: marshal: %w", err) + } + f, err := os.OpenFile(filepath.Join(dir, "audit.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("audit: open log: %w", err) + } + defer f.Close() + _, err = fmt.Fprintf(f, "%s\n", data) + return err +} + +func resolveDir(haulerDir string) string { + if haulerDir != "" { + return haulerDir + } + if d := os.Getenv(consts.HaulerDir); d != "" { + return d + } + home, _ := os.UserHomeDir() + return filepath.Join(home, consts.DefaultHaulerDirName) +} diff --git a/pkg/audit/audit_test.go b/pkg/audit/audit_test.go new file mode 100644 index 0000000..8a44945 --- /dev/null +++ b/pkg/audit/audit_test.go @@ -0,0 +1,214 @@ +package audit + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAppend(t *testing.T) { + dir := t.TempDir() + + e := Entry{ + Command: "store add image", + Args: []string{"busybox:latest"}, + Flags: map[string]any{"platform": "linux/amd64"}, + Store: filepath.Join(dir, "store"), + } + + if err := Append(dir, e); err != nil { + t.Fatalf("Append: %v", err) + } + + path := filepath.Join(dir, "audit.log") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + var got Entry + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("unmarshal audit line: %v\nraw: %s", err, data) + } + + if got.Command != e.Command { + t.Errorf("Command = %q, want %q", got.Command, e.Command) + } + if len(got.Args) != 1 || got.Args[0] != "busybox:latest" { + t.Errorf("Args = %v, want [busybox:latest]", got.Args) + } + if got.Timestamp == "" { + t.Error("Timestamp should be set") + } +} + +// TestAppend_PortableReferenceOverridesStoreCopy verifies PortableReference only affects the store audit log +func TestAppend_PortableReferenceOverridesStoreCopy(t *testing.T) { + dir := t.TempDir() + + absPath := filepath.Join(string(filepath.Separator), "home", "example", "scripts", "install.sh") + typedPath := filepath.Join(".", "scripts", "install.sh") + e := Entry{ + Command: "store add file", + Type: "file", + Args: []string{typedPath}, + Reference: absPath, + PortableReference: typedPath, + Store: filepath.Join(dir, "store"), + } + + if err := Append(dir, e); err != nil { + t.Fatalf("Append: %v", err) + } + + // global log keeps the full path and the as-typed args + globalData, err := os.ReadFile(filepath.Join(dir, "audit.log")) + if err != nil { + t.Fatalf("ReadFile global audit.log: %v", err) + } + var globalGot Entry + if err := json.Unmarshal(globalData, &globalGot); err != nil { + t.Fatalf("unmarshal global audit line: %v\nraw: %s", err, globalData) + } + if globalGot.Reference != absPath { + t.Errorf("global Reference = %q, want %q", globalGot.Reference, absPath) + } + if len(globalGot.Args) != 1 || globalGot.Args[0] != typedPath { + t.Errorf("global Args = %v, want [%s]", globalGot.Args, typedPath) + } + + // store log uses PortableReference instead, and carries no args at all + storeData, err := os.ReadFile(filepath.Join(dir, "store", "audit.log")) + if err != nil { + t.Fatalf("ReadFile store audit log: %v", err) + } + var got portableEntry + if err := json.Unmarshal(storeData, &got); err != nil { + t.Fatalf("unmarshal store audit line: %v\nraw: %s", err, storeData) + } + if got.Reference != typedPath { + t.Errorf("store Reference = %q, want %q", got.Reference, typedPath) + } + if strings.Contains(string(storeData), `"args"`) { + t.Errorf("store entry should not carry args: %s", storeData) + } +} + +// TestAppend_StoreDefaultsToReference verifies the fallback when PortableReference is unset +func TestAppend_StoreDefaultsToReference(t *testing.T) { + dir := t.TempDir() + + e := Entry{ + Command: "store remove", + Type: "file", + Args: []string{"install.sh"}, + Reference: "hauler/install.sh:latest", + Store: filepath.Join(dir, "store"), + } + + if err := Append(dir, e); err != nil { + t.Fatalf("Append: %v", err) + } + + storeData, err := os.ReadFile(filepath.Join(dir, "store", "audit.log")) + if err != nil { + t.Fatalf("ReadFile store audit log: %v", err) + } + var got portableEntry + if err := json.Unmarshal(storeData, &got); err != nil { + t.Fatalf("unmarshal store audit line: %v\nraw: %s", err, storeData) + } + if got.Reference != e.Reference { + t.Errorf("store Reference = %q, want %q (unmodified)", got.Reference, e.Reference) + } +} + +func TestAppend_StoreWriteSucceedsWhenGlobalWriteFails(t *testing.T) { + dir := t.TempDir() + + // occupy the path so appendLine's MkdirAll fails for the global write only + blockedHaulerDir := filepath.Join(dir, "blocked") + if err := os.WriteFile(blockedHaulerDir, []byte("not a dir"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + storeDir := filepath.Join(dir, "store") + e := Entry{ + Command: "store add image", + Args: []string{"busybox:latest"}, + Store: storeDir, + } + + if err := Append(blockedHaulerDir, e); err == nil { + t.Fatal("Append: expected error from unwritable global haulerDir, got nil") + } + + storeData, err := os.ReadFile(filepath.Join(storeDir, "audit.log")) + if err != nil { + t.Fatalf("ReadFile store audit log: %v", err) + } + var got portableEntry + if err := json.Unmarshal(storeData, &got); err != nil { + t.Fatalf("unmarshal store audit line: %v\nraw: %s", err, storeData) + } + if got.Command != e.Command { + t.Errorf("store command = %q, want = %q", got.Command, e.Command) + } +} + +func TestAppend_MultipleEntries(t *testing.T) { + dir := t.TempDir() + + for i := 0; i < 3; i++ { + if err := Append(dir, Entry{Command: "store add image", Args: []string{"img"}}); err != nil { + t.Fatalf("Append[%d]: %v", i, err) + } + } + + data, err := os.ReadFile(filepath.Join(dir, "audit.log")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + lines := 0 + for _, b := range data { + if b == '\n' { + lines++ + } + } + if lines != 3 { + t.Errorf("expected 3 lines, got %d\nlog:\n%s", lines, data) + } +} + +func TestShortFileRef(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"local relative path", "./scripts/install.sh", "install.sh"}, + {"local absolute path", "/home/example/scripts/install.sh", "install.sh"}, + {"bare filename", "install.sh", "install.sh"}, + {"plain URL", "https://get.rke2.io/install.sh", "install.sh"}, + {"URL with credentials", "https://user:pass@get.rke2.io/install.sh", "install.sh"}, + {"URL with query token", "https://get.rke2.io/install.sh?token=abc123", "install.sh"}, + {"URL with credentials and query", "https://user:pass@get.rke2.io/install.sh?sig=abc123", "install.sh"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := ShortFileRef(tc.in); got != tc.want { + t.Errorf("ShortFileRef(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestResolveDir_Default(t *testing.T) { + got := resolveDir("") + if got == "" { + t.Error("resolveDir(\"\") returned empty string") + } +} diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 8a3f28a..7b9c041 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -91,6 +91,8 @@ const ( HaulerTempDir = "HAULER_TEMP_DIR" HaulerStoreDir = "HAULER_STORE_DIR" HaulerIgnoreErrors = "HAULER_IGNORE_ERRORS" + HaulerLogLevel = "HAULER_LOG_LEVEL" + HaulerAuditLevel = "HAULER_AUDIT_LEVEL" // container files and directories ImageManifestFile = "manifest.json" @@ -110,6 +112,8 @@ const ( DefaultFileserverTimeout = 60 DefaultHaulerArchiveName = "haul.tar.zst" DefaultHaulerManifestName = "hauler-manifest.yaml" + DefaultStoreMetadataName = "store.json" + DefaultStoreInventoryName = "stores.json" DefaultRetries = 3 RetriesInterval = 5 CustomTimeFormat = "2006-01-02 15:04:05" diff --git a/pkg/content/oci.go b/pkg/content/oci.go index 4bb68a2..ef13127 100644 --- a/pkg/content/oci.go +++ b/pkg/content/oci.go @@ -92,7 +92,7 @@ func (o *OCI) LoadIndex() error { continue } - // Set default kind if missing; normalize legacy dev.cosignproject.cosign values. + // Set default kind if missing... normalize legacy dev.cosignproject.cosign values kind := desc.Annotations[consts.KindAnnotationName] kind = consts.NormalizeLegacyKind(kind) if kind == "" { diff --git a/pkg/content/oci_test.go b/pkg/content/oci_test.go index 442dd11..404b978 100644 --- a/pkg/content/oci_test.go +++ b/pkg/content/oci_test.go @@ -118,11 +118,11 @@ func TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations(t *testing.T) { for _, desc := range walked { kind := desc.Annotations[consts.KindAnnotationName] if strings.HasPrefix(kind, legacyPrefix) { - t.Errorf("descriptor %s: Walk returned legacy kind %q; want normalized dev.hauler/... value", + t.Errorf("descriptor %s: Walk returned legacy kind %q... want normalized dev.hauler/... value", desc.Digest, kind) } if !strings.HasPrefix(kind, newPrefix) { - t.Errorf("descriptor %s: Walk returned unexpected kind %q; want dev.hauler/... prefix", + t.Errorf("descriptor %s: Walk returned unexpected kind %q... want dev.hauler/... prefix", desc.Digest, kind) } } @@ -260,10 +260,10 @@ func TestPush_NormalizesLegacyKindInStoredDescriptor(t *testing.T) { found = true kind := d.Annotations[consts.KindAnnotationName] if strings.HasPrefix(kind, legacyPrefix) { - t.Errorf("Push stored descriptor with legacy kind %q; want normalized dev.hauler/... value", kind) + t.Errorf("Push stored descriptor with legacy kind %q... want normalized dev.hauler/... value", kind) } if !strings.HasPrefix(kind, newPrefix) { - t.Errorf("Push stored descriptor with unexpected kind %q; want dev.hauler/... prefix", kind) + t.Errorf("Push stored descriptor with unexpected kind %q... want dev.hauler/... prefix", kind) } return nil }); err != nil { diff --git a/pkg/store/inventory.go b/pkg/store/inventory.go new file mode 100644 index 0000000..e7e796a --- /dev/null +++ b/pkg/store/inventory.go @@ -0,0 +1,123 @@ +package store + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + zlog "github.com/rs/zerolog/log" + + "hauler.dev/go/hauler/v2/pkg/consts" +) + +// inventoryEntry is a store's last-known location, keyed by StoreID +type inventoryEntry struct { + Path string `json:"path"` + Updated string `json:"updated"` +} + +type storeInventory map[string]inventoryEntry + +func inventoryPath(haulerDir string) string { + return filepath.Join(haulerDir, consts.DefaultStoreInventoryName) +} + +func loadInventory(haulerDir string) storeInventory { + inv := storeInventory{} + p := inventoryPath(haulerDir) + data, err := os.ReadFile(p) + if err != nil { + return inv + } + if err := json.Unmarshal(data, &inv); err != nil { + zlog.Warn().Err(err).Str("path", p).Msg("failed to parse store inventory... ignoring") + return storeInventory{} + } + return inv +} + +func saveInventory(haulerDir string, inv storeInventory) error { + if err := os.MkdirAll(haulerDir, 0o755); err != nil { + return err + } + data, err := json.MarshalIndent(inv, "", " ") + if err != nil { + return err + } + // write to a temp file and rename to avoid a partially-written inventory + tmp := inventoryPath(haulerDir) + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, inventoryPath(haulerDir)) +} + +// updateStoreInventory records storeID's path in /stores.json and +// prunes any other entries that no longer contain the store they claim +func updateStoreInventory(haulerDir, storeID, path string) { + inv := loadInventory(haulerDir) + + for id, entry := range inv { + if id == storeID { + continue + } + if !MatchesStoreID(entry.Path, id) { + delete(inv, id) + } + } + + inv[storeID] = inventoryEntry{ + Path: path, + Updated: time.Now().UTC().Format(time.RFC3339), + } + + if err := saveInventory(haulerDir, inv); err != nil { + zlog.Warn().Err(err).Msg("failed to update store inventory... store id lookup may not find this store later") + } +} + +// MatchesStoreID reports whether path's store.json identifies it as storeID +func MatchesStoreID(path, storeID string) bool { + data, err := os.ReadFile(filepath.Join(path, consts.DefaultStoreMetadataName)) + if err != nil { + return false + } + var m storeMetadata + if json.Unmarshal(data, &m) != nil { + return false + } + return m.StoreID == storeID +} + +// ResolveStoreID looks up idOrPrefix (a StoreID or an unambiguous prefix of +// one) in /stores.json and returns the matched store's full id and +// last-known path. Callers should verify the path with MatchesStoreID before +// trusting it +func ResolveStoreID(haulerDir, idOrPrefix string) (id string, path string, err error) { + inv := loadInventory(haulerDir) + + if entry, ok := inv[idOrPrefix]; ok { + return idOrPrefix, entry.Path, nil + } + + var matchID, matchPath string + count := 0 + for invID, entry := range inv { + if strings.HasPrefix(invID, idOrPrefix) { + matchID, matchPath = invID, entry.Path + count++ + } + } + + switch count { + case 0: + return "", "", fmt.Errorf("no store found matching id %q", idOrPrefix) + case 1: + return matchID, matchPath, nil + default: + return "", "", fmt.Errorf("store id %q is ambiguous and matches multiple stores", idOrPrefix) + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index cf05bc3..bcf4bee 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -18,9 +18,11 @@ import ( "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/uuid" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog" + zlog "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" "hauler.dev/go/hauler/v2/pkg/artifacts" @@ -31,8 +33,10 @@ import ( type Layout struct { *content.OCI - Root string - cache layer.Cache + Root string + StoreID string + haulerDir string + cache layer.Cache } type Options func(*Layout) @@ -43,6 +47,14 @@ func WithCache(c layer.Cache) Options { } } +// WithHaulerDir records this store in /stores.json so it can +// later be referenced by StoreID. If unset, NewLayout skips the inventory +func WithHaulerDir(dir string) Options { + return func(l *Layout) { + l.haulerDir = dir + } +} + func NewLayout(rootdir string, opts ...Options) (*Layout, error) { ociStore, err := content.NewOCI(rootdir) if err != nil { @@ -54,17 +66,57 @@ func NewLayout(rootdir string, opts ...Options) (*Layout, error) { } l := &Layout{ - Root: rootdir, - OCI: ociStore, + Root: rootdir, + OCI: ociStore, + StoreID: loadOrCreateStoreID(rootdir), } for _, opt := range opts { opt(l) } + if l.haulerDir != "" { + updateStoreInventory(l.haulerDir, l.StoreID, rootdir) + } + return l, nil } +type storeMetadata struct { + StoreID string `json:"store-id"` +} + +// loadOrCreateStoreID returns the persistent store identity from /store.json, +// creating the file with a fresh UUID on first use. +func loadOrCreateStoreID(rootdir string) string { + metaPath := filepath.Join(rootdir, consts.DefaultStoreMetadataName) + if data, err := os.ReadFile(metaPath); err == nil { + var m storeMetadata + if uerr := json.Unmarshal(data, &m); uerr == nil && m.StoreID != "" { + return m.StoreID + } else if uerr != nil { + zlog.Warn().Err(uerr).Str("path", metaPath).Msg("failed to parse store metadata... generating new store id") + } else { + zlog.Warn().Str("path", metaPath).Msg("store metadata missing store-id... generating new store id") + } + } + m := storeMetadata{StoreID: uuid.New().String()} + data, err := json.Marshal(m) + if err != nil { + zlog.Warn().Err(err).Msg("failed to marshal store metadata... store id will not persist across runs") + return m.StoreID + } + tmp := metaPath + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + zlog.Warn().Err(err).Str("path", tmp).Msg("failed to write store metadata... store id will not persist across runs") + return m.StoreID + } + if err := os.Rename(tmp, metaPath); err != nil { + zlog.Warn().Err(err).Str("path", metaPath).Msg("failed to write store metadata... store id will not persist across runs") + } + return m.StoreID +} + // AddArtifact adds an artifacts.OCI to the store // // The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once @@ -157,7 +209,7 @@ func (l *Layout) AddArtifactCollection(ctx context.Context, collection artifacts // discovered via cosign's tag convention (.sig, .att, .sbom). // When platform is non-empty and the ref is a multi-arch index, only that platform is fetched. // When excludeExtras is true, cosign signatures, attestations, SBOMs, and OCI referrers are skipped. -func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, opts ...remote.Option) error { +func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, opts ...remote.Option) (string, error) { allOpts := append([]remote.Option{ remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithContext(ctx), @@ -165,12 +217,12 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl parsedRef, err := gname.ParseReference(ref) if err != nil { - return fmt.Errorf("parsing reference %q: %w", ref, err) + return "", fmt.Errorf("parsing reference %q: %w", ref, err) } desc, err := remote.Get(parsedRef, allOpts...) if err != nil { - return fmt.Errorf("fetching descriptor for %q: %w", ref, err) + return "", fmt.Errorf("fetching descriptor for %q: %w", ref, err) } var imageDigest v1.Hash @@ -179,10 +231,10 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl // Multi-arch image with no platform filter: save the full index. imageDigest, err = idx.Digest() if err != nil { - return fmt.Errorf("getting index digest for %q: %w", ref, err) + return "", fmt.Errorf("getting index digest for %q: %w", ref, err) } if err := l.writeIndex(parsedRef, idx, consts.KindAnnotationIndex); err != nil { - return err + return "", err } } else { // Single-platform image, or the caller requested a specific platform. @@ -190,55 +242,59 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl if platform != "" { p, err := parsePlatform(platform) if err != nil { - return err + return "", err } imgOpts = append(imgOpts, remote.WithPlatform(p)) } img, err := remote.Image(parsedRef, imgOpts...) if err != nil { - return fmt.Errorf("fetching image %q: %w", ref, err) + return "", fmt.Errorf("fetching image %q: %w", ref, err) } imageDigest, err = img.Digest() if err != nil { - return fmt.Errorf("getting image digest for %q: %w", ref, err) + return "", fmt.Errorf("getting image digest for %q: %w", ref, err) } if err := l.writeImage(parsedRef, img, consts.KindAnnotationImage, ""); err != nil { - return err + return "", err } } if !excludeExtras { savedDigests, err := l.saveRelatedArtifacts(ctx, parsedRef, imageDigest, allOpts...) if err != nil { - return err + return "", err } - return l.saveReferrers(ctx, parsedRef, imageDigest, savedDigests, allOpts...) + return imageDigest.String(), l.saveReferrers(ctx, parsedRef, imageDigest, savedDigests, allOpts...) } - return nil + return imageDigest.String(), nil } // AddLocalImage fetches a container image from the local Docker daemon and saves it to the store. // No cosign signatures, attestations, SBOMs, or OCI referrers are fetched (registry-only concepts). -func (l *Layout) AddLocalImage(ctx context.Context, ref string) error { +func (l *Layout) AddLocalImage(ctx context.Context, ref string) (string, error) { parsedRef, err := gname.ParseReference(ref) if err != nil { - return fmt.Errorf("parsing reference %q: %w", ref, err) + return "", fmt.Errorf("parsing reference %q: %w", ref, err) } if err := ensureDockerHost(); err != nil { - return fmt.Errorf("failed to locate Docker daemon socket: %w -- is the Docker daemon running?", err) + return "", fmt.Errorf("failed to locate Docker daemon socket: %w -- is the Docker daemon running?", err) } img, err := daemon.Image(parsedRef, daemon.WithContext(ctx)) if err != nil { - return fmt.Errorf("failed to fetch image from Docker daemon: %w -- is the Docker daemon running?", err) + return "", fmt.Errorf("failed to fetch image from Docker daemon: %w -- is the Docker daemon running?", err) } - if _, err := img.Digest(); err != nil { - return fmt.Errorf("getting image digest for %q: %w", ref, err) + d, err := img.Digest() + if err != nil { + return "", fmt.Errorf("getting image digest for %q: %w", ref, err) } - return l.writeImage(parsedRef, img, consts.KindAnnotationImage, "") + if err := l.writeImage(parsedRef, img, consts.KindAnnotationImage, ""); err != nil { + return "", err + } + return d.String(), nil } // ensureDockerHost sets DOCKER_HOST if it is not already set and the default diff --git a/pkg/store/store_id_test.go b/pkg/store/store_id_test.go new file mode 100644 index 0000000..63de239 --- /dev/null +++ b/pkg/store/store_id_test.go @@ -0,0 +1,197 @@ +package store_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "hauler.dev/go/hauler/v2/pkg/consts" + "hauler.dev/go/hauler/v2/pkg/store" +) + +// TestStoreID_PersistsAcrossNewLayoutCalls verifies the store-id survives repeated NewLayout calls +func TestStoreID_PersistsAcrossNewLayoutCalls(t *testing.T) { + root := t.TempDir() + + s1, err := store.NewLayout(root) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + if s1.StoreID == "" { + t.Fatal("expected a non-empty StoreID on first creation") + } + + metaPath := filepath.Join(root, consts.DefaultStoreMetadataName) + if _, err := os.Stat(metaPath); err != nil { + t.Fatalf("expected %s to be written: %v", metaPath, err) + } + + s2, err := store.NewLayout(root) + if err != nil { + t.Fatalf("second NewLayout: %v", err) + } + if s2.StoreID != s1.StoreID { + t.Errorf("StoreID changed across NewLayout calls: first=%s second=%s", s1.StoreID, s2.StoreID) + } +} + +// TestStoreID_MalformedMetadataRegenerates verifies a corrupt store.json gets a fresh id +func TestStoreID_MalformedMetadataRegenerates(t *testing.T) { + root := t.TempDir() + metaPath := filepath.Join(root, consts.DefaultStoreMetadataName) + if err := os.WriteFile(metaPath, []byte("{not valid json"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + s, err := store.NewLayout(root) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + if s.StoreID == "" { + t.Fatal("expected a freshly generated StoreID when metadata is malformed") + } + + data, err := os.ReadFile(metaPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + var m struct { + StoreID string `json:"store-id"` + } + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("regenerated metadata is not valid JSON: %v", err) + } + if m.StoreID != s.StoreID { + t.Errorf("persisted store-id %q does not match Layout.StoreID %q", m.StoreID, s.StoreID) + } +} + +// TestStoreID_MissingFieldRegenerates verifies a store.json missing store-id gets a fresh id +func TestStoreID_MissingFieldRegenerates(t *testing.T) { + root := t.TempDir() + metaPath := filepath.Join(root, consts.DefaultStoreMetadataName) + if err := os.WriteFile(metaPath, []byte(`{}`), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + s, err := store.NewLayout(root) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + if s.StoreID == "" { + t.Fatal("expected a freshly generated StoreID when store-id field is missing") + } +} + +// TestResolveStoreID covers exact-id, prefix, ambiguous-prefix, and no-match resolution +func TestResolveStoreID(t *testing.T) { + haulerDir := t.TempDir() + + type entry struct { + Path string `json:"path"` + Updated string `json:"updated"` + } + inv := map[string]entry{ + "ec520cf6-e01b-4d6f-93ea-6588de0d5159": {Path: "/store/a", Updated: "2026-01-01T00:00:00Z"}, + "ec520000-0000-0000-0000-000000000000": {Path: "/store/b", Updated: "2026-01-01T00:00:00Z"}, + "deadbeef-dead-beef-dead-beefdeadbeef": {Path: "/store/c", Updated: "2026-01-01T00:00:00Z"}, + } + data, err := json.Marshal(inv) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if err := os.WriteFile(filepath.Join(haulerDir, consts.DefaultStoreInventoryName), data, 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + t.Run("exact id match", func(t *testing.T) { + id, path, err := store.ResolveStoreID(haulerDir, "deadbeef-dead-beef-dead-beefdeadbeef") + if err != nil { + t.Fatalf("ResolveStoreID: %v", err) + } + if id != "deadbeef-dead-beef-dead-beefdeadbeef" || path != "/store/c" { + t.Errorf("got id=%s path=%s, want id=deadbeef-dead-beef-dead-beefdeadbeef path=/store/c", id, path) + } + }) + + t.Run("unambiguous prefix match", func(t *testing.T) { + id, path, err := store.ResolveStoreID(haulerDir, "deadbeef") + if err != nil { + t.Fatalf("ResolveStoreID: %v", err) + } + if id != "deadbeef-dead-beef-dead-beefdeadbeef" || path != "/store/c" { + t.Errorf("got id=%s path=%s, want id=deadbeef-dead-beef-dead-beefdeadbeef path=/store/c", id, path) + } + }) + + t.Run("ambiguous prefix returns error", func(t *testing.T) { + _, _, err := store.ResolveStoreID(haulerDir, "ec52") + if err == nil { + t.Fatal("expected error for ambiguous prefix, got nil") + } + }) + + t.Run("no match returns error", func(t *testing.T) { + _, _, err := store.ResolveStoreID(haulerDir, "abc12345") + if err == nil { + t.Fatal("expected error when no store matches, got nil") + } + }) +} + +// TestNewLayout_UpdatesInventory verifies WithHaulerDir registers the store in stores.json +func TestNewLayout_UpdatesInventory(t *testing.T) { + haulerDir := t.TempDir() + storeDir := t.TempDir() + + s, err := store.NewLayout(storeDir, store.WithHaulerDir(haulerDir)) + if err != nil { + t.Fatalf("NewLayout: %v", err) + } + + id, path, err := store.ResolveStoreID(haulerDir, s.StoreID) + if err != nil { + t.Fatalf("ResolveStoreID: %v", err) + } + if id != s.StoreID { + t.Errorf("resolved id = %s, want %s", id, s.StoreID) + } + if path != storeDir { + t.Errorf("resolved path = %s, want %s", path, storeDir) + } + if !store.MatchesStoreID(path, id) { + t.Error("MatchesStoreID should confirm the resolved path still contains this store") + } +} + +// TestNewLayout_InventoryPrunesStaleEntries verifies stale inventory entries get pruned +func TestNewLayout_InventoryPrunesStaleEntries(t *testing.T) { + haulerDir := t.TempDir() + + type entry struct { + Path string `json:"path"` + Updated string `json:"updated"` + } + staleID := "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + inv := map[string]entry{ + staleID: {Path: filepath.Join(haulerDir, "gone"), Updated: "2026-01-01T00:00:00Z"}, + } + data, err := json.Marshal(inv) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + invPath := filepath.Join(haulerDir, consts.DefaultStoreInventoryName) + if err := os.WriteFile(invPath, data, 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + storeDir := t.TempDir() + if _, err := store.NewLayout(storeDir, store.WithHaulerDir(haulerDir)); err != nil { + t.Fatalf("NewLayout: %v", err) + } + + if _, _, err := store.ResolveStoreID(haulerDir, staleID); err == nil { + t.Error("expected stale inventory entry to be pruned, but it still resolved") + } +} diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index a37e963..789b057 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -392,7 +392,7 @@ func TestCopyDescriptorGraph_Manifest(t *testing.T) { // --- Error path: delete a layer blob from source, expect Copy to fail --- if len(srcManifest.Layers) == 0 { - t.Skip("artifact has no layers; skipping missing-blob error path") + t.Skip("artifact has no layers... skipping missing blob error path") } if err := os.Remove(blobPath(srcRoot, srcManifest.Layers[0].Digest)); err != nil { t.Fatalf("could not remove layer blob to simulate corruption: %v", err) @@ -464,7 +464,7 @@ func TestCopyDescriptorGraph_Index(t *testing.T) { if err != nil { t.Fatal(err) } - if err := src.AddImage(ctx, idxTag.Name(), "", false, remoteOpts...); err != nil { + if _, err := src.AddImage(ctx, idxTag.Name(), "", false, remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) } if err := src.OCI.SaveIndex(); err != nil { @@ -751,7 +751,7 @@ func TestAddImage_OCI11Referrers(t *testing.T) { referrerImg = mutate.ConfigMediaType(referrerImg, types.MediaType(consts.OCIEmptyConfigMediaType)) referrerImg = mutate.Subject(referrerImg, baseDesc).(v1.Image) - // Push the referrer under an arbitrary tag; the in-process registry auto-wires the + // Push the referrer under an arbitrary tag... the in-process registry auto-wires the // subject field and makes the manifest discoverable via GET /v2/.../referrers/. referrerTag, err := gname.NewTag(host+"/test/image:bundle-referrer", gname.Insecure) if err != nil { @@ -767,7 +767,7 @@ func TestAddImage_OCI11Referrers(t *testing.T) { if err != nil { t.Fatalf("new layout: %v", err) } - if err := s.AddImage(context.Background(), baseTag.Name(), "", false, remoteOpts...); err != nil { + if _, err := s.AddImage(context.Background(), baseTag.Name(), "", false, remoteOpts...); err != nil { t.Fatalf("AddImage: %v", err) }