github/hauler
1
0
Fork 0
mirror of https://github.com/hauler-dev/hauler.git synced 2026-08-18 11:56:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

added audit log functionality (#632)

Browse Source 
Signed-off-by: Zack Brady <zackbrady123@gmail.com>
This commit is contained in:
Zack Brady
2026-07-10 13:52:24 -04:00
committed by GitHub
parent 9d04bbdd64
commit b9069d57c6
34 changed files with 1300 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -20,3 +20,4 @@ vulncheck.out
trivy.out
CLAUDE.md
**/CLAUDE.*
.claude**
cmd/hauler/cli/cli.go
+26 -3
View File
@@ -2,6 +2,8 @@ package cli
import (
"context"
"fmt"
"os"
cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd"
"github.com/sirupsen/logrus"
@@ -15,14 +17,35 @@ func New(ctx context.Context, ro *flags.CliRootOpts) *cobra.Command {
cmd := &cobra.Command{
Use: "hauler",
Short: "Airgap Swiss Army Knife",
Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + "\n Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.",
Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + " | " + consts.HaulerLogLevel + " | " + consts.HaulerAuditLevel + "\n Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
// check for log level env variable or flag
if ro.LogLevel == "" {
ro.LogLevel = os.Getenv(consts.HaulerLogLevel)
}
// default to info log level
if ro.LogLevel == "" {
ro.LogLevel = "info"
}
// check for audit level env variable or flag
if ro.AuditLevel == "" {
ro.AuditLevel = os.Getenv(consts.HaulerAuditLevel)
}
// default to standard audit level
if ro.AuditLevel == "" {
ro.AuditLevel = "standard"
}
switch ro.AuditLevel {
case "none", "standard", "verbose":
default:
return fmt.Errorf("invalid --audit-level %q: must be one of none, standard, verbose", ro.AuditLevel)
}
l := log.FromContext(ctx)
l.SetLevel(ro.LogLevel)
l.Debugf("running cli command [%s]", cmd.CommandPath())
// Suppress WARN-level messages from containerd and other
// libraries that use the global logrus logger.
if ro.LogLevel == "debug" {
logrus.SetLevel(logrus.DebugLevel)
} else {
cmd/hauler/cli/store.go
+14 -14
View File
@@ -50,7 +50,7 @@ func addStoreExtract(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Com
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -103,7 +103,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
return store.SyncCmd(ctx, o, nil, rso, ro)
}
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -126,7 +126,7 @@ func addStoreLoad(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -165,7 +165,7 @@ func addStoreServeRegistry(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cob
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -188,7 +188,7 @@ func addStoreServeFiles(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -212,7 +212,7 @@ func addStoreSave(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -239,7 +239,7 @@ func addStoreInfo(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -270,7 +270,7 @@ func addStoreCopy(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -319,12 +319,12 @@ func addStoreAddFile(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Com
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
return store.AddFileCmd(ctx, o, s, args[0])
return store.AddFileCmd(ctx, o, s, args[0], ro)
},
}
o.AddFlags(cmd)
@@ -363,7 +363,7 @@ func addStoreAddImage(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -406,7 +406,7 @@ func addStoreAddChart(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
s, err := o.Store(ctx, ro)
if err != nil {
return err
}
@@ -448,12 +448,12 @@ func addStoreRemove(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comm
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := rso.Store(ctx)
s, err := rso.Store(ctx, ro)
if err != nil {
return err
}
return store.RemoveCmd(ctx, o, s, args[0])
return store.RemoveCmd(ctx, o, s, args[0], ro, rso)
},
}
o.AddFlags(cmd)
cmd/hauler/cli/store/add.go
+167 -10
View File
@@ -23,6 +23,7 @@ import (
"hauler.dev/go/hauler/v2/internal/flags"
v1 "hauler.dev/go/hauler/v2/pkg/apis/hauler.cattle.io/v1"
"hauler.dev/go/hauler/v2/pkg/artifacts/file"
"hauler.dev/go/hauler/v2/pkg/audit"
"hauler.dev/go/hauler/v2/pkg/consts"
"hauler.dev/go/hauler/v2/pkg/content/chart"
"hauler.dev/go/hauler/v2/pkg/cosign"
@@ -33,17 +34,17 @@ import (
"hauler.dev/go/hauler/v2/pkg/store"
)
func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string) error {
func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string, ro *flags.CliRootOpts) error {
cfg := v1.File{
Path: reference,
}
if len(o.Name) > 0 {
cfg.Name = o.Name
}
return storeFile(ctx, s, cfg)
return storeFile(ctx, s, cfg, ro, o.StoreRootOpts)
}
func storeFile(ctx context.Context, s *store.Layout, fi v1.File) error {
func storeFile(ctx context.Context, s *store.Layout, fi v1.File, ro *flags.CliRootOpts, rso *flags.StoreRootOpts) error {
l := log.FromContext(ctx)
copts := getter.ClientOptions{
@@ -57,11 +58,45 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1.File) error {
}
l.Infof("adding file [%s] to the store as [%s]", fi.Path, ref.Name())
_, err = s.AddArtifact(ctx, f, ref.Name())
desc, err := s.AddArtifact(ctx, f, ref.Name())
if err != nil {
return err
}
resolvedPath := fi.Path
if !strings.HasPrefix(fi.Path, "http://") && !strings.HasPrefix(fi.Path, "https://") {
if abs, err := filepath.Abs(fi.Path); err == nil {
resolvedPath = abs
}
}
if auditLevel(ro) != "none" {
e := audit.Entry{
StoreID: s.StoreID,
Store: s.Root,
Type: "file",
Command: "store add file",
Args: []string{audit.SanitizeURL(fi.Path)},
Reference: audit.SanitizeURL(resolvedPath),
PortableReference: audit.ShortFileRef(fi.Path),
Digest: desc.Digest.String(),
}
if auditLevel(ro) == "verbose" {
sys := audit.BuildSystem()
g := audit.BuildGlobal(ro, rso)
e.System = &sys
e.Global = &g
e.Flags = map[string]any{
"name": fi.Name,
}
}
if err := audit.Append(ro.HaulerDir, e); err != nil {
l.Warnf("failed to write audit entry: %v", err)
}
l.Debugf("generated audit id of [%s]", audit.ID())
} else {
l.Debugf("generated audit id of [none]")
}
l.Infof("successfully added file [%s]", ref.Name())
return nil
@@ -71,9 +106,18 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
l := log.FromContext(ctx)
cfg := v1.Image{
Name: reference,
Rewrite: o.Rewrite,
Local: o.Local,
Name: reference,
Key: o.Key,
Tlog: o.Tlog,
CertIdentity: o.CertIdentity,
CertIdentityRegexp: o.CertIdentityRegexp,
CertOidcIssuer: o.CertOidcIssuer,
CertOidcIssuerRegexp: o.CertOidcIssuerRegexp,
CertGithubWorkflowRepository: o.CertGithubWorkflowRepository,
Platform: o.Platform,
Rewrite: o.Rewrite,
ExcludeExtras: o.ExcludeExtras,
Local: o.Local,
}
if o.Local {
@@ -132,7 +176,8 @@ func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags.
return err
}
if err := s.AddLocalImage(ctx, r.Name()); err != nil {
localDigest, err := s.AddLocalImage(ctx, r.Name())
if err != nil {
if ro.IgnoreErrors {
l.Warnf("unable to add image [%s] from Docker daemon to store: %v... skipping...", r.Name(), err)
return nil
@@ -160,6 +205,35 @@ func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags.
}
}
if auditLevel(ro) != "none" {
e := audit.Entry{
StoreID: s.StoreID,
Store: s.Root,
Type: "image",
Command: "store add image",
Args: []string{i.Name},
Reference: r.Name(),
Digest: localDigest,
}
if auditLevel(ro) == "verbose" {
sys := audit.BuildSystem()
g := audit.BuildGlobal(ro, nil)
e.System = &sys
e.Global = &g
e.Flags = map[string]any{
"verified": false,
"local": true,
"rewrite": rewrite,
}
}
if err := audit.Append(ro.HaulerDir, e); err != nil {
l.Warnf("failed to write audit entry: %v", err)
}
l.Debugf("generated audit id of [%s]", audit.ID())
} else {
l.Debugf("generated audit id of [none]")
}
l.Infof("successfully added image [%s] from local Docker daemon", r.Name())
return nil
}
@@ -188,8 +262,11 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
}
// fetch image along with any associated signatures and attestations
var imageDigest string
err = retry.Operation(ctx, rso, ro, func() error {
return s.AddImage(ctx, r.Name(), platform, excludeExtras)
var addErr error
imageDigest, addErr = s.AddImage(ctx, r.Name(), platform, excludeExtras)
return addErr
})
if err != nil {
if ro.IgnoreErrors {
@@ -221,6 +298,44 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
}
}
verified := i.Key != "" || i.CertIdentity != "" || i.CertIdentityRegexp != ""
if auditLevel(ro) != "none" {
e := audit.Entry{
StoreID: s.StoreID,
Store: s.Root,
Type: "image",
Command: "store add image",
Args: []string{i.Name},
Reference: r.Name(),
Digest: imageDigest,
}
if auditLevel(ro) == "verbose" {
sys := audit.BuildSystem()
g := audit.BuildGlobal(ro, rso)
e.System = &sys
e.Global = &g
e.Flags = map[string]any{
"verified": verified,
"platform": platform,
"key": i.Key,
"use-tlog-verify": i.Tlog,
"certificate-identity": i.CertIdentity,
"certificate-identity-regexp": i.CertIdentityRegexp,
"certificate-oidc-issuer": i.CertOidcIssuer,
"certificate-oidc-issuer-regexp": i.CertOidcIssuerRegexp,
"certificate-github-workflow-repository": i.CertGithubWorkflowRepository,
"rewrite": rewrite,
"exclude-extras": excludeExtras,
}
}
if err := audit.Append(ro.HaulerDir, e); err != nil {
l.Warnf("failed to write audit entry: %v", err)
}
l.Debugf("generated audit id of [%s]", audit.ID())
} else {
l.Debugf("generated audit id of [none]")
}
l.Infof("successfully added image [%s]", r.Name())
return nil
}
@@ -446,13 +561,55 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags.
return err
}
if _, err := s.AddArtifact(ctx, chrt, ref.Name()); err != nil {
chartDesc, err := s.AddArtifact(ctx, chrt, ref.Name())
if err != nil {
return err
}
if err := s.OCI.SaveIndex(); err != nil {
return err
}
if auditLevel(ro) != "none" {
e := audit.Entry{
StoreID: s.StoreID,
Store: s.Root,
Type: "chart",
Command: "store add chart",
Args: []string{c.Name()},
Reference: c.Name() + ":" + c.Metadata.Version,
Digest: chartDesc.Digest.String(),
}
if auditLevel(ro) == "verbose" {
sys := audit.BuildSystem()
g := audit.BuildGlobal(ro, rso)
e.System = &sys
e.Global = &g
e.Flags = map[string]any{
"repo": audit.SanitizeURL(cfg.RepoURL),
"version": c.Metadata.Version,
"rewrite": rewrite,
"add-images": opts.AddImages,
"add-dependencies": opts.AddDependencies,
"exclude-extras": opts.ExcludeExtras,
"values": opts.ValuesFiles,
"platform": opts.Platform,
"registry": opts.Registry,
"kube-version": opts.KubeVersion,
"verify": opts.ChartOpts.Verify,
"insecure-skip-tls-verify": opts.ChartOpts.InsecureSkipTLSVerify,
"ca-file": opts.ChartOpts.CaFile,
"cert-file": opts.ChartOpts.CertFile,
"key-file": opts.ChartOpts.KeyFile,
}
}
if err := audit.Append(ro.HaulerDir, e); err != nil {
l.Warnf("failed to write audit entry: %v", err)
}
l.Debugf("generated audit id of [%s]", audit.ID())
} else {
l.Debugf("generated audit id of [none]")
}
l.Infof("%ssuccessfully added chart [%s:%s]", prefix, c.Name(), c.Metadata.Version)
tempOverride := rso.TempOverride
cmd/hauler/cli/store/add_test.go
+8 -8
View File
@@ -237,7 +237,7 @@ func TestRewriteReference(t *testing.T) {
seedImage(t, host, "src/repo", "v1", rOpts...)
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil {
if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -294,7 +294,7 @@ func TestRewriteReference(t *testing.T) {
if err := rewriteReference(ctx, s, oldRef, newRef, rawRewrite); err != nil {
t.Fatalf("rewriteReference: %v", err)
}
// library/ must be stripped; registry stays index.docker.io
// library/ must be stripped... registry stays index.docker.io
assertAnnotationsInStore(t, s, "nginx:v2", "index.docker.io/nginx:v2")
})
@@ -339,7 +339,7 @@ func TestRewriteReference(t *testing.T) {
seedImage(t, host, "src/repo", "v1", rOpts...)
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil {
if _, err := s.AddImage(ctx, host+"/src/repo:v1", "", false, rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -371,7 +371,7 @@ func TestStoreFile(t *testing.T) {
tmp.Close()
s := newTestStore(t)
if err := storeFile(ctx, s, v1.File{Path: tmp.Name()}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: tmp.Name()}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
assertArtifactInStore(t, s, filepath.Base(tmp.Name()))
@@ -380,7 +380,7 @@ func TestStoreFile(t *testing.T) {
t.Run("HTTP URL stored under basename", func(t *testing.T) {
url := seedFileInHTTPServer(t, "script.sh", "#!/bin/sh\necho ok")
s := newTestStore(t)
if err := storeFile(ctx, s, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
assertArtifactInStore(t, s, "script.sh")
@@ -394,7 +394,7 @@ func TestStoreFile(t *testing.T) {
tmp.Close()
s := newTestStore(t)
if err := storeFile(ctx, s, v1.File{Path: tmp.Name(), Name: "custom.sh"}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: tmp.Name(), Name: "custom.sh"}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
assertArtifactInStore(t, s, "custom.sh")
@@ -402,7 +402,7 @@ func TestStoreFile(t *testing.T) {
t.Run("nonexistent local path returns error", func(t *testing.T) {
s := newTestStore(t)
err := storeFile(ctx, s, v1.File{Path: "/nonexistent/path/missing-file.txt"})
err := storeFile(ctx, s, v1.File{Path: "/nonexistent/path/missing-file.txt"}, defaultCliOpts(), defaultRootOpts(s.Root))
if err == nil {
t.Fatal("expected error for nonexistent path, got nil")
}
@@ -421,7 +421,7 @@ func TestAddFileCmd(t *testing.T) {
tmp.Close()
o := &flags.AddFileOpts{Name: "renamed.txt"}
if err := AddFileCmd(ctx, o, s, tmp.Name()); err != nil {
if err := AddFileCmd(ctx, o, s, tmp.Name(), defaultCliOpts()); err != nil {
t.Fatalf("AddFileCmd: %v", err)
}
assertArtifactInStore(t, s, "renamed.txt")
cmd/hauler/cli/store/audit.go
+14
View File
@@ -0,0 +1,14 @@
package store
import "hauler.dev/go/hauler/v2/internal/flags"
// auditLevel returns the resolved audit level (none, standard, verbose)
func auditLevel(ro *flags.CliRootOpts) string {
if ro == nil {
return "none"
}
if ro.AuditLevel == "" {
return "standard"
}
return ro.AuditLevel
}
cmd/hauler/cli/store/copy.go
+1 -1
View File
@@ -237,7 +237,7 @@ func CopyCmd(ctx context.Context, o *flags.CopyOpts, s *store.Layout, targetRef
// OCI 1.1 referrer (cosign v3 new-bundle-format): push by manifest digest so
// the target registry wires it up via the OCI Referrers API (subject field).
// For registries that don't support the Referrers API natively, the manifest
// is still pushed intact; the subject linkage depends on registry support.
// is still pushed intact... the subject linkage depends on registry support.
repo := baseRef
if colon := strings.LastIndex(baseRef, ":"); colon != -1 {
repo = baseRef[:colon]
cmd/hauler/cli/store/copy_test.go
+4 -4
View File
@@ -176,7 +176,7 @@ func TestCopyCmd_Registry_SigTagDerivation(t *testing.T) {
// AddImage discovers and stores the .sig/.att/.sbom tags automatically.
s := newTestStore(t)
if err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false); err != nil {
if _, err := s.AddImage(ctx, srcHost+"/test/signed:v1", "", false); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -257,7 +257,7 @@ func TestCopyCmd_Registry_InvalidFilenameSkipTest(t *testing.T) {
if err := os.WriteFile(p, []byte(pf.content), 0644); err != nil {
t.Fatalf("WriteFile %s: %v", pf.name, err)
}
if err := storeFile(ctx, s, v1.File{Path: p}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: p}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile %s: %v", pf.name, err)
}
}
@@ -304,7 +304,7 @@ func TestCopyCmd_Dir_Files(t *testing.T) {
url := seedFileInHTTPServer(t, "data.txt", content)
s := newTestStore(t)
if err := storeFile(ctx, s, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
@@ -395,6 +395,6 @@ func TestCopyCmd_Dir_Charts(t *testing.T) {
for i, e := range entries {
names[i] = e.Name()
}
t.Errorf("no .tgz found in destDir after chart copy; found: %v", names)
t.Errorf("no .tgz found in destDir after chart copy... found: %v", names)
}
}
cmd/hauler/cli/store/extract.go
+2 -2
View File
@@ -23,7 +23,7 @@ func isIndexMediaType(mt string) bool {
// firstLeafManifest walks a (potentially nested) OCI index and returns the
// decoded manifest of the first non-index child. It prefers non-index children
// at each level; if all children are indexes it descends into the first one.
// at each level... if all children are indexes it descends into the first one.
// Returns an error if any nested index or manifest cannot be decoded.
func firstLeafManifest(ctx context.Context, s *store.Layout, idx ocispec.Index) (ocispec.Manifest, error) {
for {
@@ -31,7 +31,7 @@ func firstLeafManifest(ctx context.Context, s *store.Layout, idx ocispec.Index)
return ocispec.Manifest{}, fmt.Errorf("image index has no child manifests")
}
// Prefer the first non-index child; fall back to the first child (an index) if all are indexes.
// Prefer the first non-index child... fall back to the first child (an index) if all are indexes.
desc := idx.Manifests[0]
for _, d := range idx.Manifests {
if !isIndexMediaType(d.MediaType) {
cmd/hauler/cli/store/extract_test.go
+10 -10
View File
@@ -31,7 +31,7 @@ func TestExtractCmd_File(t *testing.T) {
fileContent := "hello extract test"
url := seedFileInHTTPServer(t, "extract-me.txt", fileContent)
if err := storeFile(ctx, s, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
@@ -167,7 +167,7 @@ func TestExtractCmd_OciArtifactKindImage(t *testing.T) {
// Pull into a fresh store — AddImage sets kind=KindAnnotationImage on all manifests.
s := newTestStore(t)
if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -249,7 +249,7 @@ func TestExtractCmd_OciImageIndex_NoBinFiles(t *testing.T) {
}
s := newTestStore(t)
if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -288,7 +288,7 @@ func TestExtractCmd_OciImageIndex_NoBinFiles(t *testing.T) {
}
}
if !found {
t.Errorf("expected binary %q not found; got: %v", want, names)
t.Errorf("expected binary %q not found... got: %v", want, names)
}
}
}
@@ -359,7 +359,7 @@ func TestExtractCmd_NestedImageIndex_NoBinFiles(t *testing.T) {
}
s := newTestStore(t)
if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -398,7 +398,7 @@ func TestExtractCmd_NestedImageIndex_NoBinFiles(t *testing.T) {
}
}
if !found {
t.Errorf("expected binary %q not found; got: %v", want, names)
t.Errorf("expected binary %q not found... got: %v", want, names)
}
}
}
@@ -427,7 +427,7 @@ func TestExtractCmd_ContainerImage_Skipped(t *testing.T) {
}
s := newTestStore(t)
if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -497,7 +497,7 @@ func TestExtractCmd_ContainerImageIndex_Skipped(t *testing.T) {
}
s := newTestStore(t)
if err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
if _, err := s.AddImage(ctx, ref, "", false, rOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -533,7 +533,7 @@ func TestExtractCmd_SubstringMatch(t *testing.T) {
fileContent := "substring match content"
url := seedFileInHTTPServer(t, "extract-sub.txt", fileContent)
if err := storeFile(ctx, s, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
@@ -581,7 +581,7 @@ func TestExtractCmd_CosignArtifactsProduceNoContainerImageWarning(t *testing.T)
// Seed a real file artifact so ExtractCmd finds something to extract.
fileContent := "cosign-filter test file content"
url := seedFileInHTTPServer(t, "sigtest.txt", fileContent)
if err := storeFile(ctx, s, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
cmd/hauler/cli/store/info.go
+28 -22
View File
@@ -18,6 +18,12 @@ import (
"hauler.dev/go/hauler/v2/pkg/store"
)
type infoOutput struct {
StorePath string `json:"store-path"`
StoreID string `json:"store-id"`
Artifacts []item `json:"artifacts"`
}
func InfoCmd(ctx context.Context, o *flags.InfoOpts, s *store.Layout) error {
var items []item
if err := s.Walk(func(ref string, desc ocispec.Descriptor) error {
@@ -124,13 +130,20 @@ func InfoCmd(ctx context.Context, o *flags.InfoOpts, s *store.Layout) error {
// sort items by ref and arch
sort.Sort(byReferenceAndArch(items))
var msg string
switch o.OutputFormat {
case "json":
msg = buildJson(items...)
fmt.Println(msg)
out := infoOutput{
StorePath: s.Root,
StoreID: s.StoreID,
Artifacts: items,
}
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
return err
}
fmt.Println(string(data))
default:
if err := buildTable(o.ShowDigests, items...); err != nil {
if err := buildTable(s.Root, s.StoreID, o.ShowDigests, items...); err != nil {
return err
}
}
@@ -161,10 +174,11 @@ func buildListRepos(items ...item) {
}
}
func buildTable(showDigests bool, items ...item) error {
func buildTable(storePath, storeID string, showDigests bool, items ...item) error {
table := tablewriter.NewTable(os.Stdout)
table.Configure(func(cfg *tablewriter.Config) {
cfg.Header.Alignment.Global = tw.AlignLeft
cfg.Footer.Alignment.PerColumn = []tw.Align{tw.AlignLeft}
cfg.Row.Merging.Mode = tw.MergeVertical
cfg.Row.Merging.ByColumnIndex = tw.NewBoolMapper(0)
})
@@ -214,11 +228,11 @@ func buildTable(showDigests bool, items ...item) error {
}
}
// align total column based on digest visibility
footerLabel := "store-path: " + storePath + "\nstore-id: " + storeID
if showDigests {
table.Footer("", "", "", "", "Total", byteCountSI(totalSize))
table.Footer(footerLabel, "", "", "", "Total", byteCountSI(totalSize))
} else {
table.Footer("", "", "", "Total", byteCountSI(totalSize))
table.Footer(footerLabel, "", "", "Total", byteCountSI(totalSize))
}
return table.Render()
@@ -237,21 +251,13 @@ func truncateReference(ref string) string {
return ref
}
func buildJson(item ...item) string {
data, err := json.MarshalIndent(item, "", " ")
if err != nil {
return ""
}
return string(data)
}
type item struct {
Reference string
Type string
Platform string
Digest string
Layers int
Size int64
Reference string `json:"reference"`
Type string `json:"type"`
Platform string `json:"platform"`
Digest string `json:"digest,omitempty"`
Layers int `json:"layers"`
Size int64 `json:"size"`
}
type byReferenceAndArch []item
cmd/hauler/cli/store/info_test.go
+26 -18
View File
@@ -53,37 +53,45 @@ func TestTruncateReference(t *testing.T) {
}
}
func TestBuildJson(t *testing.T) {
func TestInfoOutputJSON(t *testing.T) {
items := []item{
{Reference: "myrepo/myimage:v1", Type: "image", Platform: "linux/amd64", Size: 1024, Layers: 2},
{Reference: "myrepo/mychart:v1", Type: "chart", Platform: "-", Size: 512, Layers: 1},
}
out := buildJson(items...)
if out == "" {
t.Fatal("buildJson returned empty string")
out := infoOutput{
StorePath: "/tmp/test-store",
StoreID: "test-store-id",
Artifacts: items,
}
var got []item
if err := json.Unmarshal([]byte(out), &got); err != nil {
t.Fatalf("buildJson output is not valid JSON: %v\noutput: %s", err, out)
data, err := json.Marshal(out)
if err != nil {
t.Fatalf("json.Marshal infoOutput: %v", err)
}
if len(got) != len(items) {
t.Fatalf("buildJson: got %d items, want %d", len(got), len(items))
var got infoOutput
if err := json.Unmarshal(data, &got); err != nil {
t.Fatalf("json.Unmarshal infoOutput: %v\ndata: %s", err, data)
}
if got.StorePath != out.StorePath {
t.Errorf("StorePath = %q, want %q", got.StorePath, out.StorePath)
}
if len(got.Artifacts) != len(items) {
t.Fatalf("Artifacts len = %d, want %d", len(got.Artifacts), len(items))
}
for i, want := range items {
if got[i].Reference != want.Reference {
t.Errorf("item[%d].Reference = %q, want %q", i, got[i].Reference, want.Reference)
if got.Artifacts[i].Reference != want.Reference {
t.Errorf("Artifacts[%d].Reference = %q, want %q", i, got.Artifacts[i].Reference, want.Reference)
}
if got[i].Type != want.Type {
t.Errorf("item[%d].Type = %q, want %q", i, got[i].Type, want.Type)
if got.Artifacts[i].Type != want.Type {
t.Errorf("Artifacts[%d].Type = %q, want %q", i, got.Artifacts[i].Type, want.Type)
}
if got[i].Size != want.Size {
t.Errorf("item[%d].Size = %d, want %d", i, got[i].Size, want.Size)
if got.Artifacts[i].Size != want.Size {
t.Errorf("Artifacts[%d].Size = %d, want %d", i, got.Artifacts[i].Size, want.Size)
}
}
}
func TestNewItem(t *testing.T) {
// newItem uses s only for its signature; it does not dereference s in practice.
// newItem uses s only for its signature... it does not dereference s in practice.
// We pass nil to keep tests dependency-free.
const validRef = "myrepo/myimage:latest"
@@ -199,7 +207,7 @@ func TestInfoCmd(t *testing.T) {
t.Fatalf("write tmpFile: %v", err)
}
fi := v1.File{Path: tmpFile}
if err := storeFile(ctx, s, fi); err != nil {
if err := storeFile(ctx, s, fi, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
@@ -224,7 +232,7 @@ func TestInfoCmd(t *testing.T) {
})
t.Run("TypeFilter:image json", func(t *testing.T) {
// Store has only a file artifact; image filter returns no items (no error).
// Store has only a file artifact... image filter returns no items (no error).
if err := InfoCmd(ctx, baseOpts("image", "json"), s); err != nil {
t.Errorf("InfoCmd(image, json): %v", err)
}
cmd/hauler/cli/store/lifecycle_test.go
+4 -4
View File
@@ -31,7 +31,7 @@ func TestLifecycle_FileArtifact_AddSaveLoadCopy(t *testing.T) {
// Step 2: storeFile into store A.
storeA := newTestStore(t)
if err := storeFile(ctx, storeA, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, storeA, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
assertArtifactInStore(t, storeA, "lifecycle.txt")
@@ -309,10 +309,10 @@ func TestLifecycle_Remove_ThenSave(t *testing.T) {
url2 := seedFileInHTTPServer(t, "remove-me.txt", "content to remove")
storeA := newTestStore(t)
if err := storeFile(ctx, storeA, v1.File{Path: url1}); err != nil {
if err := storeFile(ctx, storeA, v1.File{Path: url1}, defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil {
t.Fatalf("storeFile keep-me: %v", err)
}
if err := storeFile(ctx, storeA, v1.File{Path: url2}); err != nil {
if err := storeFile(ctx, storeA, v1.File{Path: url2}, defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil {
t.Fatalf("storeFile remove-me: %v", err)
}
@@ -321,7 +321,7 @@ func TestLifecycle_Remove_ThenSave(t *testing.T) {
}
// Step 2: RemoveCmd(Force:true) on the "remove-me" artifact.
if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, storeA, "remove-me"); err != nil {
if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, storeA, "remove-me", defaultCliOpts(), defaultRootOpts(storeA.Root)); err != nil {
t.Fatalf("RemoveCmd: %v", err)
}
cmd/hauler/cli/store/load_test.go
+2 -2
View File
@@ -367,11 +367,11 @@ func TestUnarchiveLayoutTo_LegacyKindMigration(t *testing.T) {
if err := s.OCI.Walk(func(_ string, desc ocispec.Descriptor) error {
kind := desc.Annotations[consts.KindAnnotationName]
if strings.HasPrefix(kind, legacyPrefix) {
t.Errorf("descriptor %s still has legacy kind %q; expected dev.hauler prefix",
t.Errorf("descriptor %s still has legacy kind %q... expected dev.hauler prefix",
desc.Digest, kind)
}
if !strings.HasPrefix(kind, newPrefix) {
t.Errorf("descriptor %s has unexpected kind %q; expected dev.hauler prefix",
t.Errorf("descriptor %s has unexpected kind %q... expected dev.hauler prefix",
desc.Digest, kind)
}
return nil
cmd/hauler/cli/store/remove.go
+73 -1
View File
@@ -3,6 +3,7 @@ package store
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -12,10 +13,50 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"hauler.dev/go/hauler/v2/internal/flags"
"hauler.dev/go/hauler/v2/pkg/audit"
"hauler.dev/go/hauler/v2/pkg/consts"
"hauler.dev/go/hauler/v2/pkg/log"
"hauler.dev/go/hauler/v2/pkg/store"
)
// artifactType derives a human-readable content type for an artifact the same way `store info`
// does: from the manifest's config media type, since AddArtifact stores every non-image-command
// artifact (files, charts) under the same "kind" annotation and can't distinguish them
func artifactType(ctx context.Context, s *store.Layout, desc ocispec.Descriptor) string {
switch {
case desc.Annotations[consts.KindAnnotationName] == consts.KindAnnotationSigs:
return "sigs"
case desc.Annotations[consts.KindAnnotationName] == consts.KindAnnotationAtts:
return "atts"
case desc.Annotations[consts.KindAnnotationName] == consts.KindAnnotationSboms:
return "sbom"
case strings.HasPrefix(desc.Annotations[consts.KindAnnotationName], consts.KindAnnotationReferrers):
return "referrer"
case desc.MediaType == consts.OCIImageIndexSchema, desc.MediaType == consts.DockerManifestListSchema2:
return "image"
}
rc, err := s.Fetch(ctx, desc)
if err != nil {
return "image"
}
defer rc.Close()
var m ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return "image"
}
switch m.Config.MediaType {
case consts.ChartConfigMediaType:
return "chart"
case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType, consts.FileDirectoryConfigMediaType:
return "file"
default:
return "image"
}
}
func formatReference(ref string) string {
tagIdx := strings.LastIndex(ref, ":")
if tagIdx == -1 {
@@ -39,7 +80,7 @@ func formatReference(ref string) string {
return fmt.Sprintf("%s [%s]", base, suffix)
}
func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref string) error {
func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref string, ro *flags.CliRootOpts, rso *flags.StoreRootOpts) error {
l := log.FromContext(ctx)
// collect matching artifacts
@@ -106,6 +147,37 @@ func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref st
return fmt.Errorf("failed to remove artifact [%s]: %w", formatReference(m.reference), err)
}
if auditLevel(ro) != "none" {
cleanRef := m.desc.Annotations[consts.ContainerdImageNameKey]
if cleanRef == "" {
cleanRef = m.desc.Annotations[ocispec.AnnotationRefName]
}
e := audit.Entry{
StoreID: s.StoreID,
Store: s.Root,
Type: artifactType(ctx, s, m.desc),
Command: "store remove",
Args: []string{ref},
Reference: cleanRef,
Digest: m.desc.Digest.String(),
}
if auditLevel(ro) == "verbose" {
sys := audit.BuildSystem()
g := audit.BuildGlobal(ro, rso)
e.System = &sys
e.Global = &g
e.Flags = map[string]any{
"force": o.Force,
}
}
if err := audit.Append(ro.HaulerDir, e); err != nil {
l.Warnf("failed to write audit entry: %v", err)
}
l.Debugf("generated audit id of [%s]", audit.ID())
} else {
l.Debugf("generated audit id of [none]")
}
l.Infof("successfully removed [%s] of type [%s] with digest [%s]", formatReference(m.reference), m.desc.MediaType, m.desc.Digest.String())
}
cmd/hauler/cli/store/remove_test.go
+6 -6
View File
@@ -81,7 +81,7 @@ func TestRemoveCmd_Force(t *testing.T) {
s := newTestStore(t)
url := seedFileInHTTPServer(t, "removeme.txt", "file-to-remove")
if err := storeFile(ctx, s, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
@@ -103,7 +103,7 @@ func TestRemoveCmd_Force(t *testing.T) {
t.Fatal("could not find stored artifact reference containing 'removeme'")
}
if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "removeme"); err != nil {
if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "removeme", defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("RemoveCmd: %v", err)
}
@@ -116,7 +116,7 @@ func TestRemoveCmd_NotFound(t *testing.T) {
ctx := newTestContext(t)
s := newTestStore(t)
err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "nonexistent-ref")
err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "nonexistent-ref", defaultCliOpts(), defaultRootOpts(s.Root))
if err == nil {
t.Fatal("expected error for non-existent ref, got nil")
}
@@ -133,10 +133,10 @@ func TestRemoveCmd_Force_MultipleMatches(t *testing.T) {
url1 := seedFileInHTTPServer(t, "testfile-alpha.txt", "content-alpha")
url2 := seedFileInHTTPServer(t, "testfile-beta.txt", "content-beta")
if err := storeFile(ctx, s, v1.File{Path: url1}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url1}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile alpha: %v", err)
}
if err := storeFile(ctx, s, v1.File{Path: url2}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url2}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile beta: %v", err)
}
@@ -145,7 +145,7 @@ func TestRemoveCmd_Force_MultipleMatches(t *testing.T) {
}
// Remove using a substring that matches both.
if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "testfile"); err != nil {
if err := RemoveCmd(ctx, &flags.RemoveOpts{Force: true}, s, "testfile", defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("RemoveCmd: %v", err)
}
cmd/hauler/cli/store/save_test.go
+8 -8
View File
@@ -59,7 +59,7 @@ func TestWriteExportsManifest(t *testing.T) {
seedIndex(t, host, "test/multiarch", "v1", rOpts...)
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false); err != nil {
if _, err := s.AddImage(ctx, host+"/test/multiarch:v1", "", false); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -78,7 +78,7 @@ func TestWriteExportsManifest(t *testing.T) {
seedIndex(t, host, "test/multiarch", "v2", rOpts...)
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false); err != nil {
if _, err := s.AddImage(ctx, host+"/test/multiarch:v2", "", false); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -126,7 +126,7 @@ func TestWriteExportsManifest_DigestOnlyImageHasRepoTag(t *testing.T) {
// Add the image BY DIGEST
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false); err != nil {
if _, err := s.AddImage(ctx, host+"/test/digestonly@"+hash.String(), "", false); err != nil {
t.Fatalf("AddImage by digest: %v", err)
}
@@ -149,7 +149,7 @@ func TestWriteExportsManifest_SkipsNonImages(t *testing.T) {
url := seedFileInHTTPServer(t, "skip.sh", "#!/bin/sh\necho skip")
s := newTestStore(t)
if err := storeFile(ctx, s, v1.File{Path: url}); err != nil {
if err := storeFile(ctx, s, v1.File{Path: url}, defaultCliOpts(), defaultRootOpts(s.Root)); err != nil {
t.Fatalf("storeFile: %v", err)
}
@@ -174,7 +174,7 @@ func TestSaveCmd(t *testing.T) {
seedImage(t, host, "test/save", "v1")
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/test/save:v1", "", false); err != nil {
if _, err := s.AddImage(ctx, host+"/test/save:v1", "", false); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -207,7 +207,7 @@ func TestSaveCmd_ContainerdCompatibility(t *testing.T) {
seedImage(t, host, "test/containerd-compat", "v1")
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false); err != nil {
if _, err := s.AddImage(ctx, host+"/test/containerd-compat:v1", "", false); err != nil {
t.Fatalf("AddImage: %v", err)
}
@@ -236,7 +236,7 @@ func TestSaveCmd_EmptyStore(t *testing.T) {
s := newTestStore(t)
// SaveCmd uses layout.FromPath which stats index.json — it must exist on
// disk. A fresh store holds the index only in memory; SaveIndex flushes it.
// disk. A fresh store holds the index only in memory... SaveIndex flushes it.
if err := s.SaveIndex(); err != nil {
t.Fatalf("SaveIndex: %v", err)
}
@@ -306,7 +306,7 @@ func TestSaveCmd_ChunkSize(t *testing.T) {
seedImage(t, host, "test/chunksave", "v1")
s := newTestStore(t)
if err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false); err != nil {
if _, err := s.AddImage(ctx, host+"/test/chunksave:v1", "", false); err != nil {
t.Fatalf("AddImage: %v", err)
}
cmd/hauler/cli/store/sync.go
+1 -1
View File
@@ -296,7 +296,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
return err
}
for _, f := range cfg.Spec.Files {
if err := storeFile(ctx, s, f); err != nil {
if err := storeFile(ctx, s, f, ro, rso); err != nil {
return err
}
}
cmd/hauler/cli/store/sync_test.go
+1 -1
View File
@@ -110,7 +110,7 @@ func TestProcessContent_Images_v1(t *testing.T) {
s := newTestStore(t)
host, _ := newLocalhostRegistry(t)
seedImage(t, host, "myorg/myimage", "v1") // transport not needed; AddImage reads via localhost scheme
seedImage(t, host, "myorg/myimage", "v1") // transport not needed... AddImage reads via localhost scheme
manifest := fmt.Sprintf(`apiVersion: content.hauler.cattle.io/v1
kind: Images
cmd/hauler/cli/store/testhelpers_test.go
+5 -2
View File
@@ -58,7 +58,7 @@ func newTestRegistry(t *testing.T) (host string, remoteOpts []remote.Option) {
}
// seedImage pushes a random single-platform image to the test registry.
// repo is a bare path like "myorg/myimage"; tag is the image tag string.
// repo is a bare path like "myorg/myimage"... tag is the image tag string.
// Pass the remoteOpts from newTestRegistry so writes use the correct transport.
func seedImage(t *testing.T, host, repo, tag string, opts ...remote.Option) gcrv1.Image {
t.Helper()
@@ -138,11 +138,14 @@ func defaultRootOpts(storeDir string) *flags.StoreRootOpts {
}
}
// defaultCliOpts returns CliRootOpts with error-level logging and IgnoreErrors=false.
// defaultCliOpts returns CliRootOpts with log level error, audit level none, and ignore errors false.
// Audit is disabled here rather than pointed at a temp HaulerDir so tests don't write to the
// developer/CI user's real $HOME/.hauler/audit.log.
func defaultCliOpts() *flags.CliRootOpts {
return &flags.CliRootOpts{
IgnoreErrors: false,
LogLevel: "error",
AuditLevel: "none",
}
}
go.mod
+1 -1
View File
@@ -9,6 +9,7 @@ require (
github.com/distribution/distribution/v3 v3.1.1
github.com/distribution/reference v0.6.0
github.com/google/go-containerregistry v0.21.7
github.com/google/uuid v1.6.0
github.com/gorilla/handlers v1.5.2
github.com/gorilla/mux v1.8.1
github.com/mholt/archives v0.1.5
@@ -184,7 +185,6 @@ require (
github.com/google/go-github/v88 v88.0.0 // indirect
github.com/google/go-querystring v1.2.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
internal/flags/cli.go
+3 -1
View File
@@ -6,12 +6,14 @@ type CliRootOpts struct {
LogLevel string
HaulerDir string
IgnoreErrors bool
AuditLevel string
}
func AddRootFlags(cmd *cobra.Command, ro *CliRootOpts) {
pf := cmd.PersistentFlags()
pf.StringVarP(&ro.LogLevel, "log-level", "l", "info", "Set the logging level (i.e. info, debug, warn)")
pf.StringVarP(&ro.LogLevel, "log-level", "l", "", "Set the logging level (i.e. info, debug, warn) (defaults info)")
pf.StringVarP(&ro.HaulerDir, "haulerdir", "d", "", "Set the location of the hauler directory (default $HOME/.hauler)")
pf.BoolVar(&ro.IgnoreErrors, "ignore-errors", false, "Ignore/Bypass errors (i.e. warn on error) (defaults false)")
pf.StringVar(&ro.AuditLevel, "audit-level", "", "Set the audit logging level (none, standard, verbose) (defaults standard)")
}
internal/flags/store.go
+39 -2
View File
@@ -3,8 +3,10 @@ package flags
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/spf13/cobra"
"hauler.dev/go/hauler/v2/pkg/consts"
@@ -12,6 +14,10 @@ import (
"hauler.dev/go/hauler/v2/pkg/store"
)
// storeIDPattern matches the full shape of a StoreID as generated by uuid.New()... e.g. "ec520cf6-e01b-4d6f-93ea-6588de0d5159"
// Short prefixes (e.g. "ec520cf6") are resolved by ResolveStoreID itself and never reach this pattern unless the lookup fails
var storeIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
type StoreRootOpts struct {
StoreDir string
Retries int
@@ -25,9 +31,11 @@ func (o *StoreRootOpts) AddFlags(cmd *cobra.Command) {
pf.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directory determined by the OS")
}
func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {
func (o *StoreRootOpts) Store(ctx context.Context, ro *CliRootOpts) (*store.Layout, error) {
l := log.FromContext(ctx)
haulerDir := resolveHaulerDir(ro)
storeDir := o.StoreDir
if storeDir == "" {
@@ -38,6 +46,22 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {
storeDir = consts.DefaultStoreName
}
// If storeDir doesn't exist as a path, see if it's a StoreID registered from a previous run elsewhere
if _, err := os.Stat(storeDir); errors.Is(err, os.ErrNotExist) {
id, resolved, rerr := store.ResolveStoreID(haulerDir, storeDir)
switch {
case rerr == nil:
if !store.MatchesStoreID(resolved, id) {
return nil, fmt.Errorf("store id %q was last seen at %s, but that path no longer contains that store", storeDir, resolved)
}
l.Debugf("resolved store id [%s] to [%s]", storeDir, resolved)
storeDir = resolved
case storeIDPattern.MatchString(storeDir):
// looks like an ID, not a directory name to create
return nil, fmt.Errorf("no store found matching id %q (for directories, use the absolute path)", storeDir)
}
}
abs, err := filepath.Abs(storeDir)
if err != nil {
return nil, err
@@ -55,9 +79,22 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {
return nil, err
}
s, err := store.NewLayout(abs)
s, err := store.NewLayout(abs, store.WithHaulerDir(haulerDir))
if err != nil {
return nil, err
}
l.Debugf("generated store id of [%s]", s.StoreID)
return s, nil
}
// resolveHaulerDir mirrors other variable detection, but duplicated to avoid an import cycle
func resolveHaulerDir(ro *CliRootOpts) string {
if ro != nil && ro.HaulerDir != "" {
return ro.HaulerDir
}
if d := os.Getenv(consts.HaulerDir); d != "" {
return d
}
home, _ := os.UserHomeDir()
return filepath.Join(home, consts.DefaultHaulerDirName)
}
internal/mapper/filestore.go
+1 -1
View File
@@ -72,7 +72,7 @@ func (s *pusher) Push(ctx context.Context, desc ocispec.Descriptor) (ccontent.Wr
// Get the filename from the mapper function.
// An empty filename means the mapper explicitly declined this descriptor (e.g. a
// config blob that has no title annotation); treat it the same as no mapper.
// config blob that has no title annotation... treat it the same as no mapper.
filename, err := mapperFn(desc)
if err != nil {
return nil, err
internal/mapper/mappers.go
+1 -1
View File
@@ -131,7 +131,7 @@ func Files() map[string]Fn {
m["application/vnd.oci.image.layer.v1.tar"] = fileMapperFn // And the tar variant
// Catch-all for OCI artifacts that use custom layer media types (e.g. rke2-binary).
// Write the blob if it carries an AnnotationTitle; silently discard everything else
// Write the blob if it carries an AnnotationTitle... silently discard everything else
// (config blobs, metadata) by returning an empty filename.
m[DefaultCatchAll] = Fn(func(desc ocispec.Descriptor) (string, error) {
if title, ok := desc.Annotations[ocispec.AnnotationTitle]; ok {
pkg/audit/audit.go
+228
View File
@@ -0,0 +1,228 @@
package audit
import (
"encoding/json"
"fmt"
"net"
"net/url"
"os"
osuser "os/user"
"path"
"path/filepath"
"strings"
"time"
"github.com/google/uuid"
"hauler.dev/go/hauler/v2/internal/flags"
"hauler.dev/go/hauler/v2/pkg/consts"
)
var auditID string
// auditID is generated once per process to group all entries from a single invocation
func init() {
if id, err := uuid.NewV7(); err == nil {
auditID = id.String()
} else {
auditID = uuid.New().String()
}
}
// auditID returns the audit ID for the current process invocation
func ID() string { return auditID }
// SystemEntry captures OS level context and only records at verbose level to the global log
type SystemEntry struct {
User string `json:"user,omitempty"`
Hostname string `json:"hostname,omitempty"`
IPAddress string `json:"ip-address,omitempty"`
}
// GlobalEntry captures hauler flag values and only records at verbose audit level
type GlobalEntry struct {
HaulerDir string `json:"haulerdir,omitempty"`
IgnoreErrors bool `json:"ignore-errors,omitempty"`
LogLevel string `json:"log-level,omitempty"`
AuditLevel string `json:"audit-level,omitempty"`
Retries int `json:"retries,omitempty"`
StoreDir string `json:"store,omitempty"`
TempDir string `json:"tempdir,omitempty"`
Env map[string]string `json:"env,omitempty"`
}
// Entry records a single auditable operation on the store
type Entry struct {
AuditID string `json:"audit-id,omitempty"`
StoreID string `json:"store-id,omitempty"`
Timestamp string `json:"timestamp"`
Command string `json:"command"`
Args []string `json:"args,omitempty"`
Type string `json:"type,omitempty"`
Reference string `json:"reference,omitempty"`
Digest string `json:"digest,omitempty"`
Store string `json:"store,omitempty"`
System *SystemEntry `json:"system,omitempty"`
Global *GlobalEntry `json:"global,omitempty"`
Flags map[string]any `json:"flags,omitempty"`
// PortableReference replaces Reference in the store audit log
PortableReference string `json:"-"`
}
// portableEntry is the machine-agnostic subset written to the store audit log
type portableEntry struct {
AuditID string `json:"audit-id"`
StoreID string `json:"store-id,omitempty"`
Timestamp string `json:"timestamp"`
Command string `json:"command"`
Type string `json:"type,omitempty"`
Reference string `json:"reference,omitempty"`
Digest string `json:"digest,omitempty"`
}
// ShortFileRef returns a portable-safe short name for a local path or URL
func ShortFileRef(raw string) string {
if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
if u, err := url.Parse(raw); err == nil {
if base := path.Base(u.Path); base != "." && base != "/" {
return base
}
}
}
return filepath.Base(raw)
}
// SanitizeURL strips userinfo and the query string from an http(s) URL so
// embedded credentials (presigned signatures, tokens, API keys) never reach
// the audit log, at any audit level. Non-URL values (local paths, image or
// chart references) are returned unchanged.
func SanitizeURL(raw string) string {
if !strings.HasPrefix(raw, "http://") && !strings.HasPrefix(raw, "https://") {
return raw
}
u, err := url.Parse(raw)
if err != nil {
return raw
}
u.User = nil
u.RawQuery = ""
u.Fragment = ""
return u.String()
}
// BuildSystem returns OS level context for verbose audit entries
func BuildSystem() SystemEntry {
s := SystemEntry{}
if u, err := osuser.Current(); err == nil {
s.User = u.Username
}
s.Hostname, _ = os.Hostname()
if addrs, err := net.InterfaceAddrs(); err == nil {
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() != nil {
s.IPAddress = ipnet.IP.String()
break
}
}
}
return s
}
// BuildGlobal returns hauler flag values for verbose audit entries
func BuildGlobal(ro *flags.CliRootOpts, rso *flags.StoreRootOpts) GlobalEntry {
g := GlobalEntry{}
if ro != nil {
g.HaulerDir = resolveDir(ro.HaulerDir)
g.IgnoreErrors = ro.IgnoreErrors
g.LogLevel = ro.LogLevel
g.AuditLevel = ro.AuditLevel
}
if rso != nil {
g.Retries = rso.Retries
g.StoreDir = rso.StoreDir
g.TempDir = rso.TempOverride
}
env := map[string]string{}
for _, key := range []string{
consts.HaulerDir,
consts.HaulerTempDir,
consts.HaulerStoreDir,
consts.HaulerIgnoreErrors,
consts.HaulerLogLevel,
consts.HaulerAuditLevel,
} {
if v := os.Getenv(key); v != "" {
env[key] = v
}
}
if len(env) > 0 {
g.Env = env
}
return g
}
// Append records a full log entry to <haulerDir>/audit.log
// When e.Store is set... a portable subset to <storeDir>/audit.log
func Append(haulerDir string, e Entry) error {
e.AuditID = auditID
e.Timestamp = time.Now().UTC().Format(time.RFC3339)
// global write... full entry including system/global/flags
var globalErr error
if err := appendLine(resolveDir(haulerDir), e); err != nil {
globalErr = fmt.Errorf("audit: global write: %w", err)
}
// store write... portable subset only, attempted even if the global write above failed
if e.Store != "" {
reference := e.Reference
if e.PortableReference != "" {
reference = e.PortableReference
}
pe := portableEntry{
AuditID: e.AuditID,
StoreID: e.StoreID,
Timestamp: e.Timestamp,
Command: e.Command,
Type: e.Type,
Reference: reference,
Digest: e.Digest,
}
if err := appendLine(e.Store, pe); err != nil {
if globalErr != nil {
return fmt.Errorf("%v: audit: store write: %w", globalErr, err)
}
return fmt.Errorf("audit: store write: %w", err)
}
}
return globalErr
}
func appendLine(dir string, v any) error {
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("audit: ensure dir: %w", err)
}
data, err := json.Marshal(v)
if err != nil {
return fmt.Errorf("audit: marshal: %w", err)
}
f, err := os.OpenFile(filepath.Join(dir, "audit.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("audit: open log: %w", err)
}
defer f.Close()
_, err = fmt.Fprintf(f, "%s\n", data)
return err
}
func resolveDir(haulerDir string) string {
if haulerDir != "" {
return haulerDir
}
if d := os.Getenv(consts.HaulerDir); d != "" {
return d
}
home, _ := os.UserHomeDir()
return filepath.Join(home, consts.DefaultHaulerDirName)
}
pkg/audit/audit_test.go
+214
View File
@@ -0,0 +1,214 @@
package audit
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
func TestAppend(t *testing.T) {
dir := t.TempDir()
e := Entry{
Command: "store add image",
Args: []string{"busybox:latest"},
Flags: map[string]any{"platform": "linux/amd64"},
Store: filepath.Join(dir, "store"),
}
if err := Append(dir, e); err != nil {
t.Fatalf("Append: %v", err)
}
path := filepath.Join(dir, "audit.log")
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
var got Entry
if err := json.Unmarshal(data, &got); err != nil {
t.Fatalf("unmarshal audit line: %v\nraw: %s", err, data)
}
if got.Command != e.Command {
t.Errorf("Command = %q, want %q", got.Command, e.Command)
}
if len(got.Args) != 1 || got.Args[0] != "busybox:latest" {
t.Errorf("Args = %v, want [busybox:latest]", got.Args)
}
if got.Timestamp == "" {
t.Error("Timestamp should be set")
}
}
// TestAppend_PortableReferenceOverridesStoreCopy verifies PortableReference only affects the store audit log
func TestAppend_PortableReferenceOverridesStoreCopy(t *testing.T) {
dir := t.TempDir()
absPath := filepath.Join(string(filepath.Separator), "home", "example", "scripts", "install.sh")
typedPath := filepath.Join(".", "scripts", "install.sh")
e := Entry{
Command: "store add file",
Type: "file",
Args: []string{typedPath},
Reference: absPath,
PortableReference: typedPath,
Store: filepath.Join(dir, "store"),
}
if err := Append(dir, e); err != nil {
t.Fatalf("Append: %v", err)
}
// global log keeps the full path and the as-typed args
globalData, err := os.ReadFile(filepath.Join(dir, "audit.log"))
if err != nil {
t.Fatalf("ReadFile global audit.log: %v", err)
}
var globalGot Entry
if err := json.Unmarshal(globalData, &globalGot); err != nil {
t.Fatalf("unmarshal global audit line: %v\nraw: %s", err, globalData)
}
if globalGot.Reference != absPath {
t.Errorf("global Reference = %q, want %q", globalGot.Reference, absPath)
}
if len(globalGot.Args) != 1 || globalGot.Args[0] != typedPath {
t.Errorf("global Args = %v, want [%s]", globalGot.Args, typedPath)
}
// store log uses PortableReference instead, and carries no args at all
storeData, err := os.ReadFile(filepath.Join(dir, "store", "audit.log"))
if err != nil {
t.Fatalf("ReadFile store audit log: %v", err)
}
var got portableEntry
if err := json.Unmarshal(storeData, &got); err != nil {
t.Fatalf("unmarshal store audit line: %v\nraw: %s", err, storeData)
}
if got.Reference != typedPath {
t.Errorf("store Reference = %q, want %q", got.Reference, typedPath)
}
if strings.Contains(string(storeData), `"args"`) {
t.Errorf("store entry should not carry args: %s", storeData)
}
}
// TestAppend_StoreDefaultsToReference verifies the fallback when PortableReference is unset
func TestAppend_StoreDefaultsToReference(t *testing.T) {
dir := t.TempDir()
e := Entry{
Command: "store remove",
Type: "file",
Args: []string{"install.sh"},
Reference: "hauler/install.sh:latest",
Store: filepath.Join(dir, "store"),
}
if err := Append(dir, e); err != nil {
t.Fatalf("Append: %v", err)
}
storeData, err := os.ReadFile(filepath.Join(dir, "store", "audit.log"))
if err != nil {
t.Fatalf("ReadFile store audit log: %v", err)
}
var got portableEntry
if err := json.Unmarshal(storeData, &got); err != nil {
t.Fatalf("unmarshal store audit line: %v\nraw: %s", err, storeData)
}
if got.Reference != e.Reference {
t.Errorf("store Reference = %q, want %q (unmodified)", got.Reference, e.Reference)
}
}
func TestAppend_StoreWriteSucceedsWhenGlobalWriteFails(t *testing.T) {
dir := t.TempDir()
// occupy the path so appendLine's MkdirAll fails for the global write only
blockedHaulerDir := filepath.Join(dir, "blocked")
if err := os.WriteFile(blockedHaulerDir, []byte("not a dir"), 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
storeDir := filepath.Join(dir, "store")
e := Entry{
Command: "store add image",
Args: []string{"busybox:latest"},
Store: storeDir,
}
if err := Append(blockedHaulerDir, e); err == nil {
t.Fatal("Append: expected error from unwritable global haulerDir, got nil")
}
storeData, err := os.ReadFile(filepath.Join(storeDir, "audit.log"))
if err != nil {
t.Fatalf("ReadFile store audit log: %v", err)
}
var got portableEntry
if err := json.Unmarshal(storeData, &got); err != nil {
t.Fatalf("unmarshal store audit line: %v\nraw: %s", err, storeData)
}
if got.Command != e.Command {
t.Errorf("store command = %q, want = %q", got.Command, e.Command)
}
}
func TestAppend_MultipleEntries(t *testing.T) {
dir := t.TempDir()
for i := 0; i < 3; i++ {
if err := Append(dir, Entry{Command: "store add image", Args: []string{"img"}}); err != nil {
t.Fatalf("Append[%d]: %v", i, err)
}
}
data, err := os.ReadFile(filepath.Join(dir, "audit.log"))
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
lines := 0
for _, b := range data {
if b == '\n' {
lines++
}
}
if lines != 3 {
t.Errorf("expected 3 lines, got %d\nlog:\n%s", lines, data)
}
}
func TestShortFileRef(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{"local relative path", "./scripts/install.sh", "install.sh"},
{"local absolute path", "/home/example/scripts/install.sh", "install.sh"},
{"bare filename", "install.sh", "install.sh"},
{"plain URL", "https://get.rke2.io/install.sh", "install.sh"},
{"URL with credentials", "https://user:pass@get.rke2.io/install.sh", "install.sh"},
{"URL with query token", "https://get.rke2.io/install.sh?token=abc123", "install.sh"},
{"URL with credentials and query", "https://user:pass@get.rke2.io/install.sh?sig=abc123", "install.sh"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := ShortFileRef(tc.in); got != tc.want {
t.Errorf("ShortFileRef(%q) = %q, want %q", tc.in, got, tc.want)
}
})
}
}
func TestResolveDir_Default(t *testing.T) {
got := resolveDir("")
if got == "" {
t.Error("resolveDir(\"\") returned empty string")
}
}
pkg/consts/consts.go
+4
View File
@@ -91,6 +91,8 @@ const (
HaulerTempDir = "HAULER_TEMP_DIR"
HaulerStoreDir = "HAULER_STORE_DIR"
HaulerIgnoreErrors = "HAULER_IGNORE_ERRORS"
HaulerLogLevel = "HAULER_LOG_LEVEL"
HaulerAuditLevel = "HAULER_AUDIT_LEVEL"
// container files and directories
ImageManifestFile = "manifest.json"
@@ -110,6 +112,8 @@ const (
DefaultFileserverTimeout = 60
DefaultHaulerArchiveName = "haul.tar.zst"
DefaultHaulerManifestName = "hauler-manifest.yaml"
DefaultStoreMetadataName = "store.json"
DefaultStoreInventoryName = "stores.json"
DefaultRetries = 3
RetriesInterval = 5
CustomTimeFormat = "2006-01-02 15:04:05"
pkg/content/oci.go
+1 -1
View File
@@ -92,7 +92,7 @@ func (o *OCI) LoadIndex() error {
continue
}
// Set default kind if missing; normalize legacy dev.cosignproject.cosign values.
// Set default kind if missing... normalize legacy dev.cosignproject.cosign values
kind := desc.Annotations[consts.KindAnnotationName]
kind = consts.NormalizeLegacyKind(kind)
if kind == "" {
pkg/content/oci_test.go
+4 -4
View File
@@ -118,11 +118,11 @@ func TestLoadIndex_NormalizesLegacyKindInDescriptorAnnotations(t *testing.T) {
for _, desc := range walked {
kind := desc.Annotations[consts.KindAnnotationName]
if strings.HasPrefix(kind, legacyPrefix) {
t.Errorf("descriptor %s: Walk returned legacy kind %q; want normalized dev.hauler/... value",
t.Errorf("descriptor %s: Walk returned legacy kind %q... want normalized dev.hauler/... value",
desc.Digest, kind)
}
if !strings.HasPrefix(kind, newPrefix) {
t.Errorf("descriptor %s: Walk returned unexpected kind %q; want dev.hauler/... prefix",
t.Errorf("descriptor %s: Walk returned unexpected kind %q... want dev.hauler/... prefix",
desc.Digest, kind)
}
}
@@ -260,10 +260,10 @@ func TestPush_NormalizesLegacyKindInStoredDescriptor(t *testing.T) {
found = true
kind := d.Annotations[consts.KindAnnotationName]
if strings.HasPrefix(kind, legacyPrefix) {
t.Errorf("Push stored descriptor with legacy kind %q; want normalized dev.hauler/... value", kind)
t.Errorf("Push stored descriptor with legacy kind %q... want normalized dev.hauler/... value", kind)
}
if !strings.HasPrefix(kind, newPrefix) {
t.Errorf("Push stored descriptor with unexpected kind %q; want dev.hauler/... prefix", kind)
t.Errorf("Push stored descriptor with unexpected kind %q... want dev.hauler/... prefix", kind)
}
return nil
}); err != nil {
pkg/store/inventory.go
+123
View File
@@ -0,0 +1,123 @@
package store
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
zlog "github.com/rs/zerolog/log"
"hauler.dev/go/hauler/v2/pkg/consts"
)
// inventoryEntry is a store's last-known location, keyed by StoreID
type inventoryEntry struct {
Path string `json:"path"`
Updated string `json:"updated"`
}
type storeInventory map[string]inventoryEntry
func inventoryPath(haulerDir string) string {
return filepath.Join(haulerDir, consts.DefaultStoreInventoryName)
}
func loadInventory(haulerDir string) storeInventory {
inv := storeInventory{}
p := inventoryPath(haulerDir)
data, err := os.ReadFile(p)
if err != nil {
return inv
}
if err := json.Unmarshal(data, &inv); err != nil {
zlog.Warn().Err(err).Str("path", p).Msg("failed to parse store inventory... ignoring")
return storeInventory{}
}
return inv
}
func saveInventory(haulerDir string, inv storeInventory) error {
if err := os.MkdirAll(haulerDir, 0o755); err != nil {
return err
}
data, err := json.MarshalIndent(inv, "", " ")
if err != nil {
return err
}
// write to a temp file and rename to avoid a partially-written inventory
tmp := inventoryPath(haulerDir) + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return err
}
return os.Rename(tmp, inventoryPath(haulerDir))
}
// updateStoreInventory records storeID's path in <haulerDir>/stores.json and
// prunes any other entries that no longer contain the store they claim
func updateStoreInventory(haulerDir, storeID, path string) {
inv := loadInventory(haulerDir)
for id, entry := range inv {
if id == storeID {
continue
}
if !MatchesStoreID(entry.Path, id) {
delete(inv, id)
}
}
inv[storeID] = inventoryEntry{
Path: path,
Updated: time.Now().UTC().Format(time.RFC3339),
}
if err := saveInventory(haulerDir, inv); err != nil {
zlog.Warn().Err(err).Msg("failed to update store inventory... store id lookup may not find this store later")
}
}
// MatchesStoreID reports whether path's store.json identifies it as storeID
func MatchesStoreID(path, storeID string) bool {
data, err := os.ReadFile(filepath.Join(path, consts.DefaultStoreMetadataName))
if err != nil {
return false
}
var m storeMetadata
if json.Unmarshal(data, &m) != nil {
return false
}
return m.StoreID == storeID
}
// ResolveStoreID looks up idOrPrefix (a StoreID or an unambiguous prefix of
// one) in <haulerDir>/stores.json and returns the matched store's full id and
// last-known path. Callers should verify the path with MatchesStoreID before
// trusting it
func ResolveStoreID(haulerDir, idOrPrefix string) (id string, path string, err error) {
inv := loadInventory(haulerDir)
if entry, ok := inv[idOrPrefix]; ok {
return idOrPrefix, entry.Path, nil
}
var matchID, matchPath string
count := 0
for invID, entry := range inv {
if strings.HasPrefix(invID, idOrPrefix) {
matchID, matchPath = invID, entry.Path
count++
}
}
switch count {
case 0:
return "", "", fmt.Errorf("no store found matching id %q", idOrPrefix)
case 1:
return matchID, matchPath, nil
default:
return "", "", fmt.Errorf("store id %q is ambiguous and matches multiple stores", idOrPrefix)
}
}
pkg/store/store.go
+79 -23
View File
@@ -18,9 +18,11 @@ import (
"github.com/google/go-containerregistry/pkg/v1/daemon"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/uuid"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
zlog "github.com/rs/zerolog/log"
"golang.org/x/sync/errgroup"
"hauler.dev/go/hauler/v2/pkg/artifacts"
@@ -31,8 +33,10 @@ import (
type Layout struct {
*content.OCI
Root string
cache layer.Cache
Root string
StoreID string
haulerDir string
cache layer.Cache
}
type Options func(*Layout)
@@ -43,6 +47,14 @@ func WithCache(c layer.Cache) Options {
}
}
// WithHaulerDir records this store in <haulerDir>/stores.json so it can
// later be referenced by StoreID. If unset, NewLayout skips the inventory
func WithHaulerDir(dir string) Options {
return func(l *Layout) {
l.haulerDir = dir
}
}
func NewLayout(rootdir string, opts ...Options) (*Layout, error) {
ociStore, err := content.NewOCI(rootdir)
if err != nil {
@@ -54,17 +66,57 @@ func NewLayout(rootdir string, opts ...Options) (*Layout, error) {
}
l := &Layout{
Root: rootdir,
OCI: ociStore,
Root: rootdir,
OCI: ociStore,
StoreID: loadOrCreateStoreID(rootdir),
}
for _, opt := range opts {
opt(l)
}
if l.haulerDir != "" {
updateStoreInventory(l.haulerDir, l.StoreID, rootdir)
}
return l, nil
}
type storeMetadata struct {
StoreID string `json:"store-id"`
}
// loadOrCreateStoreID returns the persistent store identity from <rootdir>/store.json,
// creating the file with a fresh UUID on first use.
func loadOrCreateStoreID(rootdir string) string {
metaPath := filepath.Join(rootdir, consts.DefaultStoreMetadataName)
if data, err := os.ReadFile(metaPath); err == nil {
var m storeMetadata
if uerr := json.Unmarshal(data, &m); uerr == nil && m.StoreID != "" {
return m.StoreID
} else if uerr != nil {
zlog.Warn().Err(uerr).Str("path", metaPath).Msg("failed to parse store metadata... generating new store id")
} else {
zlog.Warn().Str("path", metaPath).Msg("store metadata missing store-id... generating new store id")
}
}
m := storeMetadata{StoreID: uuid.New().String()}
data, err := json.Marshal(m)
if err != nil {
zlog.Warn().Err(err).Msg("failed to marshal store metadata... store id will not persist across runs")
return m.StoreID
}
tmp := metaPath + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
zlog.Warn().Err(err).Str("path", tmp).Msg("failed to write store metadata... store id will not persist across runs")
return m.StoreID
}
if err := os.Rename(tmp, metaPath); err != nil {
zlog.Warn().Err(err).Str("path", metaPath).Msg("failed to write store metadata... store id will not persist across runs")
}
return m.StoreID
}
// AddArtifact adds an artifacts.OCI to the store
//
// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once
@@ -157,7 +209,7 @@ func (l *Layout) AddArtifactCollection(ctx context.Context, collection artifacts
// discovered via cosign's tag convention (<digest>.sig, <digest>.att, <digest>.sbom).
// When platform is non-empty and the ref is a multi-arch index, only that platform is fetched.
// When excludeExtras is true, cosign signatures, attestations, SBOMs, and OCI referrers are skipped.
func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, opts ...remote.Option) error {
func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excludeExtras bool, opts ...remote.Option) (string, error) {
allOpts := append([]remote.Option{
remote.WithAuthFromKeychain(authn.DefaultKeychain),
remote.WithContext(ctx),
@@ -165,12 +217,12 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl
parsedRef, err := gname.ParseReference(ref)
if err != nil {
return fmt.Errorf("parsing reference %q: %w", ref, err)
return "", fmt.Errorf("parsing reference %q: %w", ref, err)
}
desc, err := remote.Get(parsedRef, allOpts...)
if err != nil {
return fmt.Errorf("fetching descriptor for %q: %w", ref, err)
return "", fmt.Errorf("fetching descriptor for %q: %w", ref, err)
}
var imageDigest v1.Hash
@@ -179,10 +231,10 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl
// Multi-arch image with no platform filter: save the full index.
imageDigest, err = idx.Digest()
if err != nil {
return fmt.Errorf("getting index digest for %q: %w", ref, err)
return "", fmt.Errorf("getting index digest for %q: %w", ref, err)
}
if err := l.writeIndex(parsedRef, idx, consts.KindAnnotationIndex); err != nil {
return err
return "", err
}
} else {
// Single-platform image, or the caller requested a specific platform.
@@ -190,55 +242,59 @@ func (l *Layout) AddImage(ctx context.Context, ref string, platform string, excl
if platform != "" {
p, err := parsePlatform(platform)
if err != nil {
return err
return "", err
}
imgOpts = append(imgOpts, remote.WithPlatform(p))
}
img, err := remote.Image(parsedRef, imgOpts...)
if err != nil {
return fmt.Errorf("fetching image %q: %w", ref, err)
return "", fmt.Errorf("fetching image %q: %w", ref, err)
}
imageDigest, err = img.Digest()
if err != nil {
return fmt.Errorf("getting image digest for %q: %w", ref, err)
return "", fmt.Errorf("getting image digest for %q: %w", ref, err)
}
if err := l.writeImage(parsedRef, img, consts.KindAnnotationImage, ""); err != nil {
return err
return "", err
}
}
if !excludeExtras {
savedDigests, err := l.saveRelatedArtifacts(ctx, parsedRef, imageDigest, allOpts...)
if err != nil {
return err
return "", err
}
return l.saveReferrers(ctx, parsedRef, imageDigest, savedDigests, allOpts...)
return imageDigest.String(), l.saveReferrers(ctx, parsedRef, imageDigest, savedDigests, allOpts...)
}
return nil
return imageDigest.String(), nil
}
// AddLocalImage fetches a container image from the local Docker daemon and saves it to the store.
// No cosign signatures, attestations, SBOMs, or OCI referrers are fetched (registry-only concepts).
func (l *Layout) AddLocalImage(ctx context.Context, ref string) error {
func (l *Layout) AddLocalImage(ctx context.Context, ref string) (string, error) {
parsedRef, err := gname.ParseReference(ref)
if err != nil {
return fmt.Errorf("parsing reference %q: %w", ref, err)
return "", fmt.Errorf("parsing reference %q: %w", ref, err)
}
if err := ensureDockerHost(); err != nil {
return fmt.Errorf("failed to locate Docker daemon socket: %w -- is the Docker daemon running?", err)
return "", fmt.Errorf("failed to locate Docker daemon socket: %w -- is the Docker daemon running?", err)
}
img, err := daemon.Image(parsedRef, daemon.WithContext(ctx))
if err != nil {
return fmt.Errorf("failed to fetch image from Docker daemon: %w -- is the Docker daemon running?", err)
return "", fmt.Errorf("failed to fetch image from Docker daemon: %w -- is the Docker daemon running?", err)
}
if _, err := img.Digest(); err != nil {
return fmt.Errorf("getting image digest for %q: %w", ref, err)
d, err := img.Digest()
if err != nil {
return "", fmt.Errorf("getting image digest for %q: %w", ref, err)
}
return l.writeImage(parsedRef, img, consts.KindAnnotationImage, "")
if err := l.writeImage(parsedRef, img, consts.KindAnnotationImage, ""); err != nil {
return "", err
}
return d.String(), nil
}
// ensureDockerHost sets DOCKER_HOST if it is not already set and the default
pkg/store/store_id_test.go
+197
View File
@@ -0,0 +1,197 @@
package store_test
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"hauler.dev/go/hauler/v2/pkg/consts"
"hauler.dev/go/hauler/v2/pkg/store"
)
// TestStoreID_PersistsAcrossNewLayoutCalls verifies the store-id survives repeated NewLayout calls
func TestStoreID_PersistsAcrossNewLayoutCalls(t *testing.T) {
root := t.TempDir()
s1, err := store.NewLayout(root)
if err != nil {
t.Fatalf("NewLayout: %v", err)
}
if s1.StoreID == "" {
t.Fatal("expected a non-empty StoreID on first creation")
}
metaPath := filepath.Join(root, consts.DefaultStoreMetadataName)
if _, err := os.Stat(metaPath); err != nil {
t.Fatalf("expected %s to be written: %v", metaPath, err)
}
s2, err := store.NewLayout(root)
if err != nil {
t.Fatalf("second NewLayout: %v", err)
}
if s2.StoreID != s1.StoreID {
t.Errorf("StoreID changed across NewLayout calls: first=%s second=%s", s1.StoreID, s2.StoreID)
}
}
// TestStoreID_MalformedMetadataRegenerates verifies a corrupt store.json gets a fresh id
func TestStoreID_MalformedMetadataRegenerates(t *testing.T) {
root := t.TempDir()
metaPath := filepath.Join(root, consts.DefaultStoreMetadataName)
if err := os.WriteFile(metaPath, []byte("{not valid json"), 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
s, err := store.NewLayout(root)
if err != nil {
t.Fatalf("NewLayout: %v", err)
}
if s.StoreID == "" {
t.Fatal("expected a freshly generated StoreID when metadata is malformed")
}
data, err := os.ReadFile(metaPath)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
var m struct {
StoreID string `json:"store-id"`
}
if err := json.Unmarshal(data, &m); err != nil {
t.Fatalf("regenerated metadata is not valid JSON: %v", err)
}
if m.StoreID != s.StoreID {
t.Errorf("persisted store-id %q does not match Layout.StoreID %q", m.StoreID, s.StoreID)
}
}
// TestStoreID_MissingFieldRegenerates verifies a store.json missing store-id gets a fresh id
func TestStoreID_MissingFieldRegenerates(t *testing.T) {
root := t.TempDir()
metaPath := filepath.Join(root, consts.DefaultStoreMetadataName)
if err := os.WriteFile(metaPath, []byte(`{}`), 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
s, err := store.NewLayout(root)
if err != nil {
t.Fatalf("NewLayout: %v", err)
}
if s.StoreID == "" {
t.Fatal("expected a freshly generated StoreID when store-id field is missing")
}
}
// TestResolveStoreID covers exact-id, prefix, ambiguous-prefix, and no-match resolution
func TestResolveStoreID(t *testing.T) {
haulerDir := t.TempDir()
type entry struct {
Path string `json:"path"`
Updated string `json:"updated"`
}
inv := map[string]entry{
"ec520cf6-e01b-4d6f-93ea-6588de0d5159": {Path: "/store/a", Updated: "2026-01-01T00:00:00Z"},
"ec520000-0000-0000-0000-000000000000": {Path: "/store/b", Updated: "2026-01-01T00:00:00Z"},
"deadbeef-dead-beef-dead-beefdeadbeef": {Path: "/store/c", Updated: "2026-01-01T00:00:00Z"},
}
data, err := json.Marshal(inv)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
if err := os.WriteFile(filepath.Join(haulerDir, consts.DefaultStoreInventoryName), data, 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
t.Run("exact id match", func(t *testing.T) {
id, path, err := store.ResolveStoreID(haulerDir, "deadbeef-dead-beef-dead-beefdeadbeef")
if err != nil {
t.Fatalf("ResolveStoreID: %v", err)
}
if id != "deadbeef-dead-beef-dead-beefdeadbeef" || path != "/store/c" {
t.Errorf("got id=%s path=%s, want id=deadbeef-dead-beef-dead-beefdeadbeef path=/store/c", id, path)
}
})
t.Run("unambiguous prefix match", func(t *testing.T) {
id, path, err := store.ResolveStoreID(haulerDir, "deadbeef")
if err != nil {
t.Fatalf("ResolveStoreID: %v", err)
}
if id != "deadbeef-dead-beef-dead-beefdeadbeef" || path != "/store/c" {
t.Errorf("got id=%s path=%s, want id=deadbeef-dead-beef-dead-beefdeadbeef path=/store/c", id, path)
}
})
t.Run("ambiguous prefix returns error", func(t *testing.T) {
_, _, err := store.ResolveStoreID(haulerDir, "ec52")
if err == nil {
t.Fatal("expected error for ambiguous prefix, got nil")
}
})
t.Run("no match returns error", func(t *testing.T) {
_, _, err := store.ResolveStoreID(haulerDir, "abc12345")
if err == nil {
t.Fatal("expected error when no store matches, got nil")
}
})
}
// TestNewLayout_UpdatesInventory verifies WithHaulerDir registers the store in stores.json
func TestNewLayout_UpdatesInventory(t *testing.T) {
haulerDir := t.TempDir()
storeDir := t.TempDir()
s, err := store.NewLayout(storeDir, store.WithHaulerDir(haulerDir))
if err != nil {
t.Fatalf("NewLayout: %v", err)
}
id, path, err := store.ResolveStoreID(haulerDir, s.StoreID)
if err != nil {
t.Fatalf("ResolveStoreID: %v", err)
}
if id != s.StoreID {
t.Errorf("resolved id = %s, want %s", id, s.StoreID)
}
if path != storeDir {
t.Errorf("resolved path = %s, want %s", path, storeDir)
}
if !store.MatchesStoreID(path, id) {
t.Error("MatchesStoreID should confirm the resolved path still contains this store")
}
}
// TestNewLayout_InventoryPrunesStaleEntries verifies stale inventory entries get pruned
func TestNewLayout_InventoryPrunesStaleEntries(t *testing.T) {
haulerDir := t.TempDir()
type entry struct {
Path string `json:"path"`
Updated string `json:"updated"`
}
staleID := "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
inv := map[string]entry{
staleID: {Path: filepath.Join(haulerDir, "gone"), Updated: "2026-01-01T00:00:00Z"},
}
data, err := json.Marshal(inv)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
invPath := filepath.Join(haulerDir, consts.DefaultStoreInventoryName)
if err := os.WriteFile(invPath, data, 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
storeDir := t.TempDir()
if _, err := store.NewLayout(storeDir, store.WithHaulerDir(haulerDir)); err != nil {
t.Fatalf("NewLayout: %v", err)
}
if _, _, err := store.ResolveStoreID(haulerDir, staleID); err == nil {
t.Error("expected stale inventory entry to be pruned, but it still resolved")
}
}
pkg/store/store_test.go
+4 -4
View File
@@ -392,7 +392,7 @@ func TestCopyDescriptorGraph_Manifest(t *testing.T) {
// --- Error path: delete a layer blob from source, expect Copy to fail ---
if len(srcManifest.Layers) == 0 {
t.Skip("artifact has no layers; skipping missing-blob error path")
t.Skip("artifact has no layers... skipping missing blob error path")
}
if err := os.Remove(blobPath(srcRoot, srcManifest.Layers[0].Digest)); err != nil {
t.Fatalf("could not remove layer blob to simulate corruption: %v", err)
@@ -464,7 +464,7 @@ func TestCopyDescriptorGraph_Index(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := src.AddImage(ctx, idxTag.Name(), "", false, remoteOpts...); err != nil {
if _, err := src.AddImage(ctx, idxTag.Name(), "", false, remoteOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}
if err := src.OCI.SaveIndex(); err != nil {
@@ -751,7 +751,7 @@ func TestAddImage_OCI11Referrers(t *testing.T) {
referrerImg = mutate.ConfigMediaType(referrerImg, types.MediaType(consts.OCIEmptyConfigMediaType))
referrerImg = mutate.Subject(referrerImg, baseDesc).(v1.Image)
// Push the referrer under an arbitrary tag; the in-process registry auto-wires the
// Push the referrer under an arbitrary tag... the in-process registry auto-wires the
// subject field and makes the manifest discoverable via GET /v2/.../referrers/<digest>.
referrerTag, err := gname.NewTag(host+"/test/image:bundle-referrer", gname.Insecure)
if err != nil {
@@ -767,7 +767,7 @@ func TestAddImage_OCI11Referrers(t *testing.T) {
if err != nil {
t.Fatalf("new layout: %v", err)
}
if err := s.AddImage(context.Background(), baseTag.Name(), "", false, remoteOpts...); err != nil {
if _, err := s.AddImage(context.Background(), baseTag.Name(), "", false, remoteOpts...); err != nil {
t.Fatalf("AddImage: %v", err)
}