From 31080936cbcc02b840df5e5bf657bc4bd012b877 Mon Sep 17 00:00:00 2001 From: CamrynCarter Date: Sat, 25 Jul 2026 15:27:06 -0700 Subject: [PATCH] create manifest from store contents --- cmd/hauler/cli/store.go | 45 +++++ cmd/hauler/cli/store/create.go | 304 +++++++++++++++++++++++++++++++++ internal/flags/create.go | 15 ++ 3 files changed, 364 insertions(+) create mode 100644 cmd/hauler/cli/store/create.go create mode 100644 internal/flags/create.go diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index e305ceb..eba7d3a 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -36,6 +36,7 @@ func addStore(parent *cobra.Command, ro *flags.CliRootOpts) { addStoreCopy(rso, ro), addStoreAdd(rso, ro), addStoreRemove(rso, ro), + addStoreCreate(rso, ro), ) parent.AddCommand(cmd) @@ -494,6 +495,50 @@ func addStoreAddChart(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co return cmd } +func addStoreCreate(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Command { + cmd := &cobra.Command{ + Use: "create", + Short: "Create content derived from the store", + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + } + + cmd.AddCommand( + addStoreCreateManifest(rso, ro), + ) + + return cmd +} + +func addStoreCreateManifest(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Command { + o := &flags.CreateManifestOpts{StoreRootOpts: rso} + + cmd := &cobra.Command{ + Use: "manifest", + Short: "Create a hauler content manifest from the store's metadata", + Example: ` # generate a manifest for the default store into ./hauler-manifest.yaml + hauler store create manifest + + # generate a manifest for a specific store into a custom path + hauler store create manifest --store /path/to/store --output my-manifest.yaml`, + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + s, err := o.Store(ctx, ro) + if err != nil { + return err + } + + return store.CreateManifestCmd(ctx, o, s) + }, + } + o.AddFlags(cmd) + + return cmd +} + func addStoreRemove(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Command { o := &flags.RemoveOpts{} cmd := &cobra.Command{ diff --git a/cmd/hauler/cli/store/create.go b/cmd/hauler/cli/store/create.go new file mode 100644 index 0000000..2c76cc6 --- /dev/null +++ b/cmd/hauler/cli/store/create.go @@ -0,0 +1,304 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + gname "github.com/google/go-containerregistry/pkg/name" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "gopkg.in/yaml.v3" + + "hauler.dev/go/hauler/v2/internal/flags" + "hauler.dev/go/hauler/v2/pkg/consts" + "hauler.dev/go/hauler/v2/pkg/log" + "hauler.dev/go/hauler/v2/pkg/store" +) + +// manifestImage, manifestChart, and manifestFile mirror the relevant fields of +// v1.Image/v1.Chart/v1.File, but keep only what can be confidently recovered from the +// store's metadata and use "omitempty" throughout (unlike the api types, which most +// callers unmarshal rather than marshal) so the generated manifest stays readable +// instead of listing every unset flag. +type manifestImage struct { + Name string `yaml:"name"` + Platform string `yaml:"platform,omitempty"` + Rewrite string `yaml:"rewrite,omitempty"` +} + +type manifestChart struct { + Name string `yaml:"name"` + RepoURL string `yaml:"repoURL,omitempty"` + Version string `yaml:"version,omitempty"` + Rewrite string `yaml:"rewrite,omitempty"` +} + +type manifestFile struct { + Path string `yaml:"path"` + Name string `yaml:"name,omitempty"` +} + +type manifestMetadata struct { + Name string `yaml:"name"` +} + +type manifestDoc struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata manifestMetadata `yaml:"metadata"` + Spec interface{} `yaml:"spec"` +} + +// CreateManifestCmd walks the store's OCI index (and the manifests/configs it +// references) to reconstruct a hauler content manifest capable of recreating the +// store's contents via `hauler store sync`. It groups discovered content into +// Images/Charts/Files documents and writes them to o.Output. +func CreateManifestCmd(ctx context.Context, o *flags.CreateManifestOpts, s *store.Layout) error { + l := log.FromContext(ctx) + + var images []manifestImage + var charts []manifestChart + var files []manifestFile + chartsMissingRepoURL := false + + if err := s.Walk(func(_ string, desc ocispec.Descriptor) error { + refName, ok := desc.Annotations[ocispec.AnnotationRefName] + if !ok { + return nil + } + + kind := desc.Annotations[consts.KindAnnotationName] + switch { + case kind == consts.KindAnnotationSigs, kind == consts.KindAnnotationAtts, kind == consts.KindAnnotationSboms: + // cosign-related artifacts are rediscovered automatically when the + // parent image is re-added, so they don't need their own entry. + return nil + case strings.HasPrefix(kind, consts.KindAnnotationReferrers): + return nil + } + + // Container images (both single-platform and multi-arch indexes) carry the + // full OCI reference under this annotation; charts and files never do. + if fullRef, isImage := desc.Annotations[consts.ContainerdImageNameKey]; isImage { + name := fullRef + rewrite := "" + if orig, ok := desc.Annotations[consts.OriginalRefAnnotation]; ok && orig != "" && orig != fullRef { + // The current ref differs from what was captured at the initial add, + // meaning --rewrite changed it since. Recover the original, pullable + // name and reapply the same rewrite so a resync reproduces this exact + // store layout. If there's no annotation at all (a store from before + // this was tracked) or it matches fullRef (never rewritten), fullRef + // is already the right, pullable name. + rewrite = fullRef + name = orig + } + + img := manifestImage{Name: name, Rewrite: rewrite} + if kind == consts.KindAnnotationImage { + // Only a single-platform manifest has an unambiguous platform to pin. + // A stored multi-arch index is left unset so a future sync re-pulls + // every platform, matching what's actually in the store. + platform, err := imagePlatform(ctx, s, desc) + if err != nil { + l.Warnf("could not determine platform for image [%s]: %v", name, err) + } else if platform != "" { + img.Platform = platform + } + } + images = append(images, img) + return nil + } + + rc, err := s.Fetch(ctx, desc) + if err != nil { + return fmt.Errorf("fetching manifest for [%s]: %w", refName, err) + } + defer rc.Close() + + var m ocispec.Manifest + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return fmt.Errorf("decoding manifest for [%s]: %w", refName, err) + } + + ref, err := gname.ParseReference(refName) + if err != nil { + return fmt.Errorf("parsing reference [%s]: %w", refName, err) + } + name := strings.TrimPrefix(ref.Context().RepositoryStr(), consts.DefaultNamespace+"/") + + switch m.Config.MediaType { + case consts.ChartConfigMediaType: + version := ref.Identifier() + if tag, ok := ref.(gname.Tag); ok { + version = tag.TagStr() + } + + repoURL := "" + rewrite := "" + if orig, ok := desc.Annotations[consts.OriginalRefAnnotation]; ok && orig != "" { + origRepoURL, origTotal := decodeOriginalChartRef(orig) + repoURL = origRepoURL + if origTotal != "" && origTotal != refName { + // The current ref differs from what was captured at the initial + // add, meaning --rewrite changed it since. Recover the original, + // pullable name/version and reapply the same rewrite so a resync + // reproduces this exact store layout. + rewrite = refName + if origRef, err := gname.ParseReference(origTotal); err == nil { + name = strings.TrimPrefix(origRef.Context().RepositoryStr(), consts.DefaultNamespace+"/") + version = origRef.Identifier() + if tag, ok := origRef.(gname.Tag); ok { + version = tag.TagStr() + } + } + } + } + + charts = append(charts, manifestChart{Name: name, RepoURL: repoURL, Version: version, Rewrite: rewrite}) + if repoURL == "" { + chartsMissingRepoURL = true + } + + case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType, consts.FileDirectoryConfigMediaType: + path := name + if orig, ok := desc.Annotations[consts.OriginalRefAnnotation]; ok && orig != "" { + path = orig + } + files = append(files, manifestFile{Path: path, Name: name}) + + default: + l.Warnf("skipping unrecognized artifact [%s] with config media type [%s]", refName, m.Config.MediaType) + } + + return nil + }); err != nil { + return err + } + + if len(images) == 0 && len(charts) == 0 && len(files) == 0 { + return fmt.Errorf("store contains no content to build a manifest from") + } + + base := sanitizeName(filepath.Base(s.Root)) + + var out strings.Builder + if len(images) > 0 { + if err := writeDoc(&out, "", consts.ImagesContentKind, base+"-images", struct { + Images []manifestImage `yaml:"images"` + }{images}); err != nil { + return err + } + } + if len(charts) > 0 { + header := "" + if chartsMissingRepoURL { + header = "# NOTE: repoURL could not be recovered from the store's metadata and must be filled in below.\n" + } + if err := writeDoc(&out, header, consts.ChartsContentKind, base+"-charts", struct { + Charts []manifestChart `yaml:"charts"` + }{charts}); err != nil { + return err + } + } + if len(files) > 0 { + if err := writeDoc(&out, "", consts.FilesContentKind, base+"-files", struct { + Files []manifestFile `yaml:"files"` + }{files}); err != nil { + return err + } + } + + if err := os.WriteFile(o.Output, []byte(out.String()), 0o644); err != nil { + return fmt.Errorf("writing manifest to [%s]: %w", o.Output, err) + } + + outPath := o.Output + if abs, err := filepath.Abs(o.Output); err == nil { + outPath = abs + } + l.Infof("wrote manifest with [%d] image(s), [%d] chart(s), [%d] file(s) to [%s]", len(images), len(charts), len(files), outPath) + + return nil +} + +func writeDoc(out *strings.Builder, header string, kind string, name string, spec interface{}) error { + doc := manifestDoc{ + APIVersion: consts.ContentGroup + "/v1", + Kind: kind, + Metadata: manifestMetadata{Name: name}, + Spec: spec, + } + data, err := yaml.Marshal(doc) + if err != nil { + return fmt.Errorf("marshaling [%s] manifest: %w", kind, err) + } + out.WriteString("---\n") + out.WriteString(header) + out.Write(data) + return nil +} + +// imagePlatform returns the "os/arch" of a single-platform image manifest by +// fetching its config blob, or "" if the platform can't be determined. +func imagePlatform(ctx context.Context, s *store.Layout, desc ocispec.Descriptor) (string, error) { + rc, err := s.Fetch(ctx, desc) + if err != nil { + return "", err + } + defer rc.Close() + + var m ocispec.Manifest + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return "", err + } + + cfgRc, err := s.FetchManifest(ctx, m) + if err != nil { + return "", err + } + defer cfgRc.Close() + + var cfg ocispec.Image + if err := json.NewDecoder(cfgRc).Decode(&cfg); err != nil { + return "", err + } + if cfg.OS == "" || cfg.Architecture == "" { + return "", nil + } + return cfg.OS + "/" + cfg.Architecture, nil +} + +// decodeOriginalChartRef splits a value produced by encodeOriginalChartRef (see +// storeChart in add.go) back into its repoURL and "repo:tag" parts. Values with no +// "|" (shouldn't occur once only encodeOriginalChartRef ever writes this annotation +// for charts) are treated as a bare ref with an unknown repoURL. +func decodeOriginalChartRef(v string) (repoURL string, total string) { + repoURL, total, found := strings.Cut(v, "|") + if !found { + return "", v + } + return repoURL, total +} + +// sanitizeName lowercases s and replaces any character outside [a-z0-9-] with '-' so +// the result is safe to use as a Kubernetes-style object name. +func sanitizeName(s string) string { + s = strings.ToLower(s) + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + out := strings.Trim(b.String(), "-") + if out == "" { + return "store" + } + return out +} diff --git a/internal/flags/create.go b/internal/flags/create.go new file mode 100644 index 0000000..e900e66 --- /dev/null +++ b/internal/flags/create.go @@ -0,0 +1,15 @@ +package flags + +import "github.com/spf13/cobra" + +type CreateManifestOpts struct { + *StoreRootOpts + + Output string +} + +func (o *CreateManifestOpts) AddFlags(cmd *cobra.Command) { + f := cmd.Flags() + + f.StringVarP(&o.Output, "output", "o", "hauler-manifest.yaml", "(Optional) Path to write the generated manifest to") +}