diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 6a8dcf3..089ec79 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -392,7 +392,7 @@ hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --rewrite cus return err } - return store.AddChartCmd(ctx, o, s, args[0]) + return store.AddChartCmd(ctx, o, s, args[0], rso, ro) }, } o.AddFlags(cmd) @@ -404,7 +404,7 @@ func addStoreRemove(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comm o := &flags.RemoveOpts{} cmd := &cobra.Command{ Use: "remove ", - Short: "Remove an artifact from the content store (experimental)", + Short: "(EXPERIMENTAL) Remove an artifact from the content store", Example: `# remove an image using full store reference hauler store info hauler store remove index.docker.io/library/busybox:stable diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index aa7d043..49edde8 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -4,10 +4,18 @@ import ( "context" "fmt" "os" + "path/filepath" + "regexp" + "slices" "strings" "github.com/google/go-containerregistry/pkg/name" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/engine" + "k8s.io/apimachinery/pkg/util/yaml" + "hauler.dev/go/hauler/internal/flags" v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1" "hauler.dev/go/hauler/pkg/artifacts/file" @@ -18,7 +26,6 @@ import ( "hauler.dev/go/hauler/pkg/log" "hauler.dev/go/hauler/pkg/reference" "hauler.dev/go/hauler/pkg/store" - "helm.sh/helm/v3/pkg/action" ) func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string) error { @@ -72,7 +79,7 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re } l.Infof("signature verified for image [%s]", cfg.Name) } else if o.CertIdentityRegexp != "" || o.CertIdentity != "" { - // verify signature using the provided keyless details + // verify signature using keyless details l.Infof("verifying keyless signature for [%s]", cfg.Name) err := cosign.VerifyKeylessSignature(ctx, s, o.CertIdentity, o.CertIdentityRegexp, o.CertOidcIssuer, o.CertOidcIssuerRegexp, o.CertGithubWorkflowRepository, o.Tlog, cfg.Name, rso, ro) if err != nil { @@ -180,7 +187,7 @@ func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Referenc } -func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, chartName string) error { +func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, chartName string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { cfg := v1.Chart{ Name: chartName, RepoURL: o.ChartOpts.RepoURL, @@ -191,19 +198,141 @@ func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, ch if o.Rewrite != "" { rewrite = o.Rewrite } - return storeChart(ctx, s, cfg, o.ChartOpts, rewrite) + return storeChart(ctx, s, cfg, o, rso, ro, rewrite) } -func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action.ChartPathOptions, rewrite string) error { +// unexported type for the context key to avoid collisions +type isSubchartKey struct{} + +// imageregex parses image references starting with "image:" and with optional spaces or optional quotes +var imageRegex = regexp.MustCompile(`(?m)^\s*image:\s*['"]?([^\s'"#]+)`) + +// helmAnnotatedImage parses images references from helm chart annotations +type helmAnnotatedImage struct { + Image string `yaml:"image"` + Name string `yaml:"name,omitempty"` +} + +// imagesFromChartAnnotations parses image references from helm chart annotations +func imagesFromChartAnnotations(c *helmchart.Chart) ([]string, error) { + if c == nil || c.Metadata == nil || c.Metadata.Annotations == nil { + return nil, nil + } + + // support multiple annotations + keys := []string{ + "helm.sh/images", + "images", + } + + var out []string + for _, k := range keys { + raw, ok := c.Metadata.Annotations[k] + if !ok || strings.TrimSpace(raw) == "" { + continue + } + + var items []helmAnnotatedImage + if err := yaml.Unmarshal([]byte(raw), &items); err != nil { + return nil, fmt.Errorf("failed to parse helm chart annotation %q: %w", k, err) + } + + for _, it := range items { + img := strings.TrimSpace(it.Image) + if img == "" { + continue + } + img = strings.TrimPrefix(img, "/") + out = append(out, img) + } + } + + slices.Sort(out) + out = slices.Compact(out) + + return out, nil +} + +// imagesFromImagesLock parses image references from images lock files in the chart directory +func imagesFromImagesLock(chartDir string) ([]string, error) { + var out []string + + for _, name := range []string{ + "images.lock", + "images-lock.yaml", + "images.lock.yaml", + ".images.lock.yaml", + } { + p := filepath.Join(chartDir, name) + b, err := os.ReadFile(p) + if err != nil { + continue + } + + matches := imageRegex.FindAllSubmatch(b, -1) + for _, m := range matches { + if len(m) > 1 { + out = append(out, string(m[1])) + } + } + } + + if len(out) == 0 { + return nil, nil + } + + for i := range out { + out[i] = strings.TrimPrefix(out[i], "/") + } + slices.Sort(out) + out = slices.Compact(out) + return out, nil +} + +func applyDefaultRegistry(img string, defaultRegistry string) (string, error) { + img = strings.TrimSpace(strings.TrimPrefix(img, "/")) + if img == "" || defaultRegistry == "" { + return img, nil + } + + ref, err := reference.Parse(img) + if err != nil { + return "", err + } + + if ref.Context().RegistryStr() != "" { + return img, nil + } + + newRef, err := reference.Relocate(img, defaultRegistry) + if err != nil { + return "", err + } + + return newRef.Name(), nil +} + +func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags.AddChartOpts, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error { l := log.FromContext(ctx) - l.Infof("adding chart [%s] to the store", cfg.Name) + // subchart logging prefix + isSubchart := ctx.Value(isSubchartKey{}) == true + prefix := "" + if isSubchart { + prefix = " ↳ " + } - // TODO: This shouldn't be necessary - opts.RepoURL = cfg.RepoURL - opts.Version = cfg.Version + // normalize chart name for logging + displayName := cfg.Name + if strings.Contains(cfg.Name, string(os.PathSeparator)) { + displayName = filepath.Base(cfg.Name) + } + l.Infof("%sadding chart [%s] to the store", prefix, displayName) - chrt, err := chart.NewChart(cfg.Name, opts) + opts.ChartOpts.RepoURL = cfg.RepoURL + opts.ChartOpts.Version = cfg.Version + + chrt, err := chart.NewChart(cfg.Name, opts.ChartOpts) if err != nil { return err } @@ -218,20 +347,226 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action return err } - _, err = s.AddOCI(ctx, chrt, ref.Name()) - if err != nil { + if _, err := s.AddOCI(ctx, chrt, ref.Name()); err != nil { + return err + } + if err := s.OCI.SaveIndex(); err != nil { return err - } else { - s.OCI.SaveIndex() } + l.Infof("%ssuccessfully added chart [%s:%s]", prefix, c.Name(), c.Metadata.Version) + + tempOverride := rso.TempOverride + if tempOverride == "" { + tempOverride = os.Getenv(consts.HaulerTempDir) + } + tempDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName) + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + + chartPath := chrt.Path() + if strings.HasSuffix(chartPath, ".tgz") { + l.Debugf("%sextracting chart archive [%s]", prefix, filepath.Base(chartPath)) + if err := chartutil.ExpandFile(tempDir, chartPath); err != nil { + return fmt.Errorf("failed to extract chart: %w", err) + } + + // expanded chart should be in a directory matching the chart name + expectedChartDir := filepath.Join(tempDir, c.Name()) + if _, err := os.Stat(expectedChartDir); err != nil { + return fmt.Errorf("chart archive did not expand into expected directory '%s': %w", c.Name(), err) + } + chartPath = expectedChartDir + } + + // add-images + if opts.AddImages { + userValues := chartutil.Values{} + if opts.HelmValues != "" { + userValues, err = chartutil.ReadValuesFile(opts.HelmValues) + if err != nil { + return fmt.Errorf("failed to read helm values file [%s]: %w", opts.HelmValues, err) + } + } + + // set helm default capabilities + caps := chartutil.DefaultCapabilities.Copy() + + // only parse and override if provided kube version + if opts.KubeVersion != "" { + kubeVersion, err := chartutil.ParseKubeVersion(opts.KubeVersion) + if err != nil { + l.Warnf("%sinvalid kube-version [%s], using default kubernetes version", prefix, opts.KubeVersion) + } else { + caps.KubeVersion = *kubeVersion + } + } + + values, err := chartutil.ToRenderValues(c, userValues, chartutil.ReleaseOptions{Namespace: "hauler"}, caps) + if err != nil { + return err + } + + // helper for normalization and deduping slices + normalizeUniq := func(in []string) []string { + if len(in) == 0 { + return nil + } + for i := range in { + in[i] = strings.TrimPrefix(in[i], "/") + } + slices.Sort(in) + return slices.Compact(in) + } + + // Collect images by method so we can debug counts + var ( + templateImages []string + annotationImages []string + lockImages []string + ) + + // parse helm chart templates and values for images + rendered, err := engine.Render(c, values) + if err != nil { + // charts may fail due to values so still try helm chart annotations and lock + l.Warnf("%sfailed to render chart [%s]: %v", prefix, c.Name(), err) + rendered = map[string]string{} + } + + for _, manifest := range rendered { + matches := imageRegex.FindAllStringSubmatch(manifest, -1) + for _, match := range matches { + if len(match) > 1 { + templateImages = append(templateImages, match[1]) + } + } + } + + // parse helm chart annotations for images + annotationImages, err = imagesFromChartAnnotations(c) + if err != nil { + l.Warnf("%sfailed to parse helm chart annotation for [%s:%s]: %v", prefix, c.Name(), c.Metadata.Version, err) + annotationImages = nil + } + + // parse images lock files for images + lockImages, err = imagesFromImagesLock(chartPath) + if err != nil { + l.Warnf("%sfailed to parse images lock: %v", prefix, err) + lockImages = nil + } + + // normalization and deduping the slices + templateImages = normalizeUniq(templateImages) + annotationImages = normalizeUniq(annotationImages) + lockImages = normalizeUniq(lockImages) + + // merge all sources then final dedupe + images := append(append(templateImages, annotationImages...), lockImages...) + images = normalizeUniq(images) + + l.Debugf("%simage references identified for helm template: [%d] image(s)", prefix, len(templateImages)) + + l.Debugf("%simage references identified for helm chart annotations: [%d] image(s)", prefix, len(annotationImages)) + + l.Debugf("%simage references identified for helm image lock file: [%d] image(s)", prefix, len(lockImages)) + l.Debugf("%ssuccessfully parsed and deduped image references: [%d] image(s)", prefix, len(images)) + + l.Debugf("%ssuccessfully parsed image references %v", prefix, images) + + if len(images) > 0 { + l.Infof("%s ↳ identified [%d] image(s) in [%s:%s]", prefix, len(images), c.Name(), c.Metadata.Version) + } + + for _, image := range images { + image, err := applyDefaultRegistry(image, opts.Registry) + if err != nil { + if ro.IgnoreErrors { + l.Warnf("%s ↳ unable to apply registry to image [%s]: %v... skipping...", prefix, image, err) + continue + } + return fmt.Errorf("unable to apply registry to image [%s]: %w", image, err) + } + + imgCfg := v1.Image{Name: image} + if err := storeImage(ctx, s, imgCfg, opts.Platform, rso, ro, ""); err != nil { + if ro.IgnoreErrors { + l.Warnf("%s ↳ failed to store image [%s]: %v... skipping...", prefix, image, err) + continue + } + return fmt.Errorf("failed to store image [%s]: %w", image, err) + } + s.OCI.LoadIndex() + if err := s.OCI.SaveIndex(); err != nil { + return err + } + } + } + + // add-dependencies + if opts.AddDependencies && len(c.Metadata.Dependencies) > 0 { + for _, dep := range c.Metadata.Dependencies { + l.Infof("%sadding dependent chart [%s:%s]", prefix, dep.Name, dep.Version) + + depOpts := *opts + depOpts.AddDependencies = false + depOpts.AddImages = false + subCtx := context.WithValue(ctx, isSubchartKey{}, true) + + var depCfg v1.Chart + var err error + + if strings.HasPrefix(dep.Repository, "file://") { + depPath := strings.TrimPrefix(dep.Repository, "file://") + subchartPath := filepath.Join(chartPath, depPath) + + depCfg = v1.Chart{Name: subchartPath, RepoURL: "", Version: ""} + depOpts.ChartOpts.RepoURL = "" + depOpts.ChartOpts.Version = "" + + err = storeChart(subCtx, s, depCfg, &depOpts, rso, ro, "") + } else { + depCfg = v1.Chart{Name: dep.Name, RepoURL: dep.Repository, Version: dep.Version} + depOpts.ChartOpts.RepoURL = dep.Repository + depOpts.ChartOpts.Version = dep.Version + + err = storeChart(subCtx, s, depCfg, &depOpts, rso, ro, "") + } + + if err != nil { + if ro.IgnoreErrors { + l.Warnf("%s ↳ failed to add dependent chart [%s]: %v... skipping...", prefix, dep.Name, err) + } else { + l.Errorf("%s ↳ failed to add dependent chart [%s]: %v", prefix, dep.Name, err) + return err + } + } + } + } + + // chart rewrite functionality if rewrite != "" { rewrite = strings.TrimPrefix(rewrite, "/") newRef, err := name.ParseReference(rewrite) if err != nil { - l.Errorf("unable to parse rewrite name: %w", err) + // error... don't continue with a bad reference + return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err) } + // if rewrite omits a tag... keep the existing tag + oldTag := ref.(name.Tag).TagStr() + if !strings.Contains(rewrite, ":") { + rewrite = strings.Join([]string{rewrite, oldTag}, ":") + newRef, err = name.ParseReference(rewrite) + if err != nil { + return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err) + } + } + + // rename chart name in store s.OCI.LoadIndex() oldRefContext := ref.Context() @@ -239,14 +574,7 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action oldRepo := oldRefContext.RepositoryStr() newRepo := newRefContext.RepositoryStr() - oldTag := ref.(name.Tag).TagStr() - - var newTag string - if strings.Contains(rewrite, ":") { - newTag = newRef.(name.Tag).TagStr() - } else { - newTag = oldTag - } + newTag := newRef.(name.Tag).TagStr() oldTotal := oldRepo + ":" + oldTag newTotal := newRepo + ":" + newTag @@ -266,11 +594,10 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action return fmt.Errorf("could not find chart [%s] in store", ref.Name()) } - cfg.Name = newRef.Name() - fmt.Println("chart name (new): ", cfg.Name) - - s.OCI.SaveIndex() + if err := s.OCI.SaveIndex(); err != nil { + return err + } } - l.Infof("successfully added chart [%s]", ref.Name()) + return nil } diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index 04b367a..ebd59de 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -24,7 +24,7 @@ import ( func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) - tempOverride := o.TempOverride + tempOverride := rso.TempOverride if tempOverride == "" { tempOverride = os.Getenv(consts.HaulerTempDir) diff --git a/cmd/hauler/cli/store/remove.go b/cmd/hauler/cli/store/remove.go index 33b820e..f8bc095 100644 --- a/cmd/hauler/cli/store/remove.go +++ b/cmd/hauler/cli/store/remove.go @@ -12,6 +12,29 @@ import ( "hauler.dev/go/hauler/pkg/store" ) +func formatReference(ref string) string { + tagIdx := strings.LastIndex(ref, ":") + if tagIdx == -1 { + return ref + } + + dashIdx := strings.Index(ref[tagIdx+1:], "-") + if dashIdx == -1 { + return ref + } + + dashIdx = tagIdx + 1 + dashIdx + + base := ref[:dashIdx] + suffix := ref[dashIdx+1:] + + if base == "" || suffix == "" { + return ref + } + + return fmt.Sprintf("%s [%s]", base, suffix) +} + func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref string) error { l := log.FromContext(ctx) @@ -38,18 +61,18 @@ func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref st } if len(matches) == 0 { - return fmt.Errorf("reference [%s] not found in store (hint: use `hauler store info` to list store contents)", ref) + return fmt.Errorf("reference [%s] not found in store (use `hauler store info` to list store contents)", ref) } if len(matches) >= 1 { l.Infof("found %d matching references:", len(matches)) for _, m := range matches { - l.Infof(" - %s", m.reference) + l.Infof(" - %s", formatReference(m.reference)) } } if !o.Force { - fmt.Printf("are you sure you want to delete %d artifact(s) from the store? (yes/no) ", len(matches)) + fmt.Printf("are you sure you want to remove [%d] artifact(s) from the store? (yes/no) ", len(matches)) var response string _, err := fmt.Scanln(&response) @@ -58,31 +81,31 @@ func RemoveCmd(ctx context.Context, o *flags.RemoveOpts, s *store.Layout, ref st } switch response { case "yes", "y": - l.Infof("deleting artifacts from store...") + l.Infof("starting to remove artifacts from store...") case "no", "n": - l.Infof("deletion cancelled") + l.Infof("successfully cancelled removal of artifacts from store") return nil default: return fmt.Errorf("invalid response '%s' - please answer 'yes' or 'no'", response) } } - //remove artifact(s) + // remove artifact(s) for _, m := range matches { if err := s.RemoveArtifact(ctx, m.reference, m.desc); err != nil { - return fmt.Errorf("failed to remove artifact %s: %w", m.reference, err) + return fmt.Errorf("failed to remove artifact %s: %w", formatReference(m.reference), err) } - l.Infof("removed [%s] of type %s with digest [%s]", m.reference, m.desc.MediaType, m.desc.Digest.String()) + l.Infof("successfully removed [%s] of type [%s] with digest [%s]", formatReference(m.reference), m.desc.MediaType, m.desc.Digest.String()) } // clean up unreferenced blobs l.Infof("cleaning up unreferenced blobs...") - deletedCount, deletedSize, err := s.CleanUp(ctx) + removedCount, removedSize, err := s.CleanUp(ctx) if err != nil { - l.Warnf("garbrage collection failed: %v", err) - } else if deletedCount > 0 { - l.Infof("removed %d unreferenced blobs (freed %d bytes)", deletedCount, deletedSize) + l.Warnf("garbage collection failed: %v", err) + } else if removedCount > 0 { + l.Infof("successfully removed [%d] unreferenced blobs [freed %d bytes]", removedCount, removedSize) } return nil diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 99eb120..7e81967 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -32,7 +32,7 @@ import ( func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) - tempOverride := o.TempOverride + tempOverride := rso.TempOverride if tempOverride == "" { tempOverride = os.Getenv(consts.HaulerTempDir) @@ -506,8 +506,17 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor if err := convert.ConvertCharts(&alphaCfg, &v1Cfg); err != nil { return err } - for i, ch := range v1Cfg.Spec.Charts { - if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}, v1Cfg.Spec.Charts[i].Rewrite); err != nil { + for _, ch := range v1Cfg.Spec.Charts { + if err := storeChart(ctx, s, ch, + &flags.AddChartOpts{ + ChartOpts: &action.ChartPathOptions{ + RepoURL: ch.RepoURL, + Version: ch.Version, + }, + }, + rso, ro, + "", + ); err != nil { return err } } @@ -517,8 +526,29 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor if err := yaml.Unmarshal(doc, &cfg); err != nil { return err } + registry := o.Registry + if registry == "" { + annotation := cfg.GetAnnotations() + if annotation != nil { + registry = annotation[consts.ImageAnnotationRegistry] + } + } + for i, ch := range cfg.Spec.Charts { - if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}, cfg.Spec.Charts[i].Rewrite); err != nil { + if err := storeChart(ctx, s, ch, + &flags.AddChartOpts{ + ChartOpts: &action.ChartPathOptions{ + RepoURL: ch.RepoURL, + Version: ch.Version, + }, + AddImages: ch.AddImages, + AddDependencies: ch.AddDependencies, + Registry: registry, + Platform: o.Platform, + }, + rso, ro, + cfg.Spec.Charts[i].Rewrite, + ); err != nil { return err } } diff --git a/internal/flags/add.go b/internal/flags/add.go index c442ff6..92089a9 100644 --- a/internal/flags/add.go +++ b/internal/flags/add.go @@ -27,9 +27,9 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { f.StringVar(&o.CertOidcIssuer, "certificate-oidc-issuer", "", "(Optional) Cosign option to validate oidc issuer") f.StringVar(&o.CertOidcIssuerRegexp, "certificate-oidc-issuer-regexp", "", "(Optional) Cosign option to validate oidc issuer with regex") f.StringVar(&o.CertGithubWorkflowRepository, "certificate-github-workflow-repository", "", "(Optional) Cosign certificate-github-workflow-repository option") - f.BoolVarP(&o.Tlog, "use-tlog-verify", "v", false, "(Optional) Allow transparency log verification. (defaults to false)") - f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specifiy the platform of the image... i.e. linux/amd64 (defaults to all)") - f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string (experimental)") + f.BoolVar(&o.Tlog, "use-tlog-verify", false, "(Optional) Allow transparency log verification (defaults to false)") + f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e. linux/amd64 (defaults to all)") + f.StringVar(&o.Rewrite, "rewrite", "", "(EXPERIMENTAL & Optional) Rewrite artifact path to specified string") } type AddFileOpts struct { @@ -45,24 +45,37 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) { type AddChartOpts struct { *StoreRootOpts - ChartOpts *action.ChartPathOptions - Rewrite string + ChartOpts *action.ChartPathOptions + Rewrite string + AddDependencies bool + AddImages bool + HelmValues string + Platform string + Registry string + KubeVersion string } func (o *AddChartOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() f.StringVar(&o.ChartOpts.RepoURL, "repo", "", "Location of the chart (https:// | http:// | oci://)") - f.StringVar(&o.ChartOpts.Version, "version", "", "(Optional) Specifiy the version of the chart (v1.0.0 | 2.0.0 | ^2.0.0)") + f.StringVar(&o.ChartOpts.Version, "version", "", "(Optional) Specify the version of the chart (v1.0.0 | 2.0.0 | ^2.0.0)") f.BoolVar(&o.ChartOpts.Verify, "verify", false, "(Optional) Verify the chart before fetching it") f.StringVar(&o.ChartOpts.Username, "username", "", "(Optional) Username to use for authentication") f.StringVar(&o.ChartOpts.Password, "password", "", "(Optional) Password to use for authentication") - f.StringVar(&o.ChartOpts.CertFile, "cert-file", "", "(Optional) Location of the TLS Certificate to use for client authenication") - f.StringVar(&o.ChartOpts.KeyFile, "key-file", "", "(Optional) Location of the TLS Key to use for client authenication") + f.StringVar(&o.ChartOpts.CertFile, "cert-file", "", "(Optional) Location of the TLS Certificate to use for client authentication") + f.StringVar(&o.ChartOpts.KeyFile, "key-file", "", "(Optional) Location of the TLS Key to use for client authentication") f.BoolVar(&o.ChartOpts.InsecureSkipTLSverify, "insecure-skip-tls-verify", false, "(Optional) Skip TLS certificate verification") f.StringVar(&o.ChartOpts.CaFile, "ca-file", "", "(Optional) Location of CA Bundle to enable certification verification") - f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string (EXPERIMENTAL)") + f.StringVar(&o.Rewrite, "rewrite", "", "(EXPERIMENTAL & Optional) Rewrite artifact path to specified string") cmd.MarkFlagsRequiredTogether("username", "password") cmd.MarkFlagsRequiredTogether("cert-file", "key-file", "ca-file") + + cmd.Flags().BoolVar(&o.AddDependencies, "add-dependencies", false, "(EXPERIMENTAL & Optional) Fetch dependent helm charts") + f.BoolVar(&o.AddImages, "add-images", false, "(EXPERIMENTAL & Optional) Fetch images referenced in helm charts") + f.StringVar(&o.HelmValues, "values", "", "(EXPERIMENTAL & Optional) Specify helm chart values when fetching images") + f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image, e.g. linux/amd64") + f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one") + f.StringVar(&o.KubeVersion, "kube-version", "v1.34.1", "(EXPERIMENTAL & Optional) Override the kubernetes version for helm template rendering") } diff --git a/internal/flags/sync.go b/internal/flags/sync.go index 8c73e64..3246c65 100644 --- a/internal/flags/sync.go +++ b/internal/flags/sync.go @@ -36,6 +36,6 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e linux/amd64 (defaults to all)") f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one") f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)") - f.BoolVarP(&o.Tlog, "use-tlog-verify", "v", false, "(Optional) Allow transparency log verification. (defaults to false)") - f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string (experimental)") + f.BoolVar(&o.Tlog, "use-tlog-verify", false, "(Optional) Allow transparency log verification (defaults to false)") + f.StringVar(&o.Rewrite, "rewrite", "", "(EXPERIMENTAL & Optional) Rewrite artifact path to specified string") } diff --git a/pkg/apis/hauler.cattle.io/v1/chart.go b/pkg/apis/hauler.cattle.io/v1/chart.go index d0196c5..5c8b6ac 100644 --- a/pkg/apis/hauler.cattle.io/v1/chart.go +++ b/pkg/apis/hauler.cattle.io/v1/chart.go @@ -20,6 +20,9 @@ type Chart struct { RepoURL string `json:"repoURL,omitempty"` Version string `json:"version,omitempty"` Rewrite string `json:"rewrite,omitempty"` + + AddImages bool `json:"add-images,omitempty"` + AddDependencies bool `json:"add-dependencies,omitempty"` } type ThickCharts struct { diff --git a/pkg/apis/hauler.cattle.io/v1alpha1/chart.go b/pkg/apis/hauler.cattle.io/v1alpha1/chart.go index ef167fc..fdf1748 100644 --- a/pkg/apis/hauler.cattle.io/v1alpha1/chart.go +++ b/pkg/apis/hauler.cattle.io/v1alpha1/chart.go @@ -19,7 +19,6 @@ type Chart struct { Name string `json:"name,omitempty"` RepoURL string `json:"repoURL,omitempty"` Version string `json:"version,omitempty"` - Rewrite string `json:"rewrite,omitempty"` } type ThickCharts struct { diff --git a/pkg/content/chart/chart.go b/pkg/content/chart/chart.go index b01c1db..5f86b12 100644 --- a/pkg/content/chart/chart.go +++ b/pkg/content/chart/chart.go @@ -33,14 +33,13 @@ var ( settings = cli.New() ) -// Chart implements the OCI interface for Chart API objects. API spec values are -// stored into the Repo, Name, and Version fields. +// chart implements the oci interface for chart api objects... api spec values are stored into the name, repo, and version fields type Chart struct { path string annotations map[string]string } -// NewChart is a helper method that returns NewLocalChart or NewRemoteChart depending on chart contents +// newchart is a helper method that returns newlocalchart or newremotechart depending on chart contents func NewChart(name string, opts *action.ChartPathOptions) (*Chart, error) { chartRef := name actionConfig := new(action.Configuration) @@ -60,13 +59,31 @@ func NewChart(name string, opts *action.ChartPathOptions) (*Chart, error) { client.SetRegistryClient(registryClient) if registry.IsOCI(opts.RepoURL) { chartRef = opts.RepoURL + "/" + name - } else if isUrl(opts.RepoURL) { // OCI Protocol registers as a valid URL + } else if isUrl(opts.RepoURL) { // oci protocol registers as a valid url client.ChartPathOptions.RepoURL = opts.RepoURL - } else { // Handles cases like grafana/loki + } else { // handles cases like grafana and loki chartRef = opts.RepoURL + "/" + name } + // suppress helm downloader oci logs (stdout/stderr) + oldStdout := os.Stdout + oldStderr := os.Stderr + rOut, wOut, _ := os.Pipe() + rErr, wErr, _ := os.Pipe() + os.Stdout = wOut + os.Stderr = wErr + chartPath, err := client.ChartPathOptions.LocateChart(chartRef, settings) + + wOut.Close() + wErr.Close() + os.Stdout = oldStdout + os.Stderr = oldStderr + _, _ = io.Copy(io.Discard, rOut) + _, _ = io.Copy(io.Discard, rErr) + rOut.Close() + rErr.Close() + if err != nil { return nil, err } @@ -151,9 +168,8 @@ func (h *Chart) RawChartData() ([]byte, error) { return os.ReadFile(h.path) } -// chartData loads the chart contents into memory and returns a NopCloser for the contents -// -// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB +// chartdata loads the chart contents into memory and returns a NopCloser for the contents +// normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB func (h *Chart) chartData() (gv1.Layer, error) { info, err := os.Stat(h.path) if err != nil { @@ -256,14 +272,14 @@ func newDefaultRegistryClient(plainHTTP bool) (*registry.Client, error) { opts := []registry.ClientOption{ registry.ClientOptDebug(settings.Debug), registry.ClientOptEnableCache(true), - registry.ClientOptWriter(os.Stderr), + registry.ClientOptWriter(io.Discard), registry.ClientOptCredentialsFile(settings.RegistryConfig), } if plainHTTP { opts = append(opts, registry.ClientOptPlainHTTP()) } - // Create a new registry client + // create a new registry client registryClient, err := registry.NewClient(opts...) if err != nil { return nil, err @@ -272,12 +288,21 @@ func newDefaultRegistryClient(plainHTTP bool) (*registry.Client, error) { } func newRegistryClientWithTLS(certFile, keyFile, caFile string, insecureSkipTLSverify bool) (*registry.Client, error) { - // Create a new registry client - registryClient, err := registry.NewRegistryClientWithTLS(os.Stderr, certFile, keyFile, caFile, insecureSkipTLSverify, - settings.RegistryConfig, settings.Debug, + // create a new registry client + registryClient, err := registry.NewRegistryClientWithTLS( + io.Discard, + certFile, keyFile, caFile, + insecureSkipTLSverify, + settings.RegistryConfig, + settings.Debug, ) if err != nil { return nil, err } return registryClient, nil } + +// path returns the local filesystem path to the chart archive or directory +func (h *Chart) Path() string { + return h.path +} diff --git a/testdata/hauler-manifest-pipeline.yaml b/testdata/hauler-manifest-pipeline.yaml index 4c751b0..6188541 100755 --- a/testdata/hauler-manifest-pipeline.yaml +++ b/testdata/hauler-manifest-pipeline.yaml @@ -5,8 +5,8 @@ metadata: name: hauler-content-images-example spec: images: - - name: busybox - - name: busybox:stable + - name: ghcr.io/hauler-dev/library/busybox + - name: ghcr.io/hauler-dev/library/busybox:stable platform: linux/amd64 - name: gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5 --- @@ -55,8 +55,8 @@ metadata: name: hauler-content-images-example spec: images: - - name: busybox - - name: busybox:stable + - name: ghcr.io/hauler-dev/library/busybox + - name: ghcr.io/hauler-dev/library/busybox:stable platform: linux/amd64 - name: gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5 --- diff --git a/testdata/hauler-manifest.yaml b/testdata/hauler-manifest.yaml index 4691779..4c7d013 100755 --- a/testdata/hauler-manifest.yaml +++ b/testdata/hauler-manifest.yaml @@ -5,8 +5,8 @@ metadata: name: hauler-content-images-example spec: images: - - name: busybox - - name: busybox:stable + - name: ghcr.io/hauler-dev/library/busybox + - name: ghcr.io/hauler-dev/library/busybox:stable platform: linux/amd64 - name: gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5 --- @@ -39,8 +39,8 @@ metadata: name: hauler-content-images-example spec: images: - - name: busybox - - name: busybox:stable + - name: ghcr.io/hauler-dev/library/busybox + - name: ghcr.io/hauler-dev/library/busybox:stable platform: linux/amd64 - name: gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5 ---