Compare commits

...

6 Commits

Author SHA1 Message Date
Zack Brady
090f4dc905 fixed cli desc for store env var (#374) 2024-12-09 08:13:33 -05:00
Zack Brady
74aa40c69b updated versions for go/k8s/helm (#373) 2024-12-04 16:10:36 -05:00
Zack Brady
f5e3b38a6d updated version flag to internal/flags (#369) 2024-12-04 14:29:51 -05:00
Zack Brady
01faf396bb renamed incorrectly named consts (#371) 2024-12-04 14:29:26 -05:00
Zack Brady
235218cfff added store env var (#370)
* added strictmode flag and consts
* updated code with strictmode
* added flag for retryoperation
* updated registry short flag
* added store env var
* added/fixed more env var code

---------

Signed-off-by: Zack Brady <zackbrady123@gmail.com>
2024-12-04 14:26:53 -05:00
Zack Brady
4270a27819 adding ignore errors and retries for continue on error/fail on error (#368)
* added strictmode flag and consts
* updated code with strictmode
* added flag for retryoperation
* updated registry short flag
* updated strictmode to ignore errors
* fixed command description
* cleaned up error/debug logs
2024-12-02 17:18:58 -05:00
18 changed files with 200 additions and 103 deletions

View File

@@ -16,7 +16,7 @@ func New(ctx context.Context, binaries embed.FS, ro *flags.CliRootOpts) *cobra.C
cmd := &cobra.Command{
Use: "hauler",
Short: "Airgap Swiss Army Knife",
Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir,
Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
l := log.FromContext(ctx)
l.SetLevel(ro.LogLevel)

View File

@@ -75,7 +75,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
return err
}
return store.SyncCmd(ctx, o, s, ro)
return store.SyncCmd(ctx, o, s, rso, ro)
},
}
o.AddFlags(cmd)
@@ -327,7 +327,7 @@ hauler store add image rgcrprod.azurecr.us/hauler/rke2-manifest.yaml:v1.28.12-rk
return err
}
return store.AddImageCmd(ctx, o, s, args[0], ro)
return store.AddImageCmd(ctx, o, s, args[0], rso, ro)
},
}
o.AddFlags(cmd)

View File

@@ -2,6 +2,7 @@ package store
import (
"context"
"os"
"github.com/google/go-containerregistry/pkg/name"
"hauler.dev/go/hauler/pkg/artifacts/file/getter"
@@ -52,7 +53,7 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
return nil
}
func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, reference string, ro *flags.CliRootOpts) error {
func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, reference string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
cfg := v1alpha1.Image{
@@ -62,31 +63,48 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
// Check if the user provided a key.
if o.Key != "" {
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name, ro)
err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name, rso, ro)
if err != nil {
return err
}
l.Infof("signature verified for image [%s]", cfg.Name)
}
return storeImage(ctx, s, cfg, o.Platform, ro)
return storeImage(ctx, s, cfg, o.Platform, rso, ro)
}
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform string, ro *flags.CliRootOpts) error {
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
if !ro.IgnoreErrors {
envVar := os.Getenv(consts.HaulerIgnoreErrors)
if envVar == "true" {
ro.IgnoreErrors = true
}
}
l.Infof("adding 'image' [%s] to the store", i.Name)
r, err := name.ParseReference(i.Name)
if err != nil {
l.Warnf("unable to parse 'image' [%s], skipping...", r.Name())
return nil
if ro.IgnoreErrors {
l.Warnf("unable to parse 'image' [%s]: %v... skipping...", i.Name, err)
return nil
} else {
l.Errorf("unable to parse 'image' [%s]: %v", i.Name, err)
return err
}
}
err = cosign.SaveImage(ctx, s, r.Name(), platform, ro)
err = cosign.SaveImage(ctx, s, r.Name(), platform, rso, ro)
if err != nil {
l.Warnf("unable to add 'image' [%s] to store. skipping...", r.Name())
return nil
if ro.IgnoreErrors {
l.Warnf("unable to add 'image' [%s] to store: %v... skipping...", r.Name(), err)
return nil
} else {
l.Errorf("unable to add 'image' [%s] to store: %v", r.Name(), err)
return err
}
}
l.Infof("successfully added 'image' [%s]", r.Name())

View File

@@ -18,9 +18,19 @@ import (
func LoadCmd(ctx context.Context, o *flags.LoadOpts, archiveRefs ...string) error {
l := log.FromContext(ctx)
storeDir := o.StoreDir
if storeDir == "" {
storeDir = os.Getenv(consts.HaulerStoreDir)
}
if storeDir == "" {
storeDir = consts.DefaultStoreName
}
for _, archiveRef := range archiveRefs {
l.Infof("loading content from [%s] to [%s]", archiveRef, o.StoreDir)
err := unarchiveLayoutTo(ctx, archiveRef, o.StoreDir, o.TempOverride)
l.Infof("loading content from [%s] to [%s]", archiveRef, storeDir)
err := unarchiveLayoutTo(ctx, archiveRef, storeDir, o.TempOverride)
if err != nil {
return err
}

View File

@@ -28,6 +28,16 @@ import (
func SaveCmd(ctx context.Context, o *flags.SaveOpts, outputFile string) error {
l := log.FromContext(ctx)
storeDir := o.StoreDir
if storeDir == "" {
storeDir = os.Getenv(consts.HaulerStoreDir)
}
if storeDir == "" {
storeDir = consts.DefaultStoreName
}
// TODO: Support more formats?
a := archiver.NewTarZstd()
a.OverwriteExisting = true
@@ -42,7 +52,7 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, outputFile string) error {
return err
}
defer os.Chdir(cwd)
if err := os.Chdir(o.StoreDir); err != nil {
if err := os.Chdir(storeDir); err != nil {
return err
}
@@ -55,7 +65,7 @@ func SaveCmd(ctx context.Context, o *flags.SaveOpts, outputFile string) error {
return err
}
l.Infof("saved store [%s] -> [%s]", o.StoreDir, absOutputfile)
l.Infof("saved store [%s] -> [%s]", storeDir, absOutputfile)
return nil
}
@@ -161,7 +171,7 @@ func writeExportsManifest(ctx context.Context, dir string, platformStr string) e
return err
}
return oci.WriteFile(consts.OCIImageManifestFile, buf.Bytes(), 0666)
return oci.WriteFile(consts.ImageManifestFile, buf.Bytes(), 0666)
}
func (x *exports) describe() tarball.Manifest {

View File

@@ -24,7 +24,7 @@ import (
"hauler.dev/go/hauler/pkg/store"
)
func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.CliRootOpts) error {
func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
// if passed products, check for a remote manifest to retrieve and use.
@@ -44,7 +44,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
img := v1alpha1.Image{
Name: manifestLoc,
}
err := storeImage(ctx, s, img, o.Platform, ro)
err := storeImage(ctx, s, img, o.Platform, rso, ro)
if err != nil {
return err
}
@@ -58,7 +58,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
if err != nil {
return err
}
err = processContent(ctx, fi, o, s, ro)
err = processContent(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
@@ -71,7 +71,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
if err != nil {
return err
}
err = processContent(ctx, fi, o, s, ro)
err = processContent(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
@@ -80,7 +80,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
return nil
}
func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, ro *flags.CliRootOpts) error {
func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
reader := yaml.NewYAMLReader(bufio.NewReader(fi))
@@ -169,7 +169,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
l.Debugf("key for image [%s]", key)
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name, ro)
err := cosign.VerifySignature(ctx, s, key, i.Name, rso, ro)
if err != nil {
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
@@ -188,7 +188,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
platform = i.Platform
}
err = storeImage(ctx, s, i, platform, ro)
err = storeImage(ctx, s, i, platform, rso, ro)
if err != nil {
return err
}

View File

@@ -10,7 +10,7 @@ import (
)
func addVersion(parent *cobra.Command, ro *flags.CliRootOpts) {
var json bool
o := &flags.VersionOpts{}
cmd := &cobra.Command{
Use: "version",
@@ -23,7 +23,7 @@ func addVersion(parent *cobra.Command, ro *flags.CliRootOpts) {
v.FontName = "starwars"
cmd.SetOut(cmd.OutOrStdout())
if json {
if o.JSON {
out, err := v.JSONString()
if err != nil {
return fmt.Errorf("unable to generate JSON from version info: %w", err)
@@ -35,7 +35,7 @@ func addVersion(parent *cobra.Command, ro *flags.CliRootOpts) {
return nil
},
}
cmd.Flags().BoolVar(&json, "json", false, "toggle output in JSON")
o.AddFlags(cmd)
parent.AddCommand(cmd)
}

31
go.mod
View File

@@ -1,12 +1,12 @@
module hauler.dev/go/hauler
go 1.23.0
go 1.23
toolchain go1.23.2
toolchain go1.23.4
require (
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
github.com/containerd/containerd v1.7.12
github.com/containerd/containerd v1.7.23
github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2
github.com/docker/go-metrics v0.0.1
github.com/google/go-containerregistry v0.16.1
@@ -23,9 +23,9 @@ require (
github.com/spf13/afero v1.10.0
github.com/spf13/cobra v1.8.1
golang.org/x/sync v0.8.0
helm.sh/helm/v3 v3.16.2
k8s.io/apimachinery v0.31.2
k8s.io/client-go v0.31.2
helm.sh/helm/v3 v3.16.3
k8s.io/apimachinery v0.31.3
k8s.io/client-go v0.31.3
oras.land/oras-go v1.2.5
)
@@ -39,7 +39,6 @@ require (
github.com/Masterminds/semver/v3 v3.3.0 // indirect
github.com/Masterminds/sprig/v3 v3.3.0 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/Microsoft/hcsshim v0.11.4 // indirect
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
@@ -51,9 +50,11 @@ require (
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/containerd/errdefs v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/cyphar/filepath-securejoin v0.3.1 // indirect
github.com/cyphar/filepath-securejoin v0.3.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/cli v25.0.1+incompatible // indirect
@@ -100,7 +101,7 @@ require (
github.com/jmoiron/sqlx v1.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
@@ -162,14 +163,14 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.31.2 // indirect
k8s.io/apiextensions-apiserver v0.31.2 // indirect
k8s.io/apiserver v0.31.2 // indirect
k8s.io/cli-runtime v0.31.2 // indirect
k8s.io/component-base v0.31.2 // indirect
k8s.io/api v0.31.3 // indirect
k8s.io/apiextensions-apiserver v0.31.3 // indirect
k8s.io/apiserver v0.31.3 // indirect
k8s.io/cli-runtime v0.31.3 // indirect
k8s.io/component-base v0.31.3 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 // indirect
k8s.io/kubectl v0.31.2 // indirect
k8s.io/kubectl v0.31.3 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
sigs.k8s.io/kustomize/api v0.18.0 // indirect

62
go.sum
View File

@@ -61,10 +61,10 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe
github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0=
github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ=
github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU=
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -108,12 +108,16 @@ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0=
github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk=
github.com/containerd/containerd v1.7.23 h1:H2CClyUkmpKAGlhQp95g2WXHfLYc7whAuvZGBNYOOwQ=
github.com/containerd/containerd v1.7.23/go.mod h1:7QUzfURqZWCZV7RLNEn1XjUCQLEf0bkaK4GjUaZehxw=
github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM=
github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4=
github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -122,8 +126,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/cyphar/filepath-securejoin v0.3.1 h1:1V7cHiaW+C+39wEfpH6XlLBQo3j/PciWFrgfCLS8XrE=
github.com/cyphar/filepath-securejoin v0.3.1/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc=
github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8=
github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -332,8 +336,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -391,6 +395,8 @@ github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU
github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -884,8 +890,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
helm.sh/helm/v3 v3.16.2 h1:Y9v7ry+ubQmi+cb5zw1Llx8OKHU9Hk9NQ/+P+LGBe2o=
helm.sh/helm/v3 v3.16.2/go.mod h1:SyTXgKBjNqi2NPsHCW5dDAsHqvGIu0kdNYNH9gQaw70=
helm.sh/helm/v3 v3.16.3 h1:kb8bSxMeRJ+knsK/ovvlaVPfdis0X3/ZhYCSFRP+YmY=
helm.sh/helm/v3 v3.16.3/go.mod h1:zeVWGDR4JJgiRbT3AnNsjYaX8OTJlIE9zC+Q7F7iUSU=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -893,26 +899,26 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0=
k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk=
k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0=
k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM=
k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
k8s.io/apiserver v0.31.2 h1:VUzOEUGRCDi6kX1OyQ801m4A7AUPglpsmGvdsekmcI4=
k8s.io/apiserver v0.31.2/go.mod h1:o3nKZR7lPlJqkU5I3Ove+Zx3JuoFjQobGX1Gctw6XuE=
k8s.io/cli-runtime v0.31.2 h1:7FQt4C4Xnqx8V1GJqymInK0FFsoC+fAZtbLqgXYVOLQ=
k8s.io/cli-runtime v0.31.2/go.mod h1:XROyicf+G7rQ6FQJMbeDV9jqxzkWXTYD6Uxd15noe0Q=
k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
k8s.io/component-base v0.31.2 h1:Z1J1LIaC0AV+nzcPRFqfK09af6bZ4D1nAOpWsy9owlA=
k8s.io/component-base v0.31.2/go.mod h1:9PeyyFN/drHjtJZMCTkSpQJS3U9OXORnHQqMLDz0sUQ=
k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8=
k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE=
k8s.io/apiextensions-apiserver v0.31.3 h1:+GFGj2qFiU7rGCsA5o+p/rul1OQIq6oYpQw4+u+nciE=
k8s.io/apiextensions-apiserver v0.31.3/go.mod h1:2DSpFhUZZJmn/cr/RweH1cEVVbzFw9YBu4T+U3mf1e4=
k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4=
k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
k8s.io/apiserver v0.31.3 h1:+1oHTtCB+OheqFEz375D0IlzHZ5VeQKX1KGXnx+TTuY=
k8s.io/apiserver v0.31.3/go.mod h1:PrxVbebxrxQPFhJk4powDISIROkNMKHibTg9lTRQ0Qg=
k8s.io/cli-runtime v0.31.3 h1:fEQD9Xokir78y7pVK/fCJN090/iYNrLHpFbGU4ul9TI=
k8s.io/cli-runtime v0.31.3/go.mod h1:Q2jkyTpl+f6AtodQvgDI8io3jrfr+Z0LyQBPJJ2Btq8=
k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4=
k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs=
k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ=
k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 h1:GKE9U8BH16uynoxQii0auTjmmmuZ3O0LFMN6S0lPPhI=
k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA=
k8s.io/kubectl v0.31.2 h1:gTxbvRkMBwvTSAlobiTVqsH6S8Aa1aGyBcu5xYLsn8M=
k8s.io/kubectl v0.31.2/go.mod h1:EyASYVU6PY+032RrTh5ahtSOMgoDRIux9V1JLKtG5xM=
k8s.io/kubectl v0.31.3 h1:3r111pCjPsvnR98oLLxDMwAeM6OPGmPty6gSKaLTQes=
k8s.io/kubectl v0.31.3/go.mod h1:lhMECDCbJN8He12qcKqs2QfmVo9Pue30geovBVpH5fs=
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=

View File

@@ -3,8 +3,9 @@ package flags
import "github.com/spf13/cobra"
type CliRootOpts struct {
LogLevel string
HaulerDir string
LogLevel string
HaulerDir string
IgnoreErrors bool
}
func AddRootFlags(cmd *cobra.Command, ro *CliRootOpts) {
@@ -12,4 +13,5 @@ func AddRootFlags(cmd *cobra.Command, ro *CliRootOpts) {
pf.StringVarP(&ro.LogLevel, "log-level", "l", "info", "Set the logging level (i.e. info, debug, warn)")
pf.StringVarP(&ro.HaulerDir, "haulerdir", "d", "", "Set the location of the hauler directory (default $HOME/.hauler)")
pf.BoolVar(&ro.IgnoreErrors, "ignore-errors", false, "Ignore/Bypass errors (i.e. warn on error) (defaults false)")
}

View File

@@ -10,5 +10,5 @@ type ExtractOpts struct {
func (o *ExtractOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringVarP(&o.DestinationDir, "output", "o", "", "(Optional) Specify the directory to output (defaults to current directory)")
f.StringVarP(&o.DestinationDir, "output", "o", "", "(Optional) Set the directory to output (defaults to current directory)")
}

View File

@@ -24,7 +24,7 @@ type ServeRegistryOpts struct {
func (o *ServeRegistryOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.IntVarP(&o.Port, "port", "p", consts.DefaultRegistryPort, "(Optional) Specify the port to use for incoming connections")
f.IntVarP(&o.Port, "port", "p", consts.DefaultRegistryPort, "(Optional) Set the port to use for incoming connections")
f.StringVar(&o.RootDir, "directory", consts.DefaultRegistryRootDir, "(Optional) Directory to use for backend. Defaults to $PWD/registry")
f.StringVarP(&o.ConfigFile, "config", "c", "", "(Optional) Location of config file (overrides all flags)")
f.BoolVar(&o.ReadOnly, "readonly", true, "(Optional) Run the registry as readonly")
@@ -77,7 +77,7 @@ type ServeFilesOpts struct {
func (o *ServeFilesOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.IntVarP(&o.Port, "port", "p", consts.DefaultFileserverPort, "(Optional) Specify the port to use for incoming connections")
f.IntVarP(&o.Port, "port", "p", consts.DefaultFileserverPort, "(Optional) Set the port to use for incoming connections")
f.IntVarP(&o.Timeout, "timeout", "t", consts.DefaultFileserverTimeout, "(Optional) Timeout duration for HTTP Requests in seconds for both reads/writes")
f.StringVar(&o.RootDir, "directory", consts.DefaultFileserverRootDir, "(Optional) Directory to use for backend. Defaults to $PWD/fileserver")

View File

@@ -14,11 +14,13 @@ import (
type StoreRootOpts struct {
StoreDir string
Retries int
}
func (o *StoreRootOpts) AddFlags(cmd *cobra.Command) {
pf := cmd.PersistentFlags()
pf.StringVarP(&o.StoreDir, "store", "s", consts.DefaultStoreName, "(Optional) Specify the directory to use for the content store")
pf.StringVarP(&o.StoreDir, "store", "s", "", "Set the directory to use for the content store")
pf.IntVarP(&o.Retries, "retries", "r", consts.DefaultRetries, "Set the number of retries for operations")
}
func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {
@@ -26,6 +28,14 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {
storeDir := o.StoreDir
if storeDir == "" {
storeDir = os.Getenv(consts.HaulerStoreDir)
}
if storeDir == "" {
storeDir = consts.DefaultStoreName
}
abs, err := filepath.Abs(storeDir)
if err != nil {
return nil, err
@@ -34,8 +44,7 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {
l.Debugf("using store at %s", abs)
if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) {
err := os.Mkdir(abs, os.ModePerm)
if err != nil {
if err := os.MkdirAll(abs, os.ModePerm); err != nil {
return nil, err
}
} else if err != nil {

View File

@@ -19,6 +19,6 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification")
f.StringSliceVar(&o.Products, "products", []string{}, "(Optional) Specify the product name to fetch collections from the product registry i.e. rancher=v2.8.5,rke2=v1.28.11+rke2r1")
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e linux/amd64 (defaults to all)")
f.StringVarP(&o.Registry, "registry", "r", "", "(Optional) Specify the registry of the image for images that do not alredy define one")
f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one")
f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)")
}

12
internal/flags/version.go Normal file
View File

@@ -0,0 +1,12 @@
package flags
import "github.com/spf13/cobra"
type VersionOpts struct {
JSON bool
}
func (o *VersionOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.BoolVar(&o.JSON, "json", false, "Set the output format to JSON")
}

View File

@@ -36,7 +36,7 @@ func Images() map[string]Fn {
m := make(map[string]Fn)
manifestMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) {
return consts.OCIImageManifestFile, nil
return consts.ImageManifestFile, nil
})
for _, l := range []string{consts.DockerManifestSchema2, consts.DockerManifestListSchema2, consts.OCIManifestSchema1} {
@@ -52,7 +52,7 @@ func Images() map[string]Fn {
}
configMapperFn := Fn(func(desc ocispec.Descriptor) (string, error) {
return consts.OCIImageConfigFile, nil
return consts.ImageConfigFile, nil
})
for _, l := range []string{consts.DockerConfigJSON} {

View File

@@ -62,15 +62,17 @@ const (
CollectionGroup = "collection.hauler.cattle.io"
// environment variables
HaulerDir = "HAULER_DIR"
HaulerTempDir = "HAULER_TEMP_DIR"
HaulerDir = "HAULER_DIR"
HaulerTempDir = "HAULER_TEMP_DIR"
HaulerStoreDir = "HAULER_STORE_DIR"
HaulerIgnoreErrors = "HAULER_IGNORE_ERRORS"
// container files and directories
OCIImageIndexFile = "index.json"
OCIImageManifestFile = "manifest.json"
OCIImageConfigFile = "config.json"
OCIImageLayoutFile = "oci-layout"
OCIImageBlobsDir = "blobs"
OCIImageIndexFile = "index.json"
OCIImageLayoutFile = "oci-layout"
OCIImageBlobsDir = "blobs"
ImageManifestFile = "manifest.json"
ImageConfigFile = "config.json"
// other constraints
CarbideRegistry = "rgcrprod.azurecr.us"

View File

@@ -23,7 +23,7 @@ import (
)
// VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign.
func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string, ro *flags.CliRootOpts) error {
func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
operation := func() error {
cosignBinaryPath, err := getCosignPath(ro.HaulerDir)
if err != nil {
@@ -33,19 +33,26 @@ func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref s
cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, ref)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error verifying signature: %v, output: %s", err, output)
return fmt.Errorf("error verifying signature: %v\n%s", err, output)
}
return nil
}
return RetryOperation(ctx, operation)
return RetryOperation(ctx, rso, ro, operation)
}
// SaveImage saves image and any signatures/attestations to the store.
func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string, ro *flags.CliRootOpts) error {
func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
if !ro.IgnoreErrors {
envVar := os.Getenv(consts.HaulerIgnoreErrors)
if envVar == "true" {
ro.IgnoreErrors = true
}
}
operation := func() error {
cosignBinaryPath, err := getCosignPath(ro.HaulerDir)
if err != nil {
@@ -92,7 +99,10 @@ func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string
// read command's stderr line by line
errors := bufio.NewScanner(stderr)
for errors.Scan() {
l.Warnf(errors.Text()) // write each line to your log, or anything you need
if ro.IgnoreErrors {
l.Warnf(errors.Text())
}
l.Errorf(errors.Text())
}
if err := errors.Err(); err != nil {
cmd.Wait()
@@ -108,7 +118,7 @@ func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string
return nil
}
return RetryOperation(ctx, operation)
return RetryOperation(ctx, rso, ro, operation)
}
// LoadImage loads store to a remote registry.
@@ -190,27 +200,43 @@ func RegistryLogin(ctx context.Context, s *store.Layout, registry string, ropts
return nil
}
func RetryOperation(ctx context.Context, operation func() error) error {
func RetryOperation(ctx context.Context, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, operation func() error) error {
l := log.FromContext(ctx)
for attempt := 1; attempt <= consts.DefaultRetries; attempt++ {
if !ro.IgnoreErrors {
envVar := os.Getenv(consts.HaulerIgnoreErrors)
if envVar == "true" {
ro.IgnoreErrors = true
}
}
// Validate retries and fall back to a default
retries := rso.Retries
if retries <= 0 {
retries = consts.DefaultRetries
}
for attempt := 1; attempt <= rso.Retries; attempt++ {
err := operation()
if err == nil {
// If the operation succeeds, return nil (no error).
// If the operation succeeds, return nil (no error)
return nil
}
// Log the error for the current attempt.
l.Warnf("error (attempt %d/%d): %v", attempt, consts.DefaultRetries, err)
if ro.IgnoreErrors {
l.Warnf("warning (attempt %d/%d)... %v", attempt, rso.Retries, err)
} else {
l.Errorf("error (attempt %d/%d)... %v", attempt, rso.Retries, err)
}
// If this is not the last attempt, wait before retrying.
if attempt < consts.DefaultRetries {
// If this is not the last attempt, wait before retrying
if attempt < rso.Retries {
time.Sleep(time.Second * consts.RetriesInterval)
}
}
// If all attempts fail, return an error.
return fmt.Errorf("operation failed after %d attempts", consts.DefaultRetries)
// If all attempts fail, return an error
return fmt.Errorf("operation unsuccessful after %d attempts", rso.Retries)
}
func EnsureBinaryExists(ctx context.Context, bin embed.FS, ro *flags.CliRootOpts) error {
@@ -251,6 +277,7 @@ func EnsureBinaryExists(ctx context.Context, bin embed.FS, ro *flags.CliRootOpts
// getCosignPath returns the binary path
func getCosignPath(haulerDir string) (string, error) {
if haulerDir == "" {
haulerDir = os.Getenv(consts.HaulerDir)
}