github/hauler
1
0
Fork 0
mirror of https://github.com/hauler-dev/hauler.git synced 2026-02-14 09:59:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding ignore errors and retries for continue on error/fail on error (#368)

Browse Source 
* added strictmode flag and consts
* updated code with strictmode
* added flag for retryoperation
* updated registry short flag
* updated strictmode to ignore errors
* fixed command description
* cleaned up error/debug logs
This commit is contained in:
Zack Brady
2024-12-02 17:18:58 -05:00
committed by GitHub
parent 1b77295438
commit 4270a27819
11 changed files with 92 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/hauler/cli/cli.go
View File

@@ -16,7 +16,7 @@ func New(ctx context.Context, binaries embed.FS, ro *flags.CliRootOpts) *cobra.C
cmd := &cobra.Command{
Use: "hauler",
Short: "Airgap Swiss Army Knife",
Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir,
Example: " View the Docs: https://docs.hauler.dev\n Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerIgnoreErrors,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
l := log.FromContext(ctx)
l.SetLevel(ro.LogLevel)

4
cmd/hauler/cli/store.go
View File

@@ -75,7 +75,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
return err
}
return store.SyncCmd(ctx, o, s, ro)
return store.SyncCmd(ctx, o, s, rso, ro)
},
}
o.AddFlags(cmd)
@@ -327,7 +327,7 @@ hauler store add image rgcrprod.azurecr.us/hauler/rke2-manifest.yaml:v1.28.12-rk
return err
}
return store.AddImageCmd(ctx, o, s, args[0], ro)
return store.AddImageCmd(ctx, o, s, args[0], rso, ro)
},
}
o.AddFlags(cmd)

36
cmd/hauler/cli/store/add.go
View File

@@ -2,6 +2,7 @@ package store
import (
"context"
"os"
"github.com/google/go-containerregistry/pkg/name"
"hauler.dev/go/hauler/pkg/artifacts/file/getter"
@@ -52,7 +53,7 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
return nil
}
func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, reference string, ro *flags.CliRootOpts) error {
func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, reference string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
cfg := v1alpha1.Image{
@@ -62,31 +63,48 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
// Check if the user provided a key.
if o.Key != "" {
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name, ro)
err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name, rso, ro)
if err != nil {
return err
}
l.Infof("signature verified for image [%s]", cfg.Name)
}
return storeImage(ctx, s, cfg, o.Platform, ro)
return storeImage(ctx, s, cfg, o.Platform, rso, ro)
}
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform string, ro *flags.CliRootOpts) error {
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
if !ro.IgnoreErrors {
envVar := os.Getenv(consts.HaulerIgnoreErrors)
if envVar == "true" {
ro.IgnoreErrors = true
}
}
l.Infof("adding 'image' [%s] to the store", i.Name)
r, err := name.ParseReference(i.Name)
if err != nil {
l.Warnf("unable to parse 'image' [%s], skipping...", r.Name())
return nil
if ro.IgnoreErrors {
l.Warnf("unable to parse 'image' [%s]: %v... skipping...", i.Name, err)
return nil
} else {
l.Errorf("unable to parse 'image' [%s]: %v", i.Name, err)
return err
}
}
err = cosign.SaveImage(ctx, s, r.Name(), platform, ro)
err = cosign.SaveImage(ctx, s, r.Name(), platform, rso, ro)
if err != nil {
l.Warnf("unable to add 'image' [%s] to store. skipping...", r.Name())
return nil
if ro.IgnoreErrors {
l.Warnf("unable to add 'image' [%s] to store: %v... skipping...", r.Name(), err)
return nil
} else {
l.Errorf("unable to add 'image' [%s] to store: %v", r.Name(), err)
return err
}
}
l.Infof("successfully added 'image' [%s]", r.Name())

14
cmd/hauler/cli/store/sync.go
View File

@@ -24,7 +24,7 @@ import (
"hauler.dev/go/hauler/pkg/store"
)
func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.CliRootOpts) error {
func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
// if passed products, check for a remote manifest to retrieve and use.
@@ -44,7 +44,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
img := v1alpha1.Image{
Name: manifestLoc,
}
err := storeImage(ctx, s, img, o.Platform, ro)
err := storeImage(ctx, s, img, o.Platform, rso, ro)
if err != nil {
return err
}
@@ -58,7 +58,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
if err != nil {
return err
}
err = processContent(ctx, fi, o, s, ro)
err = processContent(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
@@ -71,7 +71,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
if err != nil {
return err
}
err = processContent(ctx, fi, o, s, ro)
err = processContent(ctx, fi, o, s, rso, ro)
if err != nil {
return err
}
@@ -80,7 +80,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, ro *flags.
return nil
}
func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, ro *flags.CliRootOpts) error {
func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
reader := yaml.NewYAMLReader(bufio.NewReader(fi))
@@ -169,7 +169,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
l.Debugf("key for image [%s]", key)
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name, ro)
err := cosign.VerifySignature(ctx, s, key, i.Name, rso, ro)
if err != nil {
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
@@ -188,7 +188,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
platform = i.Platform
}
err = storeImage(ctx, s, i, platform, ro)
err = storeImage(ctx, s, i, platform, rso, ro)
if err != nil {
return err
}

6
internal/flags/cli.go
View File

@@ -3,8 +3,9 @@ package flags
import "github.com/spf13/cobra"
type CliRootOpts struct {
LogLevel string
HaulerDir string
LogLevel string
HaulerDir string
IgnoreErrors bool
}
func AddRootFlags(cmd *cobra.Command, ro *CliRootOpts) {
@@ -12,4 +13,5 @@ func AddRootFlags(cmd *cobra.Command, ro *CliRootOpts) {
pf.StringVarP(&ro.LogLevel, "log-level", "l", "info", "Set the logging level (i.e. info, debug, warn)")
pf.StringVarP(&ro.HaulerDir, "haulerdir", "d", "", "Set the location of the hauler directory (default $HOME/.hauler)")
pf.BoolVar(&ro.IgnoreErrors, "ignore-errors", false, "Ignore/Bypass errors (i.e. warn on error) (defaults false)")
}

2
internal/flags/extract.go
View File

@@ -10,5 +10,5 @@ type ExtractOpts struct {
func (o *ExtractOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringVarP(&o.DestinationDir, "output", "o", "", "(Optional) Specify the directory to output (defaults to current directory)")
f.StringVarP(&o.DestinationDir, "output", "o", "", "(Optional) Set the directory to output (defaults to current directory)")
}

4
internal/flags/serve.go
View File

@@ -24,7 +24,7 @@ type ServeRegistryOpts struct {
func (o *ServeRegistryOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.IntVarP(&o.Port, "port", "p", consts.DefaultRegistryPort, "(Optional) Specify the port to use for incoming connections")
f.IntVarP(&o.Port, "port", "p", consts.DefaultRegistryPort, "(Optional) Set the port to use for incoming connections")
f.StringVar(&o.RootDir, "directory", consts.DefaultRegistryRootDir, "(Optional) Directory to use for backend. Defaults to $PWD/registry")
f.StringVarP(&o.ConfigFile, "config", "c", "", "(Optional) Location of config file (overrides all flags)")
f.BoolVar(&o.ReadOnly, "readonly", true, "(Optional) Run the registry as readonly")
@@ -77,7 +77,7 @@ type ServeFilesOpts struct {
func (o *ServeFilesOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.IntVarP(&o.Port, "port", "p", consts.DefaultFileserverPort, "(Optional) Specify the port to use for incoming connections")
f.IntVarP(&o.Port, "port", "p", consts.DefaultFileserverPort, "(Optional) Set the port to use for incoming connections")
f.IntVarP(&o.Timeout, "timeout", "t", consts.DefaultFileserverTimeout, "(Optional) Timeout duration for HTTP Requests in seconds for both reads/writes")
f.StringVar(&o.RootDir, "directory", consts.DefaultFileserverRootDir, "(Optional) Directory to use for backend. Defaults to $PWD/fileserver")

4
internal/flags/store.go
View File

@@ -14,11 +14,13 @@ import (
type StoreRootOpts struct {
StoreDir string
Retries int
}
func (o *StoreRootOpts) AddFlags(cmd *cobra.Command) {
pf := cmd.PersistentFlags()
pf.StringVarP(&o.StoreDir, "store", "s", consts.DefaultStoreName, "(Optional) Specify the directory to use for the content store")
pf.StringVarP(&o.StoreDir, "store", "s", consts.DefaultStoreName, "Set the directory to use for the content store")
pf.IntVarP(&o.Retries, "retries", "r", consts.DefaultRetries, "Set the number of retries for operations")
}
func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {

2
internal/flags/sync.go
View File

@@ -19,6 +19,6 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification")
f.StringSliceVar(&o.Products, "products", []string{}, "(Optional) Specify the product name to fetch collections from the product registry i.e. rancher=v2.8.5,rke2=v1.28.11+rke2r1")
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e linux/amd64 (defaults to all)")
f.StringVarP(&o.Registry, "registry", "r", "", "(Optional) Specify the registry of the image for images that do not alredy define one")
f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one")
f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)")
}

5
pkg/consts/consts.go
View File

@@ -62,8 +62,9 @@ const (
CollectionGroup = "collection.hauler.cattle.io"
// environment variables
HaulerDir = "HAULER_DIR"
HaulerTempDir = "HAULER_TEMP_DIR"
HaulerDir = "HAULER_DIR"
HaulerTempDir = "HAULER_TEMP_DIR"
HaulerIgnoreErrors = "HAULER_IGNORE_ERRORS"
// container files and directories
OCIImageIndexFile = "index.json"

56
pkg/cosign/cosign.go
View File

@@ -23,7 +23,7 @@ import (
)
// VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign.
func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string, ro *flags.CliRootOpts) error {
func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
operation := func() error {
cosignBinaryPath, err := getCosignPath(ro.HaulerDir)
if err != nil {
@@ -33,19 +33,26 @@ func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref s
cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, ref)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error verifying signature: %v, output: %s", err, output)
return fmt.Errorf("error verifying signature: %v\n%s", err, output)
}
return nil
}
return RetryOperation(ctx, operation)
return RetryOperation(ctx, rso, ro, operation)
}
// SaveImage saves image and any signatures/attestations to the store.
func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string, ro *flags.CliRootOpts) error {
func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
l := log.FromContext(ctx)
if !ro.IgnoreErrors {
envVar := os.Getenv(consts.HaulerIgnoreErrors)
if envVar == "true" {
ro.IgnoreErrors = true
}
}
operation := func() error {
cosignBinaryPath, err := getCosignPath(ro.HaulerDir)
if err != nil {
@@ -92,7 +99,10 @@ func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string
// read command's stderr line by line
errors := bufio.NewScanner(stderr)
for errors.Scan() {
l.Warnf(errors.Text()) // write each line to your log, or anything you need
if ro.IgnoreErrors {
l.Warnf(errors.Text())
}
l.Errorf(errors.Text())
}
if err := errors.Err(); err != nil {
cmd.Wait()
@@ -108,7 +118,7 @@ func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string
return nil
}
return RetryOperation(ctx, operation)
return RetryOperation(ctx, rso, ro, operation)
}
// LoadImage loads store to a remote registry.
@@ -190,27 +200,43 @@ func RegistryLogin(ctx context.Context, s *store.Layout, registry string, ropts
return nil
}
func RetryOperation(ctx context.Context, operation func() error) error {
func RetryOperation(ctx context.Context, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, operation func() error) error {
l := log.FromContext(ctx)
for attempt := 1; attempt <= consts.DefaultRetries; attempt++ {
if !ro.IgnoreErrors {
envVar := os.Getenv(consts.HaulerIgnoreErrors)
if envVar == "true" {
ro.IgnoreErrors = true
}
}
// Validate retries and fall back to a default
retries := rso.Retries
if retries <= 0 {
retries = consts.DefaultRetries
}
for attempt := 1; attempt <= rso.Retries; attempt++ {
err := operation()
if err == nil {
// If the operation succeeds, return nil (no error).
// If the operation succeeds, return nil (no error)
return nil
}
// Log the error for the current attempt.
l.Warnf("error (attempt %d/%d): %v", attempt, consts.DefaultRetries, err)
if ro.IgnoreErrors {
l.Warnf("warning (attempt %d/%d)... %v", attempt, rso.Retries, err)
} else {
l.Errorf("error (attempt %d/%d)... %v", attempt, rso.Retries, err)
}
// If this is not the last attempt, wait before retrying.
if attempt < consts.DefaultRetries {
// If this is not the last attempt, wait before retrying
if attempt < rso.Retries {
time.Sleep(time.Second * consts.RetriesInterval)
}
}
// If all attempts fail, return an error.
return fmt.Errorf("operation failed after %d attempts", consts.DefaultRetries)
// If all attempts fail, return an error
return fmt.Errorf("operation unsuccessful after %d attempts", rso.Retries)
}
func EnsureBinaryExists(ctx context.Context, bin embed.FS, ro *flags.CliRootOpts) error {