diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d989ba6..ac50546 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -222,11 +222,13 @@ jobs: run: | hauler store load --help # verify via load - hauler store load haul.tar.zst + hauler store load + # verify via load with multiple files + hauler store load --filename haul.tar.zst --filename store.tar.zst # verify via load with filename and temp directory - hauler store load store.tar.zst --tempdir /opt + hauler store load --filename store.tar.zst --tempdir /opt # verify via load with filename and platform (amd64) - hauler store load store-amd64.tar.zst + hauler store load --filename store-amd64.tar.zst - name: Verify Hauler Store Contents run: | diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 75ca061..01d4274 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -90,7 +90,7 @@ func addStoreLoad(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman cmd := &cobra.Command{ Use: "load", Short: "Load a content store from a store archive", - Args: cobra.MinimumNArgs(1), + Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -100,7 +100,7 @@ func addStoreLoad(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman } _ = s - return store.LoadCmd(ctx, o, args...) + return store.LoadCmd(ctx, o, rso, ro) }, } o.AddFlags(cmd) diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index fbee6ae..c93977a 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -12,24 +12,13 @@ import ( "hauler.dev/go/hauler/pkg/store" ) -// LoadCmd -// TODO: Just use mholt/archiver for now, even though we don't need most of it -func LoadCmd(ctx context.Context, o *flags.LoadOpts, archiveRefs ...string) error { +// extracts the contents of an archived oci layout to an existing oci layout +func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { l := log.FromContext(ctx) - storeDir := o.StoreDir - - if storeDir == "" { - storeDir = os.Getenv(consts.HaulerStoreDir) - } - - if storeDir == "" { - storeDir = consts.DefaultStoreName - } - - for _, archiveRef := range archiveRefs { - l.Infof("loading content from [%s] to [%s]", archiveRef, storeDir) - err := unarchiveLayoutTo(ctx, archiveRef, storeDir, o.TempOverride) + for _, fileName := range o.FileName { + l.Infof("loading haul [%s] to [%s]", o.FileName, o.StoreDir) + err := unarchiveLayoutTo(ctx, fileName, o.StoreDir, o.TempOverride) if err != nil { return err } @@ -38,23 +27,28 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, archiveRefs ...string) erro return nil } -// unarchiveLayoutTo accepts an archived oci layout and extracts the contents to an existing oci layout, preserving the index -func unarchiveLayoutTo(ctx context.Context, archivePath string, dest string, tempOverride string) error { +// unarchiveLayoutTo accepts an archived OCI layout, extracts the contents to an existing OCI layout, and preserves the index +func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempOverride string) error { l := log.FromContext(ctx) - if tempOverride == "" { - tempOverride = os.Getenv(consts.HaulerTempDir) + var tempDir string + + if tempOverride != "" { + tempDir = tempOverride + } else { + + parent := os.Getenv(consts.HaulerTempDir) + var err error + tempDir, err = os.MkdirTemp(parent, consts.DefaultHaulerTempDirName) + if err != nil { + return err + } + defer os.RemoveAll(tempDir) } - tempDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName) - if err != nil { - return err - } - defer os.RemoveAll(tempDir) + l.Debugf("using temporary directory [%s]", tempDir) - l.Debugf("using temporary directory at [%s]", tempDir) - - if err := archives.Unarchive(ctx, archivePath, tempDir); err != nil { + if err := archives.Unarchive(ctx, haulPath, tempDir); err != nil { return err } diff --git a/internal/flags/load.go b/internal/flags/load.go index adefc1b..66037e7 100644 --- a/internal/flags/load.go +++ b/internal/flags/load.go @@ -1,18 +1,21 @@ package flags -import "github.com/spf13/cobra" +import ( + "github.com/spf13/cobra" + "hauler.dev/go/hauler/pkg/consts" +) type LoadOpts struct { *StoreRootOpts + FileName []string TempOverride string } func (o *LoadOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() - // On Unix systems, the default is $TMPDIR if non-empty, else /tmp. - // On Windows, the default is GetTempPath, returning the first non-empty - // value from %TMP%, %TEMP%, %USERPROFILE%, or the Windows directory. - // On Plan 9, the default is /tmp. + // On Unix systems, the default is $TMPDIR if non-empty, else /tmp + // On Windows, the default is GetTempPath, returning the first value from %TMP%, %TEMP%, %USERPROFILE%, or Windows directory + f.StringSliceVarP(&o.FileName, "filename", "f", []string{consts.DefaultHaulerArchiveName}, "Specify the name of haul(s) to sync") f.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directiory determined by the OS") } diff --git a/internal/flags/save.go b/internal/flags/save.go index a447d76..3ff531b 100644 --- a/internal/flags/save.go +++ b/internal/flags/save.go @@ -14,6 +14,6 @@ type SaveOpts struct { func (o *SaveOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() - f.StringVarP(&o.FileName, "filename", "f", consts.DefaultHaulArchiveName, "(Optional) Specify the name of outputted archive") + f.StringVarP(&o.FileName, "filename", "f", consts.DefaultHaulerArchiveName, "(Optional) Specify the name of inputted haul(s)") f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform for runtime imports... i.e. linux/amd64 (unspecified implies all)") } diff --git a/pkg/archives/archiver.go b/pkg/archives/archiver.go index 5bf2d02..bedbd8f 100644 --- a/pkg/archives/archiver.go +++ b/pkg/archives/archiver.go @@ -10,7 +10,7 @@ import ( "hauler.dev/go/hauler/pkg/log" ) -// Maps to handle compression and archival types +// maps to handle compression types var CompressionMap = map[string]archives.Compression{ "gz": archives.Gz{}, "bz2": archives.Bz2{}, @@ -20,6 +20,7 @@ var CompressionMap = map[string]archives.Compression{ "br": archives.Brotli{}, } +// maps to handle archival types var ArchivalMap = map[string]archives.Archival{ "tar": archives.Tar{}, "zip": archives.Zip{}, @@ -31,31 +32,31 @@ func isExist(path string) bool { return !os.IsNotExist(statErr) } -// Archive is a function that archives the files in a directory +// archives the files in a directory // dir: the directory to Archive // outfile: the output file // compression: the compression to use (gzip, bzip2, etc.) // archival: the archival to use (tar, zip, etc.) func Archive(ctx context.Context, dir, outfile string, compression archives.Compression, archival archives.Archival) error { l := log.FromContext(ctx) - l.Debugf("Starting the archival process for directory: %s", dir) + l.Debugf("starting the archival process for [%s]", dir) // remove outfile - l.Debugf("Removing any existing output file: %s", outfile) + l.Debugf("removing existing output file: [%s]", outfile) if err := os.RemoveAll(outfile); err != nil { - errMsg := fmt.Errorf("failed to remove existing output file '%s': %w", outfile, err) + errMsg := fmt.Errorf("failed to remove existing output file [%s]: %w", outfile, err) l.Debugf(errMsg.Error()) return errMsg } if !isExist(dir) { - errMsg := fmt.Errorf("directory '%s' does not exist, cannot proceed with archival", dir) + errMsg := fmt.Errorf("directory [%s] does not exist, cannot proceed with archival", dir) l.Debugf(errMsg.Error()) return errMsg } // map files on disk to their paths in the archive - l.Debugf("Mapping files in directory: %s", dir) + l.Debugf("mapping files in directory [%s]", dir) archiveDirName := filepath.Base(filepath.Clean(dir)) if dir == "." { archiveDirName = "" @@ -64,40 +65,40 @@ func Archive(ctx context.Context, dir, outfile string, compression archives.Comp dir: archiveDirName, }) if err != nil { - errMsg := fmt.Errorf("error mapping files from directory '%s': %w", dir, err) + errMsg := fmt.Errorf("error mapping files from directory [%s]: %w", dir, err) l.Debugf(errMsg.Error()) return errMsg } - l.Debugf("Successfully mapped files for directory: %s", dir) + l.Debugf("successfully mapped files for directory [%s]", dir) // create the output file we'll write to - l.Debugf("Creating output file: %s", outfile) + l.Debugf("creating output file [%s]", outfile) outf, err := os.Create(outfile) if err != nil { - errMsg := fmt.Errorf("error creating output file '%s': %w", outfile, err) + errMsg := fmt.Errorf("error creating output file [%s]: %w", outfile, err) l.Debugf(errMsg.Error()) return errMsg } defer func() { - l.Debugf("Closing output file: %s", outfile) + l.Debugf("closing output file [%s]", outfile) outf.Close() }() // define the archive format - l.Debugf("Defining the archive format with compression: %T and archival: %T", compression, archival) + l.Debugf("defining the archive format: [%T]/[%T]", archival, compression) format := archives.CompressedArchive{ Compression: compression, Archival: archival, } // create the archive - l.Debugf("Starting archive creation: %s", outfile) + l.Debugf("starting archive for [%s]", outfile) err = format.Archive(context.Background(), outf, files) if err != nil { - errMsg := fmt.Errorf("error during archive creation for output file '%s': %w", outfile, err) + errMsg := fmt.Errorf("error during archive creation for output file [%s]: %w", outfile, err) l.Debugf(errMsg.Error()) return errMsg } - l.Debugf("Archive created successfully: %s", outfile) + l.Debugf("archive created successfully [%s]", outfile) return nil } diff --git a/pkg/archives/unarchiver.go b/pkg/archives/unarchiver.go index eb225d0..722f195 100644 --- a/pkg/archives/unarchiver.go +++ b/pkg/archives/unarchiver.go @@ -13,14 +13,14 @@ import ( ) const ( - dirPermissions = 0o700 // Default directory permissions - filePermissions = 0o600 // Default file permissions + dirPermissions = 0o700 // default directory permissions + filePermissions = 0o600 // default file permissions ) -// securePath ensures the path is safely relative to the target directory. +// ensures the path is safely relative to the target directory func securePath(basePath, relativePath string) (string, error) { - relativePath = filepath.Clean("/" + relativePath) // Normalize path with a leading slash - relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator)) // Remove leading separator + relativePath = filepath.Clean("/" + relativePath) + relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator)) dstPath := filepath.Join(basePath, relativePath) @@ -30,110 +30,110 @@ func securePath(basePath, relativePath string) (string, error) { return dstPath, nil } -// createDirWithPermissions creates a directory with specified permissions. +// creates a directory with specified permissions func createDirWithPermissions(ctx context.Context, path string, mode os.FileMode) error { l := log.FromContext(ctx) - l.Debugf("Creating directory: %s", path) + l.Debugf("creating directory [%s]", path) if err := os.MkdirAll(path, mode); err != nil { - return fmt.Errorf("mkdir: %w", err) + return fmt.Errorf("failed to mkdir: %w", err) } return nil } -// setPermissions applies permissions to a file or directory. +// sets permissions to a file or directory func setPermissions(path string, mode os.FileMode) error { if err := os.Chmod(path, mode); err != nil { - return fmt.Errorf("chmod: %w", err) + return fmt.Errorf("failed to chmod: %w", err) } return nil } -// handleFile handles the extraction of a file from the archive. +// handles the extraction of a file from the archive. func handleFile(ctx context.Context, f archives.FileInfo, dst string) error { l := log.FromContext(ctx) - l.Debugf("Handling file: %s", f.NameInArchive) + l.Debugf("handling file [%s]", f.NameInArchive) - // Validate and construct the destination path + // validate and construct the destination path dstPath, pathErr := securePath(dst, f.NameInArchive) if pathErr != nil { return pathErr } - // Ensure the parent directory exists + // ensure the parent directory exists parentDir := filepath.Dir(dstPath) if dirErr := createDirWithPermissions(ctx, parentDir, dirPermissions); dirErr != nil { return dirErr } - // Handle directories + // handle directories if f.IsDir() { - // Create the directory with permissions from the archive + // create the directory with permissions from the archive if dirErr := createDirWithPermissions(ctx, dstPath, f.Mode()); dirErr != nil { - return fmt.Errorf("creating directory: %w", dirErr) + return fmt.Errorf("failed to create directory: %w", dirErr) } - l.Debugf("Successfully created directory: %s", dstPath) + l.Debugf("successfully created directory [%s]", dstPath) return nil } - // Ignore symlinks (or hardlinks) + // ignore symlinks (or hardlinks) if f.LinkTarget != "" { - l.Debugf("Skipping symlink: %s -> %s", dstPath, f.LinkTarget) + l.Debugf("skipping symlink [%s] to [%s]", dstPath, f.LinkTarget) return nil } - // Check and handle parent directory permissions + // check and handle parent directory permissions originalMode, statErr := os.Stat(parentDir) if statErr != nil { - return fmt.Errorf("stat parent directory: %w", statErr) + return fmt.Errorf("failed to stat parent directory: %w", statErr) } - // If parent directory is read-only, temporarily make it writable + // if parent directory is read only, temporarily make it writable if originalMode.Mode().Perm()&0o200 == 0 { - l.Debugf("Parent directory is read-only, temporarily making it writable: %s", parentDir) + l.Debugf("parent directory is read only... temporarily making it writable [%s]", parentDir) if chmodErr := os.Chmod(parentDir, originalMode.Mode()|0o200); chmodErr != nil { - return fmt.Errorf("chmod parent directory: %w", chmodErr) + return fmt.Errorf("failed to chmod parent directory: %w", chmodErr) } defer func() { - // Restore the original permissions after writing + // restore the original permissions after writing if chmodErr := os.Chmod(parentDir, originalMode.Mode()); chmodErr != nil { - l.Debugf("Failed to restore original permissions for %s: %v", parentDir, chmodErr) + l.Debugf("failed to restore original permissions for [%s]: %v", parentDir, chmodErr) } }() } - // Handle regular files + // handle regular files reader, openErr := f.Open() if openErr != nil { - return fmt.Errorf("open file: %w", openErr) + return fmt.Errorf("failed to open file: %w", openErr) } defer reader.Close() dstFile, createErr := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY, f.Mode()) if createErr != nil { - return fmt.Errorf("create file: %w", createErr) + return fmt.Errorf("failed to create file: %w", createErr) } defer dstFile.Close() if _, copyErr := io.Copy(dstFile, reader); copyErr != nil { - return fmt.Errorf("copy: %w", copyErr) + return fmt.Errorf("failed to copy: %w", copyErr) } - l.Debugf("Successfully extracted file: %s", dstPath) + l.Debugf("successfully extracted file [%s]", dstPath) return nil } -// Unarchive unarchives a tarball to a directory, symlinks and hardlinks are ignored. +// unarchives a tarball to a directory, symlinks, and hardlinks are ignored func Unarchive(ctx context.Context, tarball, dst string) error { l := log.FromContext(ctx) - l.Debugf("Unarchiving %s to %s", tarball, dst) + l.Debugf("unarchiving temporary archive [%s] to temporary store [%s]", tarball, dst) archiveFile, openErr := os.Open(tarball) if openErr != nil { - return fmt.Errorf("open tarball %s: %w", tarball, openErr) + return fmt.Errorf("failed to open tarball %s: %w", tarball, openErr) } defer archiveFile.Close() format, input, identifyErr := archives.Identify(context.Background(), tarball, archiveFile) if identifyErr != nil { - return fmt.Errorf("identify format: %w", identifyErr) + return fmt.Errorf("failed to identify format: %w", identifyErr) } extractor, ok := format.(archives.Extractor) @@ -142,7 +142,7 @@ func Unarchive(ctx context.Context, tarball, dst string) error { } if dirErr := createDirWithPermissions(ctx, dst, dirPermissions); dirErr != nil { - return fmt.Errorf("creating destination directory: %w", dirErr) + return fmt.Errorf("failed to create destination directory: %w", dirErr) } handler := func(ctx context.Context, f archives.FileInfo) error { @@ -150,9 +150,9 @@ func Unarchive(ctx context.Context, tarball, dst string) error { } if extractErr := extractor.Extract(context.Background(), input, handler); extractErr != nil { - return fmt.Errorf("extracting files: %w", extractErr) + return fmt.Errorf("failed to extract: %w", extractErr) } - l.Debugf("Unarchiving completed successfully.") + l.Infof("unarchiving completed successfully") return nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 123f9c5..a40b03d 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -83,7 +83,7 @@ const ( DefaultFileserverRootDir = "fileserver" DefaultFileserverPort = 8080 DefaultFileserverTimeout = 60 - DefaultHaulArchiveName = "haul.tar.zst" + DefaultHaulerArchiveName = "haul.tar.zst" DefaultHaulerManifestName = "hauler-manifest.yaml" DefaultRetries = 3 RetriesInterval = 5