From 06687f3f01cadd14895712ad31f6f2f5cbcbb536 Mon Sep 17 00:00:00 2001 From: Matt Nikkel Date: Fri, 16 Apr 2021 12:16:30 -0400 Subject: [PATCH] WIP - add partial archive format detection, begin archive.Writer implementation --- cmd/haulerctl/app/deploy.go | 91 +++++++++++++++++++++++- cmd/haulerctl/app/outputformat.go | 37 ++++++++++ cmd/haulerctl/app/outputformat_string.go | 25 +++++++ cmd/haulerctl/app/package.go | 79 +++++++++++++------- pkg/archive/kind.go | 15 ++++ pkg/archive/package_kind.go | 20 ++++++ pkg/archive/writer.go | 48 +++++-------- pkg/deployer/doc.go | 5 ++ pkg/packager_new/packager.go | 2 +- 9 files changed, 266 insertions(+), 56 deletions(-) create mode 100644 cmd/haulerctl/app/outputformat.go create mode 100644 cmd/haulerctl/app/outputformat_string.go create mode 100644 pkg/archive/kind.go create mode 100644 pkg/archive/package_kind.go create mode 100644 pkg/deployer/doc.go diff --git a/cmd/haulerctl/app/deploy.go b/cmd/haulerctl/app/deploy.go index aff3911..ae563c2 100644 --- a/cmd/haulerctl/app/deploy.go +++ b/cmd/haulerctl/app/deploy.go @@ -1,10 +1,27 @@ package app import ( + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/rancherfederal/hauler/pkg/archive" + "github.com/spf13/cobra" ) +const ( + archiveFileNameFlag = "archive-file" + archiveFileNameShorthand = "f" + archiveFileNameDefault = "" + archiveFormatFlag = "archive-format" +) + func NewDeployCommand() *cobra.Command { + opts := &DeployOptions{} + cmd := &cobra.Command{ Use: "deploy", Short: "deploy all dependencies from a generated package", @@ -12,20 +29,92 @@ func NewDeployCommand() *cobra.Command { Given an archive generated from the package command, deploy all needed components to serve packaged dependencies.`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := opts.Preprocess(args); err != nil { + return err + } + return opts.Run() + }, } + cmd.Flags().StringVarP(&opts.ArchiveFileName, + archiveFileNameFlag, archiveFileNameShorthand, archiveFileNameDefault, + "specify the archive to deploy; - reads from stdin", + ) + cmd.Flags().Var(&opts.ArchiveFormat, + archiveFormatFlag, + "specify the format of the target archive (TarGZ, Tar); if unset, will auto-complete based on "+archiveFileNameFlag, + ) + return cmd } type DeployOptions struct { + ArchiveFileName string + ArchiveFormat OutputFormat + // UseRPMs bool + // SELinux bool + + co *completedDeployOptions +} + +// TODO - decide if "frozen" options from DeployOptions should be stored in completedDeployOptions + +type completedDeployOptions struct { + ArchiveKind archive.Kind + + Src io.Reader } // Preprocess infers any remaining options and performs any required validation. -func (o *DeployOptions) Preprocess() error { +func (o *DeployOptions) Preprocess(_ []string) error { + // TODO - perform as much validation as possible and return error containing all known issues + + co := &completedDeployOptions{} + + if o.ArchiveFileName == "" { + return errors.New("archive file is required") + } + if o.ArchiveFileName == "-" && o.ArchiveFormat == UnknownFormat { + return errors.New("must specify a format when reading from stdin") + } + + if o.ArchiveFileName == "-" { + co.Src = os.Stdin + } else { + if srcFile, err := os.Open(o.ArchiveFileName); err != nil { + return fmt.Errorf( + "couldn't read archive file %s: %v", + o.ArchiveFileName, err, + ) + } else { + co.Src = srcFile + } + } + + // TODO - improve scalability of format auto-detections + switch { + case o.ArchiveFileName == "-" || o.ArchiveFormat != UnknownFormat: + co.ArchiveKind = o.ArchiveFormat.ToArchiveKind() + case strings.HasSuffix(o.ArchiveFileName, ".tar"): + co.ArchiveKind = archive.KindTar + case strings.HasSuffix(o.ArchiveFileName, ".tar.gz") || strings.HasSuffix(o.ArchiveFileName, ".tgz"): + co.ArchiveKind = archive.KindTarGz + default: + return errors.New("unable to determine archive format, please specify flag or allow auto-detection by using known file type") + } + + o.co = co return nil } // Run performs the operation. func (o *DeployOptions) Run() error { + if o.co == nil { + return errors.New("DeployOptions must be preprocessed before Run is called") + } + + // TODO - use deployer and actually deploy! + return nil } diff --git a/cmd/haulerctl/app/outputformat.go b/cmd/haulerctl/app/outputformat.go new file mode 100644 index 0000000..b3a8b50 --- /dev/null +++ b/cmd/haulerctl/app/outputformat.go @@ -0,0 +1,37 @@ +package app + +import ( + "errors" + + "github.com/rancherfederal/hauler/pkg/archive" +) + +type OutputFormat int + +//go:generate stringer -type=OutputFormat +const ( + UnknownFormat = OutputFormat(0) + TarGz = OutputFormat(archive.KindTarGz) + Tar = OutputFormat(archive.KindTar) +) + +func (i *OutputFormat) ToArchiveKind() archive.Kind { + return archive.Kind(*i) +} + +func (i *OutputFormat) Set(s string) error { + switch s { + case "TarGz": + *i = TarGz + case "Tar": + *i = Tar + default: + return errors.New("unknown format") + } + return nil + +} + +func (i OutputFormat) Type() string { + return "OutputFormat" +} diff --git a/cmd/haulerctl/app/outputformat_string.go b/cmd/haulerctl/app/outputformat_string.go new file mode 100644 index 0000000..fb28a4b --- /dev/null +++ b/cmd/haulerctl/app/outputformat_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=OutputFormat"; DO NOT EDIT. + +package app + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[UnknownFormat-0] + _ = x[TarGz-2] + _ = x[Tar-1] +} + +const _OutputFormat_name = "UnknownFormatTarTarGz" + +var _OutputFormat_index = [...]uint8{0, 13, 16, 21} + +func (i OutputFormat) String() string { + if i < 0 || i >= OutputFormat(len(_OutputFormat_index)-1) { + return "OutputFormat(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _OutputFormat_name[_OutputFormat_index[i]:_OutputFormat_index[i+1]] +} diff --git a/cmd/haulerctl/app/package.go b/cmd/haulerctl/app/package.go index 1290f4c..e3ad01a 100644 --- a/cmd/haulerctl/app/package.go +++ b/cmd/haulerctl/app/package.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "os" + "strings" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" "github.com/rancherfederal/hauler/pkg/archive" @@ -18,8 +19,10 @@ import ( const ( packageConfigFileNameFlag = "package-config" packageConfigFileNameDefault = "" - outputFileNameFlag = "out-file" + outputFileNameFlag = "output-file" + outputFileNameShorthand = "f" outputFileNameDefault = "hauler-archive.tar.gz" + outputFormatFlag = "output-format" ) func NewPackageCommand() *cobra.Command { @@ -32,7 +35,7 @@ func NewPackageCommand() *cobra.Command { Container images, git repositories, and more, packaged and ready to be served within an air gap.`, RunE: func(cmd *cobra.Command, args []string) error { - if err := opts.Preprocess(); err != nil { + if err := opts.Preprocess(args); err != nil { return err } return opts.Run() @@ -41,13 +44,19 @@ Container images, git repositories, and more, packaged and ready to be served wi // TODO - set EnvConfig options through CLI - cmd.Flags().StringVar( - &opts.PackageConfigFileName, packageConfigFileNameFlag, packageConfigFileNameDefault, + cmd.Flags().StringVar(&opts.PackageConfigFileName, + packageConfigFileNameFlag, packageConfigFileNameDefault, "package config YAML used for creating archive", ) - cmd.Flags().StringVar( - &opts.OutputFileName, outputFileNameFlag, outputFileNameDefault, - "specify the package's output location; '-' writes to standard out", + // TODO - determine if OutputFileName should be positional arg or flag + cmd.Flags().StringVarP(&opts.OutputFileName, + outputFileNameFlag, outputFileNameShorthand, outputFileNameDefault, + "specify the package's output location; - writes to stdout", + ) + // TODO - improve usage message, dynamically populate all formats for easier future additions + cmd.Flags().Var(&opts.OutputFormat, + outputFormatFlag, + "choose the format of the outputted archive (TarGz, Tar); if unset, will auto-complete based on "+outputFileNameFlag, ) return cmd @@ -56,6 +65,7 @@ Container images, git repositories, and more, packaged and ready to be served wi type PackageOptions struct { PackageConfigFileName string OutputFileName string + OutputFormat OutputFormat // ImageLists []string // ImageArchives []string @@ -63,14 +73,17 @@ type PackageOptions struct { co *completedPackageOptions } +// TODO - decide if "frozen" options from PackageOptions should be stored in completedPackageOptions + type completedPackageOptions struct { PackageConfig v1alpha1.PackageConfig - OutputFileName string - OutputArchiveKind archive.WriterKind + OutputArchiveKind archive.Kind + + Dst io.Writer } // Preprocess infers any remaining options and performs any required validation. -func (o *PackageOptions) Preprocess() error { +func (o *PackageOptions) Preprocess(_ []string) error { // TODO - perform as much validation as possible and return error containing all known issues co := &completedPackageOptions{} @@ -81,6 +94,9 @@ func (o *PackageOptions) Preprocess() error { if o.OutputFileName == "" { return errors.New("output file is required") } + if o.OutputFileName == "-" && o.OutputFormat == UnknownFormat { + return errors.New("must specify a format when outputting to stdout") + } pconfigBytes, err := ioutil.ReadFile(o.PackageConfigFileName) if err != nil { @@ -97,9 +113,33 @@ func (o *PackageOptions) Preprocess() error { o.PackageConfigFileName, err, ) } - co.PackageConfig = pconfig + if o.OutputFileName == "-" { + co.Dst = os.Stdout + } else { + if dstFile, err := os.Create(o.OutputFileName); err != nil { + return fmt.Errorf( + "couldn't create output file %s: %v", + o.OutputFileName, err, + ) + } else { + co.Dst = dstFile + } + } + + // TODO - improve scalability of format auto-detection + switch { + case o.OutputFileName == "-" || o.OutputFormat != UnknownFormat: + co.OutputArchiveKind = o.OutputFormat.ToArchiveKind() + case strings.HasSuffix(o.OutputFileName, ".tar"): + co.OutputArchiveKind = archive.KindTar + case strings.HasSuffix(o.OutputFileName, ".tar.gz") || strings.HasSuffix(o.OutputFileName, ".tgz"): + co.OutputArchiveKind = archive.KindTarGz + default: + return errors.New("unable to determine output format, please specify flag or allow auto-detection by using known file type") + } + o.co = co return nil } @@ -107,24 +147,13 @@ func (o *PackageOptions) Preprocess() error { // Run performs the operation. func (o *PackageOptions) Run() error { if o.co == nil { - return errors.New("package options must be preprocessed before Run is called") - } - - var dst io.Writer - if o.OutputFileName == "-" { - dst = os.Stdout - } else { - dstFile, err := os.Create(o.OutputFileName) - if err != nil { - return fmt.Errorf("create output file: %v", err) - } - dst = dstFile + return errors.New("PackageOptions must be preprocessed before Run is called") } // TODO - set EnvConfig options through CLI p := packager.New(nil) - - if err := p.Package(dst, o.co.PackageConfig); err != nil { + // TODO - use o.co.OutputArchiveKind + if err := p.Package(o.co.Dst, o.co.PackageConfig); err != nil { return err } diff --git a/pkg/archive/kind.go b/pkg/archive/kind.go new file mode 100644 index 0000000..d8196cd --- /dev/null +++ b/pkg/archive/kind.go @@ -0,0 +1,15 @@ +package archive + +import ( + "errors" +) + +type Kind int + +const ( + KindUnknown Kind = iota + KindTar + KindTarGz +) + +var ErrNoWriterKind = errors.New("no Kind specified for Writer") diff --git a/pkg/archive/package_kind.go b/pkg/archive/package_kind.go new file mode 100644 index 0000000..42a25f1 --- /dev/null +++ b/pkg/archive/package_kind.go @@ -0,0 +1,20 @@ +package archive + +import ( + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" +) + +var packageKindMap = map[v1alpha1.PackageType]string{ + v1alpha1.PackageTypeK3s: "k3s", + v1alpha1.PackageTypeContainerImages: "containers", + v1alpha1.PackageTypeGitRepository: "git", + v1alpha1.PackageTypeFileTree: "files", +} + +var packageStringMap map[string]v1alpha1.PackageType + +func init() { + for k, v := range packageKindMap { + packageStringMap[v] = k + } +} diff --git a/pkg/archive/writer.go b/pkg/archive/writer.go index bbaaeba..ffdd7e7 100644 --- a/pkg/archive/writer.go +++ b/pkg/archive/writer.go @@ -14,16 +14,6 @@ import ( "sigs.k8s.io/yaml" ) -type WriterKind int - -const ( - WriterKindUnknown = iota - WriterKindTar - WriterKindTarGz -) - -var ErrNoKind = errors.New("no kind specified for Writer") - type writerOptions struct { archiveName string } @@ -50,26 +40,26 @@ func WithArchiveName(name string) WriterOption { } type Writer struct { - kind WriterKind + Kind Kind tarWriter *tar.Writer gzipWriter *gzip.Writer } -func NewWriter(dst io.Writer, kind WriterKind, opts ...WriterOption) (*Writer, error) { +func NewWriter(dst io.Writer, kind Kind, opts ...WriterOption) (*Writer, error) { opt := defaultWriterOptions() for _, o := range opts { o.Apply(opt) } w := &Writer{ - kind: kind, + Kind: kind, } switch kind { - case WriterKindTar: + case KindTar: w.tarWriter = tar.NewWriter(dst) - case WriterKindTarGz: + case KindTarGz: w.gzipWriter = gzip.NewWriter(dst) w.tarWriter = tar.NewWriter(w.gzipWriter) default: @@ -93,7 +83,7 @@ func NewWriter(dst io.Writer, kind WriterKind, opts ...WriterOption) (*Writer, e } switch kind { - case WriterKindTar, WriterKindTarGz: + case KindTar, KindTarGz: archiveHeader := &tar.Header{ Typeflag: tar.TypeReg, Name: "./hauler_archive.yaml", @@ -120,12 +110,12 @@ func (w *Writer) MkdirP( packageName string, path string, ) error { - switch w.kind { - case WriterKindTar, WriterKindTarGz: + switch w.Kind { + case KindTar, KindTarGz: return errors.New("unimplemented") default: - return ErrNoKind + return ErrNoWriterKind } } @@ -173,8 +163,8 @@ func (w *Writer) CreateFile( cleanFileName := "./" + path.Clean(path.Join(".", packageKind, packageName, fileName)) - switch w.kind { - case WriterKindTar, WriterKindTarGz: + switch w.Kind { + case KindTar, KindTarGz: tarHeader := &tar.Header{ Typeflag: tar.TypeReg, Name: cleanFileName, @@ -184,35 +174,35 @@ func (w *Writer) CreateFile( return w.tarWriter.WriteHeader(tarHeader) default: - return ErrNoKind + return ErrNoWriterKind } } // Write implements io.Writer func (w *Writer) Write(b []byte) (int, error) { - switch w.kind { - case WriterKindTar, WriterKindTarGz: + switch w.Kind { + case KindTar, KindTarGz: return w.tarWriter.Write(b) default: - return 0, ErrNoKind + return 0, ErrNoWriterKind } } // Close flushes and closes the underlying writers when this archive is done // being written to. func (w *Writer) Close() error { - switch w.kind { - case WriterKindTar: + switch w.Kind { + case KindTar: return w.tarWriter.Close() - case WriterKindTarGz: + case KindTarGz: if err := w.tarWriter.Close(); err != nil { return err } return w.gzipWriter.Close() default: - return ErrNoKind + return ErrNoWriterKind } } diff --git a/pkg/deployer/doc.go b/pkg/deployer/doc.go new file mode 100644 index 0000000..4bccf7a --- /dev/null +++ b/pkg/deployer/doc.go @@ -0,0 +1,5 @@ +/* +Package deployer implements a Deployer that inspects a generated archive, +installs k3s, and deploys all dependencies bundled in that archive. +*/ +package deployer diff --git a/pkg/packager_new/packager.go b/pkg/packager_new/packager.go index dc4b48d..f803bde 100644 --- a/pkg/packager_new/packager.go +++ b/pkg/packager_new/packager.go @@ -48,7 +48,7 @@ func (p *Packager) Package( ) error { // TODO - allow changing writer kind // wrap around dst - aw, err := archive.NewWriter(dst, archive.WriterKindTar) + aw, err := archive.NewWriter(dst, archive.KindTar) if err != nil { return fmt.Errorf("create archive writer: %v", err) }