github/hauler
1
0
Fork 0
mirror of https://github.com/hauler-dev/hauler.git synced 2026-02-14 18:09:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #24 from rancherfederal/dev-jp

Browse Source 
Dev jp
This commit is contained in:
joshrwolf
2021-06-17 16:13:32 -06:00
committed by GitHub
parent 1136a6ecc3 cb898136ca
commit b546d326d4
12 changed files with 250 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cmd/hauler/app/copy.go
View File

@@ -4,11 +4,19 @@ import (
"context"
"github.com/rancherfederal/hauler/pkg/oci"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
copyLong = `hauler copies artifacts stored on a registry to local disk`
copyExample = `
# Run Hauler
hauler copy locahost:5000/artifacts:latest`
)
type copyOpts struct {
*rootOpts
dir string
sourceRef string
}
@@ -16,11 +24,15 @@ type copyOpts struct {
// NewCopyCommand creates a new sub command under
// hauler for coping files to local disk
func NewCopyCommand() *cobra.Command {
opts := &copyOpts{}
opts := &copyOpts{
rootOpts: &ro,
}
cmd := &cobra.Command{
Use: "copy",
Short: "Download artifacts from OCI registry to local disk",
Long: copyLong,
Example: copyExample,
Aliases: []string{"c", "cp"},
//Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
@@ -41,7 +53,7 @@ func (o *copyOpts) Run(src string) error {
defer cancel()
if err := oci.Get(ctx, src, o.dir); err != nil {
logrus.Error(err)
o.logger.Errorf("error copy artifact %s to local directory %s: %v", src, o.dir, err)
}
return nil

9
cmd/hauler/app/relocate.go
View File

@@ -6,12 +6,15 @@ import (
type relocateOpts struct {
inputFile string
*rootOpts
}
// NewRelocateCommand creates a new sub command under
// haulterctl for relocating images and artifacts
func NewRelocateCommand() *cobra.Command {
opts := &relocateOpts{}
opts := &relocateOpts{
rootOpts: &ro,
}
cmd := &cobra.Command{
Use: "relocate",
@@ -23,10 +26,6 @@ func NewRelocateCommand() *cobra.Command {
},
}
f := cmd.PersistentFlags()
f.StringVarP(&opts.inputFile, "input", "i", "haul.tar.zst",
"package output location relative to the current directory (haul.tar.zst)")
cmd.AddCommand(NewRelocateArtifactsCommand(opts))
cmd.AddCommand(NewRelocateImagesCommand(opts))

57
cmd/hauler/app/relocate_artifacts.go
View File

@@ -2,62 +2,53 @@ package app
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"github.com/rancherfederal/hauler/pkg/oci"
"github.com/rancherfederal/hauler/pkg/packager"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
type relocateArtifactsOpts struct {
relocate *relocateOpts
destRef string
*relocateOpts
destRef string
}
var (
relocateArtifactsLong = `hauler relocate artifacts process an archive with files
to be pushed to a registry`
relocateArtifactsExample = `
# Run Hauler
hauler relocate artifacts artifacts.tar.zst locahost:5000/artifacts:latest`
)
// NewRelocateArtifactsCommand creates a new sub command of relocate for artifacts
func NewRelocateArtifactsCommand(relocate *relocateOpts) *cobra.Command {
opts := &relocateArtifactsOpts{relocate: relocate}
opts := &relocateArtifactsOpts{
relocateOpts: relocate,
}
cmd := &cobra.Command{
Use: "artifacts",
Short: "Use artifact from bundle artifacts to populate a target file server with the artifact's contents",
Use: "artifacts",
Short: "Use artifact from bundle artifacts to populate a target file server with the artifact's contents",
Long: relocateArtifactsLong,
Example: relocateArtifactsExample,
RunE: func(cmd *cobra.Command, args []string) error {
opts.destRef = args[0]
return opts.Run(opts.destRef)
opts.inputFile = args[0]
opts.destRef = args[1]
return opts.Run(opts.destRef, opts.inputFile)
},
}
return cmd
}
func (o *relocateArtifactsOpts) Run(dst string) error {
func (o *relocateArtifactsOpts) Run(dst string, input string) error {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
ar := packager.NewArchiver()
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
logrus.Error(err)
}
packager.Unpackage(ar, o.relocate.inputFile, tmpdir)
files, err := ioutil.ReadDir(tmpdir)
if err != nil {
logrus.Error(err)
}
for _, f := range files {
if err := oci.Put(ctx, filepath.Join(tmpdir, f.Name()), dst); err != nil {
logrus.Error(err)
}
if err := oci.Put(ctx, input, dst); err != nil {
o.logger.Errorf("error pushing artifact to registry %s: %v", dst, err)
}
return nil

53
cmd/hauler/app/relocate_images.go
View File

@@ -1,7 +1,6 @@
package app
import (
"fmt"
"os"
"path/filepath"
"strings"
@@ -14,49 +13,62 @@ import (
"github.com/spf13/cobra"
)
var (
relocateImagesLong = `hauler relocate images processes a bundle provides by hauler
package build and copies all of the collected images to a registry`
relocateImagesExample = `
# Run Hauler
hauler relocate images pkg.tar.zst locahost:5000`
)
type relocateImagesOpts struct {
relocate *relocateOpts
destRef string
*relocateOpts
destRef string
}
// NewRelocateImagesCommand creates a new sub command of relocate for images
func NewRelocateImagesCommand(relocate *relocateOpts) *cobra.Command {
opts := &relocateImagesOpts{relocate: relocate}
opts := &relocateImagesOpts{
relocateOpts: relocate,
}
cmd := &cobra.Command{
Use: "images",
Short: "Use artifact from bundle images to populate a target registry with the artifact's images",
Use: "images",
Short: "Use artifact from bundle images to populate a target registry with the artifact's images",
Long: relocateImagesLong,
Example: relocateImagesExample,
RunE: func(cmd *cobra.Command, args []string) error {
opts.destRef = args[0]
return opts.Run(opts.destRef)
opts.inputFile = args[0]
opts.destRef = args[1]
return opts.Run(opts.destRef, opts.inputFile)
},
}
return cmd
}
func (o *relocateImagesOpts) Run(dst string) error {
ar := packager.NewArchiver()
func (o *relocateImagesOpts) Run(dst string, input string) error {
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
return err
}
o.logger.Debugf("Using temporary working directory: %s", tmpdir)
packager.Unpackage(ar, o.relocate.inputFile, tmpdir)
a := packager.NewArchiver()
if err != nil {
return err
if err := packager.Unpackage(a, input, tmpdir); err != nil {
o.logger.Errorf("error unpackaging input %s: %v", input, err)
}
o.logger.Debugf("Unpackaged %s", input)
path := filepath.Join(tmpdir, "layout")
ly, err := layout.FromPath(path)
if err != nil {
return err
o.logger.Errorf("error creating OCI layout: %v", err)
}
for nm, hash := range oci.ListImages(ly) {
@@ -65,11 +77,10 @@ func (o *relocateImagesOpts) Run(dst string) error {
img, err := ly.Image(hash)
fmt.Printf("Copy %s to %s", n[1], dst)
fmt.Println()
o.logger.Infof("Copy %s to %s", n[1], dst)
if err != nil {
return err
o.logger.Errorf("error creating image from layout: %v", err)
}
dstimg := dst + "/" + n[1]
@@ -77,11 +88,11 @@ func (o *relocateImagesOpts) Run(dst string) error {
tag, err := name.ParseReference(dstimg)
if err != nil {
return err
o.logger.Errorf("err parsing destination image %s: %v", dstimg, err)
}
if err := remote.Write(tag, img); err != nil {
return err
o.logger.Errorf("error writing image to destination registry %s: %v", dst, err)
}
}

11
cmd/hauler/app/root.go
View File

@@ -2,11 +2,12 @@ package app
import (
"fmt"
"github.com/rancherfederal/hauler/pkg/log"
"io"
"os"
"time"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/spf13/cobra"
homedir "github.com/mitchellh/go-homedir"
@@ -30,9 +31,9 @@ var (
hauler pkg run pkg.tar.zst
hauler bundle images <images>
hauler bundle artifacts <artfiacts>
hauler relocate artifacts -i <package-name>
hauler relocate images -i <package-name> locahost:5000
hauler copy`
hauler relocate artifacts artifacts.tar.zst
hauler relocate images pkg.tar.zst locahost:5000
hauler copy local:5000/artifacts:latest`
)
type rootOpts struct {
@@ -72,7 +73,7 @@ func NewRootCommand() *cobra.Command {
cmd.AddCommand(NewPkgCommand())
f := cmd.PersistentFlags()
f.StringVarP(&loglevel, "loglevel", "l", "info",
f.StringVarP(&loglevel, "loglevel", "l", "debug",
"Log level (debug, info, warn, error, fatal, panic)")
f.StringVarP(&cfgFile, "config", "c", "./hauler.yaml",
"config file (./hauler.yaml)")

8
pkg/bootstrap/booter.go
View File

@@ -12,7 +12,6 @@ import (
"github.com/rancherfederal/hauler/pkg/log"
"helm.sh/helm/v3/pkg/chart/loader"
"io"
"k8s.io/cli-runtime/pkg/genericclioptions"
"os"
"path/filepath"
)
@@ -105,8 +104,7 @@ func (b booter) Boot(ctx context.Context, d driver.Driver) error {
func (b booter) PostBoot(ctx context.Context, d driver.Driver) error {
b.logger.Infof("Beginning post boot")
cf := genericclioptions.NewConfigFlags(true)
cf.KubeConfig = stringptr(d.KubeConfigPath())
cf := NewBootConfig("fleet-system", d.KubeConfigPath())
fleetCrdChartPath := b.fs.Chart().Path(fmt.Sprintf("fleet-crd-%s.tgz", b.Package.Spec.Fleet.VLess()))
fleetCrdChart, err := loader.Load(fleetCrdChartPath)
@@ -115,7 +113,7 @@ func (b booter) PostBoot(ctx context.Context, d driver.Driver) error {
}
b.logger.Infof("Installing fleet crds")
fleetCrdRelease, fleetCrdErr := installChart(cf, fleetCrdChart, "fleet-crd", "fleet-system", nil, b.logger)
fleetCrdRelease, fleetCrdErr := installChart(cf, fleetCrdChart, "fleet-crd", nil, b.logger)
if fleetCrdErr != nil {
return fleetCrdErr
}
@@ -129,7 +127,7 @@ func (b booter) PostBoot(ctx context.Context, d driver.Driver) error {
}
b.logger.Infof("Installing fleet")
fleetRelease, fleetErr := installChart(cf, fleetChart, "fleet", "fleet-system", nil, b.logger)
fleetRelease, fleetErr := installChart(cf, fleetChart, "fleet", nil, b.logger)
if fleetErr != nil {
return fleetErr
}

29
pkg/bootstrap/config.go Normal file
View File

@@ -0,0 +1,29 @@
package bootstrap
import (
"k8s.io/cli-runtime/pkg/genericclioptions"
)
type BootSettings struct {
config *genericclioptions.ConfigFlags
Namespace string
KubeConfig string
}
func NewBootConfig(ns, kubepath string) *BootSettings {
env := &BootSettings{
Namespace: ns,
KubeConfig: kubepath,
}
env.config = &genericclioptions.ConfigFlags{
Namespace: &env.Namespace,
KubeConfig: &env.KubeConfig,
}
return env
}
// RESTClientGetter gets the kubeconfig from BootSettings
func (s *BootSettings) RESTClientGetter() genericclioptions.RESTClientGetter {
return s.config
}

20
pkg/bootstrap/config_test.go Normal file
View File

@@ -0,0 +1,20 @@
package bootstrap
import (
"testing"
)
func TestBootSettings(t *testing.T) {
ns := "test"
kpath := "somepath"
settings := NewBootConfig(ns, kpath)
if settings.Namespace != ns {
t.Errorf("expected namespace %q, got %q", ns, settings.Namespace)
}
if settings.KubeConfig != kpath {
t.Errorf("expected kube-config %q, got %q", kpath, settings.KubeConfig)
}
}

10
pkg/bootstrap/kube.go
View File

@@ -9,7 +9,6 @@ import (
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/release"
"k8s.io/cli-runtime/pkg/genericclioptions"
"os"
"time"
)
@@ -46,9 +45,9 @@ func waitForDriver(ctx context.Context, d driver.Driver) error {
}
//TODO: This is likely way too fleet specific
func installChart(cf *genericclioptions.ConfigFlags, chart *chart.Chart, releaseName, namespace string, vals map[string]interface{}, logger log.Logger) (*release.Release, error) {
func installChart(cf *BootSettings, chart *chart.Chart, releaseName string, vals map[string]interface{}, logger log.Logger) (*release.Release, error) {
actionConfig := new(action.Configuration)
if err := actionConfig.Init(cf, namespace, os.Getenv("HELM_DRIVER"), logger.Debugf); err != nil {
if err := actionConfig.Init(cf.RESTClientGetter(), cf.Namespace, os.Getenv("HELM_DRIVER"), logger.Debugf); err != nil {
return nil, err
}
@@ -58,10 +57,7 @@ func installChart(cf *genericclioptions.ConfigFlags, chart *chart.Chart, release
client.Wait = true
//TODO: Do this better
client.Namespace, cf.Namespace = namespace, stringptr(namespace)
client.Namespace = cf.Namespace
return client.Run(chart, vals)
}
//still can't figure out why helm does it this way
func stringptr(val string) *string { return &val }

53
pkg/oci/layout_test.go Normal file
View File

@@ -0,0 +1,53 @@
package oci
import (
"fmt"
"testing"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/random"
)
func Test_ListImages(t *testing.T) {
img, err := random.Image(1024, 5)
if err != nil {
fmt.Printf("error creating test image: %v", err)
}
ly := createLayout(img, ".")
dg := getDigest(img)
m := ListImages(ly)
for _, hash := range m {
if hash != dg {
t.Errorf("error got %v want %v", hash, dg)
}
}
}
func createLayout(img v1.Image, path string) layout.Path {
p, err := layout.FromPath(path)
if err != nil {
fmt.Printf("error creating layout: %v", err)
}
p.AppendImage(img)
return p
}
func getDigest(img v1.Image) v1.Hash {
digest, err := img.Digest()
if err != nil {
fmt.Printf("error getting digest: %v", err)
}
return digest
}

58
pkg/oci/oci_test.go
View File

@@ -1 +1,59 @@
package oci
import (
"context"
"fmt"
"io/ioutil"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
"github.com/google/go-containerregistry/pkg/registry"
)
const timeout = 1 * time.Minute
func Test_Get_Put(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
// Set up a fake registry.
s := httptest.NewServer(registry.New())
defer s.Close()
u, err := url.Parse(s.URL)
if err != nil {
t.Fatal(err)
}
file, err := ioutil.TempFile(".", "artifact.txt")
if err != nil {
t.Fatal(err)
}
text := []byte("Some stuff!")
if _, err = file.Write(text); err != nil {
t.Fatal(err)
}
img := fmt.Sprintf("%s/artifact:latest", u.Host)
if err := Put(ctx, file.Name(), img); err != nil {
t.Fatal(err)
}
dir, err := ioutil.TempDir(".", "tmp")
if err != nil {
t.Fatal(err)
}
if err := Get(ctx, img, dir); err != nil {
t.Fatal(err)
}
defer os.Remove(file.Name())
defer os.RemoveAll(dir)
}

5
pkg/packager/packager_test.go
View File

@@ -2,9 +2,10 @@ package packager
import (
"context"
"testing"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/fs"
"testing"
)
func Test_pkg_driver(t *testing.T) {
@@ -13,7 +14,7 @@ func Test_pkg_driver(t *testing.T) {
}
type args struct {
ctx context.Context
d v1alpha1.IDriver
d v1alpha1.Driver
}
tests := []struct {
name string