update readme, docs, roadmap, and several cli docs (#67)

* update readme, docs, roadmap, and several cli docs * update dead links
2026-02-14 18:09:51 +00:00 · 2021-11-12 09:49:28 -07:00
parent 49eb9e2527
commit fc6332d587
8 changed files with 282 additions and 54 deletions
--- a/README.md
+++ b/README.md
@@ -1,11 +1,28 @@
 # Hauler: Airgap Assistant

-__⚠️ WARNING: This is an experimental, work in progress project.  _Everything_ is subject to change, and it is actively in development, so let us know what you think!__
+> ⚠️ This project is still in active development and _not_ GA.  While a lot of the core features are ready, we're still adding a _ton_, and we may make breaking api and feature changes version to version.

-`hauler` is a command line tool for that aims to simplify the painpoints that exist around airgapped Kubernetes deployments.
-It remains as unopinionated as possible, and does _not_ attempt to enforce a specific cluster type or application deployment model.
-Instead, it focuses solely on simplifying the primary airgap pain points:
-* artifact collection
-* artifact distribution
+`hauler` simplifies the airgap experience without forcing you to adopt a specific workflow for your infrastructure or application.  

-`hauler` achieves this by leaning heavily on the [oci spec](https://github.com/opencontainers), and the vast ecosystem of tooling available for fetching and distributing oci content.
+To accomplish this, it focuses strictly on two of the biggest airgap pain points:
+
+* content collection
+* content distribution
+
+As OCI registries have become ubiquitous nowadays for storing and distributing containers.  Their success and widespread adoption has led many projects to expand beyond containers.
+
+`hauler` capitalizes on this, and leverages the [`oci`](https://github.com/opencontainers) spec to be a simple, zero dependency tool to collect, transport, and distribute your artifacts.
+
+## Getting started
+
+See the [quickstart](docs/walkthrough.md#Quickstart) for a quick way to get started with some of `haulers` capabilities.
+
+For a guided example of all of `haulers` capabilities, check out the [guided example](docs/walkthrough.md#guided-examples).
+
+## Acknowledgements
+
+`hauler` wouldn't be possible without the open source community, but there are a few dependent projects that stand out:
+
+* [go-containerregistry](https://github.com/google/go-containerregistry)
+* [oras](https://github.com/oras-project/oras)
+* [cosign](https://github.com/sigstore/cosign)
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -1,10 +1,29 @@
 # Hauler Roadmap

-## v0.0.x
+## \> v0.2.0

- Install single-node k3s cluster into an Ubuntu machine using the tarball installation method
+- Leverage `referrers` api to robustly link content/collection
+- Support signing for all `artifact.OCI` contents
+- Support encryption for `artifact.OCI` layers
+- Safely embed container runtime for user created `collections` creation and transformation
+- Better defaults/configuration/security around for long-lived embedded registry
+- Better support multi-platform content
+- Better leverage `oras` (`>=0.5.0`) for content relocation
+- Store git repos as CAS in OCI format

-## v0.1.0
+## v0.2.0 - MVP 2
+
+- Re-focus on cli and framework for oci content fetching and delivery
+- Focus on initial key contents
+  - Files (local/remote)
+  - Charts (local/remote)
+  - Images
+- Establish framework for `content` and `collections`
+- Define initial `content` types (`file`, `chart`, `image`)
+- Define initial `collection` types (`thickchart`, `k3s`)
+- Define framework for manipulating OCI content (`artifact.OCI`, `artifact.Collection`)
+
+## v0.1.0 - MVP 1

 - Install single-node k3s cluster
  - Support tarball and rpm installation methods
@@ -25,18 +44,6 @@
  - NOTE: "generic" option - most other use cases can be satisfied by a specially crafted file
    server directory

+## v0.0.x

-## Potential future features
-
- Helm charts
-  - Pull charts, migrate chart artifacts
-  - Analyze required container images, add to dependency list
- Yum repo
-  - Provide package list, collect all dependencies
-  - Deploy fully configured yum repo into file server
- Deploy Minio for S3 API
-  - MVP: backed by HA storage solution (e.g. AWS S3, Azure Blob Storage)
-  - Stable: backed by local storage, including backups
- Split archives into chunks of chosen size
-  - Enables easier transfer via physical media
-  - Allows smaller network transfers, losing less progress on failed upload (or working around timeouts)
+- Install single-node k3s cluster into an Ubuntu machine using the tarball installation method
--- a/cmd/hauler/cli/download/download.go
+++ b/cmd/hauler/cli/download/download.go
@@ -79,7 +79,12 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 			return err
 		}

-		lgr.Infof("downloaded [%s] to [%s]", ref.Name(), outputFile)
+		d, err := img.Digest()
+		if err != nil {
+			return err
+		}
+
+		lgr.Infof("downloaded image [%s] to [%s] with digest [%s]", ref.Name(), outputFile, d.String())

 	case types.FileConfigMediaType:
 		lgr.Infof("identified [file] (%s) content", manifest.Config.MediaType)
@@ -92,7 +97,7 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 			return err
 		}

-		lgr.Infof("downloaded [%d] files with digest [%s]", len(descs), mdesc)
+		lgr.Infof("downloaded [%d] file(s) with digest [%s]", len(descs), mdesc)

 	case types.ChartLayerMediaType, types.ChartConfigMediaType:
 		lgr.Infof("identified [chart] (%s) content", manifest.Config.MediaType)
@@ -100,12 +105,19 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 		fs := content.NewFileStore(o.DestinationDir)

 		resolver := docker.NewResolver(docker.ResolverOptions{})
-		mdesc, _, err := oras.Pull(ctx, resolver, ref.Name(), fs)
+		mdesc, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
 		if err != nil {
 			return err
 		}

-		lgr.Infof("downloaded chart [%s] with digest [%s]", "donkey", mdesc.Digest.String())
+		cn := path.Base(ref.Name())
+		for _, d := range descs {
+			if n, ok := d.Annotations[ocispec.AnnotationTitle]; ok {
+				cn = n
+			}
+		}
+
+		lgr.Infof("downloaded chart [%s] to [%s] with digest [%s]", ref.String(), cn, mdesc.Digest.String())

 	default:
 		return fmt.Errorf("unrecognized content type: %s", manifest.Config.MediaType)
--- a/cmd/hauler/cli/store.go
+++ b/cmd/hauler/cli/store.go
@@ -8,8 +8,9 @@ import (

 func addStore(parent *cobra.Command) {
 	cmd := &cobra.Command{
-		Use:   "store",
-		Short: "Interact with hauler's embedded content store",
+		Use:     "store",
+		Aliases: []string{"s"},
+		Short:   "Interact with hauler's embedded content store",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return cmd.Help()
 		},
@@ -149,9 +150,10 @@ func addStoreList() *cobra.Command {
 	o := &store.ListOpts{}

 	cmd := &cobra.Command{
-		Use:   "list",
-		Short: "List all content references in a store",
-		Args:  cobra.ExactArgs(0),
+		Use:     "list",
+		Short:   "List all content references in a store",
+		Args:    cobra.ExactArgs(0),
+		Aliases: []string{"ls"},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()

@@ -263,7 +265,7 @@ func addStoreAddChart() *cobra.Command {
 		Short: "Add a chart to the content store",
 		Example: `
 # add a chart
-hauler store add longhorn --repo "https://charts.longhorn.io"
+hauler store add chart longhorn --repo "https://charts.longhorn.io"

 # add a specific version of a chart
 hauler store add chart rancher --repo "https://releases.rancher.com/server-charts/latest" --version "2.6.2"
--- a/cmd/hauler/cli/store/load.go
+++ b/cmd/hauler/cli/store/load.go
@@ -9,11 +9,14 @@ import (
 	"github.com/rancherfederal/hauler/pkg/log"
 )

-type LoadOpts struct{}
+type LoadOpts struct {
+	OutputDir string
+}

 func (o *LoadOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
-	_ = f
+
+	f.StringVarP(&o.OutputDir, "output", "o", "", "Directory to unload archived contents to (defaults to $PWD/haul)")
 }

 // LoadCmd
@@ -26,9 +29,14 @@ func LoadCmd(ctx context.Context, o *LoadOpts, dir string, archiveRefs ...string
 	a := archiver.NewTarZstd()
 	a.OverwriteExisting = true

+	odir := dir
+	if o.OutputDir != "" {
+		odir = o.OutputDir
+	}
+
 	for _, archiveRef := range archiveRefs {
-		l.Infof("Loading content from %s to %s", archiveRef, dir)
-		err := a.Unarchive(archiveRef, dir)
+		l.Infof("loading content from [%s] to [%s]", archiveRef, odir)
+		err := a.Unarchive(archiveRef, odir)
 		if err != nil {
 			return err
 		}
--- a/cmd/hauler/cli/store/save.go
+++ b/cmd/hauler/cli/store/save.go
@@ -36,7 +36,6 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er
 		return err
 	}

-	l.Infof("Saving data dir (%s) as compressed archive to %s", dir, absOutputfile)
 	cwd, err := os.Getwd()
 	if err != nil {
 		return err
@@ -51,5 +50,6 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er
 		return err
 	}

+	l.Infof("saved haul [%s] -> [%s]", dir, absOutputfile)
 	return nil
 }
--- a/docs/walkthrough.md
+++ b/docs/walkthrough.md
@@ -0,0 +1,177 @@
+# Walkthrough
+
+## Installation
+
+The latest version of `hauler` is available as statically compiled binaries for most combinations of operating systems and architectures on the GitHub [releases](https://github.com/rancherfederal/hauler/releases) page.
+
+## Quickstart
+
+The tl;dr for how to use `hauler` to fetch, transport, and distribute `content`:
+
+```bash
+# fetch some content
+hauler store add file "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+hauler store add chart longhorn --repo "https://charts.longhorn.io"
+hauler store add image "rancher/cowsay"
+
+# transport the content
+hauler store save
+
+# <-airgap the haul.tar.zst file generated->
+
+# load the content
+hauler store load
+
+# serve the content
+hauler store serve
+```
+
+While the example above fits into a quickstart, it falls short of demonstrating all the capabilities `hauler` has to offer, including taking advantage of its fully declarative nature.  Keep reading the [Guided Examples](#Guided-Examples) below for a more thorough walkthrough of `haulers` full capabilities.
+
+## Guided Examples
+
+Since `hauler`'s primary objective is to simplify the content collection/distribution airgap process, a lot of the design revolves around the typical airgap workflow:
+
+```bash
+fetch -> save - | <airgap> | -> validate/load -> distribute
+```
+
+This is accomplished as follows:
+
+```bash
+# fetch content
+hauler store add ...
+
+# compress and archive content
+hauler store save
+
+# <airgap>
+
+# validate/load content
+hauler store load ...
+
+# distribute content
+hauler store serve
+```
+
+At this point you're probably wondering: what is `content`? In `hauler` land, there are a few important terms given to important resources:
+
+* `artifact`: anything that can be represented as an [`oci artifact`](https://github.com/opencontainers/artifacts)
+* `content`: built in "primitive" types of `artifacts` that `hauler` understands
+
+### Built in content
+
+As of today, `hauler` understands three types of `content`, one with a strong legacy of community support and consensus ([`image-spec`]()), one with a finalized spec and experimental support ([`chart-spec`]()), and one generic type created just for `hauler`.  These `content` types are outlined below:
+
+__`files`__:
+
+Generic content that can be represented as a file, either sourced locally or remotely.
+
+```bash
+# local file
+hauler store add file path/to/local/file.txt
+
+# remote file
+hauler store add file https://get.k3s.io
+```
+
+__`images`__:
+
+Any OCI compatible image can be fetched remotely.
+
+```bash
+# "shorthand"  image references
+hauler store add image rancher/k3s:v1.22.2-k3s1
+
+# fully qualified image references
+hauler store add image ghcr.io/fluxcd/flux-cli@sha256:02aa820c3a9c57d67208afcfc4bce9661658c17d15940aea369da259d2b976dd
+```
+
+__`charts`__:
+
+Helm charts represented as OCI content.
+
+```bash
+# add a helm chart (defaults to latest version)
+hauler store add chart loki --repo "https://grafana.github.io/helm-charts"
+
+# add a specific version of a helm chart
+hauler store add chart loki --repo "https://grafana.github.io/helm-charts" --version 2.8.1
+
+# install directly from the oci content
+HELM_EXPERIMENTAL_OCI=1 helm install loki oci://localhost:3000/library/loki --version 2.8.1
+```
+
+> Note: `hauler` supports the currently experimental format of helm as OCI content, but can also be represented as the usual tarball if necessary
+
+### Content API
+
+While imperatively adding `content` to `hauler` is a simple way to get started, the recommended long term approach is to use the provided api that each `content` has, in conjunction with the `sync` command.
+
+```bash
+# create a haul from declaratively defined content
+hauler store sync -f testdata/contents.yaml
+```
+
+> For a commented view of the `contents` api, take a look at the `testdata` folder in the root of the project.
+
+The API for each type of built-in `content` allows you to easily and declaratively define all the `content` that exist within a `haul`, and ensures a more gitops compatible workflow for managing the lifecycle of your `hauls`.
+
+### Collections
+
+Earlier we referred to `content` as "primitives".  While the quotes justify the loose definition of that term, we call it that because they can be used to build groups of `content`, which we call `collections`.
+
+`collections` are groups of 1 or more `contents` that collectively represent something desirable.  Just like `content`, there are a handful that are built in to `hauler`.
+
+Since `collections` usually contain more purposefully crafted `contents`, we restrict their use to the declarative commands (`sync`):
+
+```bash
+# sync a collection
+hauler store sync -f my-collection.yaml
+
+# sync sets of content/collection
+hauler store sync -f collection.yaml -f content.yaml
+```
+
+__`thickcharts`__:
+
+Thick Charts represent the combination of `charts` and `images`.  When storing a thick chart, the chart _and_ the charts dependent images will be fetched and stored by `hauler`.  
+
+```yaml
+# thick-chart.yaml
+apiVersion: collection.hauler.cattle.io/v1alpha1
+kind: ThickCharts
+metadata:
+  name: loki
+spec:
+  charts:
+    - name: loki
+      repoURL: https://grafana.github.io/helm-charts
+```
+
+When syncing the collection above, `hauler` will identify the images the chart depends on and store those too
+
+> The method for identifying images is constantly changing, as of today, the chart is rendered and a configurable set of container defining json path's are processed.  The most common paths are recognized by hauler, but this can be configured for the more niche CRDs out there.
+
+__`k3s`__:
+
+Combining `files` and `images`, full clusters can also be captured by `hauler` for further simplifying the already simple nature of `k3s`.
+
+```yaml
+# k3s.yaml
+---
+apiVersion: collection.hauler.cattle.io/v1alpha1
+kind: K3s
+metadata:
+  name: k3s
+spec:
+  version: stable
+```
+
+Using the collection above, the dependent files (`k3s` executable and `https://get.k3s.io` script) will be fetched, as well as all the dependent images.
+
+> We know not everyone uses the get.k3s.io script to provision k3s, in the future this may change, but until then you're welcome to mix and match the `collection` with any of your own additional `content` 
+
+#### User defined `collections`
+
+Although `content` and `collections` can only be used when they are baked in to `hauler`, the goal is to allow these to be securely user-defined, allowing you to define your own desirable `collection` types, and leave the heavy lifting to `hauler`.  Check out our [roadmap](../ROADMAP.md) and [milestones]() for more info on that.
--- a/pkg/store/add.go
+++ b/pkg/store/add.go
@@ -38,11 +38,16 @@ func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference nam
 	}

 	lgr.Debugf("staging %s", reference.Name())
-	if err := stg.add(ctx, oci, reference); err != nil {
+	pdesc, err := stg.add(ctx, oci, reference)
+	if err != nil {
 		return ocispec.Descriptor{}, err
 	}

-	return stg.commit(ctx, s)
+	if err := stg.commit(ctx, s); err != nil {
+		return ocispec.Descriptor{}, nil
+	}
+
+	return pdesc, nil
 }

 // Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting everything in the underlying store directory
@@ -105,25 +110,25 @@ type oci struct {
 	root   string
 }

-func (o *oci) add(ctx context.Context, oci artifact.OCI, reference name.Reference) error {
+func (o *oci) add(ctx context.Context, oci artifact.OCI, reference name.Reference) (ocispec.Descriptor, error) {
 	mdesc, err := o.layout.WriteOci(oci, reference)
-	if err != nil {
-		return err
-	}
-	_ = mdesc
-	return nil
-}
-
-func (o *oci) commit(ctx context.Context, s *Store) (ocispec.Descriptor, error) {
-	ts, err := layout.NewOCIStore(o.root)
 	if err != nil {
 		return ocispec.Descriptor{}, err
 	}
+	return mdesc, err
+}

-	err = layout.Copy(ctx, ts, s.Registry())
-
+func (o *oci) commit(ctx context.Context, s *Store) error {
 	defer o.close()
-	return ocispec.Descriptor{}, err
+	ts, err := layout.NewOCIStore(o.root)
+	if err != nil {
+		return err
+	}
+
+	if err = layout.Copy(ctx, ts, s.Registry()); err != nil {
+		return err
+	}
+	return err
 }

 func (o *oci) close() error {