From fc6332d587daf5f2aba04562a7f0678dffe707d0 Mon Sep 17 00:00:00 2001 From: Josh Wolf Date: Fri, 12 Nov 2021 09:49:28 -0700 Subject: [PATCH] update readme, docs, roadmap, and several cli docs (#67) * update readme, docs, roadmap, and several cli docs * update dead links --- README.md | 31 +++-- ROADMAP.md | 41 ++++--- cmd/hauler/cli/download/download.go | 20 +++- cmd/hauler/cli/store.go | 14 ++- cmd/hauler/cli/store/load.go | 16 ++- cmd/hauler/cli/store/save.go | 2 +- docs/walkthrough.md | 177 ++++++++++++++++++++++++++++ pkg/store/add.go | 35 +++--- 8 files changed, 282 insertions(+), 54 deletions(-) create mode 100644 docs/walkthrough.md diff --git a/README.md b/README.md index 745335d..4890082 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,28 @@ # Hauler: Airgap Assistant -__⚠️ WARNING: This is an experimental, work in progress project. _Everything_ is subject to change, and it is actively in development, so let us know what you think!__ +> ⚠️ This project is still in active development and _not_ GA. While a lot of the core features are ready, we're still adding a _ton_, and we may make breaking api and feature changes version to version. -`hauler` is a command line tool for that aims to simplify the painpoints that exist around airgapped Kubernetes deployments. -It remains as unopinionated as possible, and does _not_ attempt to enforce a specific cluster type or application deployment model. -Instead, it focuses solely on simplifying the primary airgap pain points: -* artifact collection -* artifact distribution +`hauler` simplifies the airgap experience without forcing you to adopt a specific workflow for your infrastructure or application. -`hauler` achieves this by leaning heavily on the [oci spec](https://github.com/opencontainers), and the vast ecosystem of tooling available for fetching and distributing oci content. +To accomplish this, it focuses strictly on two of the biggest airgap pain points: + +* content collection +* content distribution + +As OCI registries have become ubiquitous nowadays for storing and distributing containers. Their success and widespread adoption has led many projects to expand beyond containers. + +`hauler` capitalizes on this, and leverages the [`oci`](https://github.com/opencontainers) spec to be a simple, zero dependency tool to collect, transport, and distribute your artifacts. + +## Getting started + +See the [quickstart](docs/walkthrough.md#Quickstart) for a quick way to get started with some of `haulers` capabilities. + +For a guided example of all of `haulers` capabilities, check out the [guided example](docs/walkthrough.md#guided-examples). + +## Acknowledgements + +`hauler` wouldn't be possible without the open source community, but there are a few dependent projects that stand out: + +* [go-containerregistry](https://github.com/google/go-containerregistry) +* [oras](https://github.com/oras-project/oras) +* [cosign](https://github.com/sigstore/cosign) \ No newline at end of file diff --git a/ROADMAP.md b/ROADMAP.md index c313ec1..1b48960 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,10 +1,29 @@ # Hauler Roadmap -## v0.0.x +## \> v0.2.0 -- Install single-node k3s cluster into an Ubuntu machine using the tarball installation method +- Leverage `referrers` api to robustly link content/collection +- Support signing for all `artifact.OCI` contents +- Support encryption for `artifact.OCI` layers +- Safely embed container runtime for user created `collections` creation and transformation +- Better defaults/configuration/security around for long-lived embedded registry +- Better support multi-platform content +- Better leverage `oras` (`>=0.5.0`) for content relocation +- Store git repos as CAS in OCI format -## v0.1.0 +## v0.2.0 - MVP 2 + +- Re-focus on cli and framework for oci content fetching and delivery +- Focus on initial key contents + - Files (local/remote) + - Charts (local/remote) + - Images +- Establish framework for `content` and `collections` +- Define initial `content` types (`file`, `chart`, `image`) +- Define initial `collection` types (`thickchart`, `k3s`) +- Define framework for manipulating OCI content (`artifact.OCI`, `artifact.Collection`) + +## v0.1.0 - MVP 1 - Install single-node k3s cluster - Support tarball and rpm installation methods @@ -25,18 +44,6 @@ - NOTE: "generic" option - most other use cases can be satisfied by a specially crafted file server directory +## v0.0.x -## Potential future features - -- Helm charts - - Pull charts, migrate chart artifacts - - Analyze required container images, add to dependency list -- Yum repo - - Provide package list, collect all dependencies - - Deploy fully configured yum repo into file server -- Deploy Minio for S3 API - - MVP: backed by HA storage solution (e.g. AWS S3, Azure Blob Storage) - - Stable: backed by local storage, including backups -- Split archives into chunks of chosen size - - Enables easier transfer via physical media - - Allows smaller network transfers, losing less progress on failed upload (or working around timeouts) +- Install single-node k3s cluster into an Ubuntu machine using the tarball installation method diff --git a/cmd/hauler/cli/download/download.go b/cmd/hauler/cli/download/download.go index 2bfe434..c6fac6b 100644 --- a/cmd/hauler/cli/download/download.go +++ b/cmd/hauler/cli/download/download.go @@ -79,7 +79,12 @@ func Cmd(ctx context.Context, o *Opts, reference string) error { return err } - lgr.Infof("downloaded [%s] to [%s]", ref.Name(), outputFile) + d, err := img.Digest() + if err != nil { + return err + } + + lgr.Infof("downloaded image [%s] to [%s] with digest [%s]", ref.Name(), outputFile, d.String()) case types.FileConfigMediaType: lgr.Infof("identified [file] (%s) content", manifest.Config.MediaType) @@ -92,7 +97,7 @@ func Cmd(ctx context.Context, o *Opts, reference string) error { return err } - lgr.Infof("downloaded [%d] files with digest [%s]", len(descs), mdesc) + lgr.Infof("downloaded [%d] file(s) with digest [%s]", len(descs), mdesc) case types.ChartLayerMediaType, types.ChartConfigMediaType: lgr.Infof("identified [chart] (%s) content", manifest.Config.MediaType) @@ -100,12 +105,19 @@ func Cmd(ctx context.Context, o *Opts, reference string) error { fs := content.NewFileStore(o.DestinationDir) resolver := docker.NewResolver(docker.ResolverOptions{}) - mdesc, _, err := oras.Pull(ctx, resolver, ref.Name(), fs) + mdesc, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs) if err != nil { return err } - lgr.Infof("downloaded chart [%s] with digest [%s]", "donkey", mdesc.Digest.String()) + cn := path.Base(ref.Name()) + for _, d := range descs { + if n, ok := d.Annotations[ocispec.AnnotationTitle]; ok { + cn = n + } + } + + lgr.Infof("downloaded chart [%s] to [%s] with digest [%s]", ref.String(), cn, mdesc.Digest.String()) default: return fmt.Errorf("unrecognized content type: %s", manifest.Config.MediaType) diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 012647a..5098361 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -8,8 +8,9 @@ import ( func addStore(parent *cobra.Command) { cmd := &cobra.Command{ - Use: "store", - Short: "Interact with hauler's embedded content store", + Use: "store", + Aliases: []string{"s"}, + Short: "Interact with hauler's embedded content store", RunE: func(cmd *cobra.Command, args []string) error { return cmd.Help() }, @@ -149,9 +150,10 @@ func addStoreList() *cobra.Command { o := &store.ListOpts{} cmd := &cobra.Command{ - Use: "list", - Short: "List all content references in a store", - Args: cobra.ExactArgs(0), + Use: "list", + Short: "List all content references in a store", + Args: cobra.ExactArgs(0), + Aliases: []string{"ls"}, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -263,7 +265,7 @@ func addStoreAddChart() *cobra.Command { Short: "Add a chart to the content store", Example: ` # add a chart -hauler store add longhorn --repo "https://charts.longhorn.io" +hauler store add chart longhorn --repo "https://charts.longhorn.io" # add a specific version of a chart hauler store add chart rancher --repo "https://releases.rancher.com/server-charts/latest" --version "2.6.2" diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index 683b3dd..41c2f10 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -9,11 +9,14 @@ import ( "github.com/rancherfederal/hauler/pkg/log" ) -type LoadOpts struct{} +type LoadOpts struct { + OutputDir string +} func (o *LoadOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() - _ = f + + f.StringVarP(&o.OutputDir, "output", "o", "", "Directory to unload archived contents to (defaults to $PWD/haul)") } // LoadCmd @@ -26,9 +29,14 @@ func LoadCmd(ctx context.Context, o *LoadOpts, dir string, archiveRefs ...string a := archiver.NewTarZstd() a.OverwriteExisting = true + odir := dir + if o.OutputDir != "" { + odir = o.OutputDir + } + for _, archiveRef := range archiveRefs { - l.Infof("Loading content from %s to %s", archiveRef, dir) - err := a.Unarchive(archiveRef, dir) + l.Infof("loading content from [%s] to [%s]", archiveRef, odir) + err := a.Unarchive(archiveRef, odir) if err != nil { return err } diff --git a/cmd/hauler/cli/store/save.go b/cmd/hauler/cli/store/save.go index dd1d045..b00c0eb 100644 --- a/cmd/hauler/cli/store/save.go +++ b/cmd/hauler/cli/store/save.go @@ -36,7 +36,6 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er return err } - l.Infof("Saving data dir (%s) as compressed archive to %s", dir, absOutputfile) cwd, err := os.Getwd() if err != nil { return err @@ -51,5 +50,6 @@ func SaveCmd(ctx context.Context, o *SaveOpts, outputFile string, dir string) er return err } + l.Infof("saved haul [%s] -> [%s]", dir, absOutputfile) return nil } diff --git a/docs/walkthrough.md b/docs/walkthrough.md new file mode 100644 index 0000000..1d8876e --- /dev/null +++ b/docs/walkthrough.md @@ -0,0 +1,177 @@ +# Walkthrough + +## Installation + +The latest version of `hauler` is available as statically compiled binaries for most combinations of operating systems and architectures on the GitHub [releases](https://github.com/rancherfederal/hauler/releases) page. + +## Quickstart + +The tl;dr for how to use `hauler` to fetch, transport, and distribute `content`: + +```bash +# fetch some content +hauler store add file "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +hauler store add chart longhorn --repo "https://charts.longhorn.io" +hauler store add image "rancher/cowsay" + +# transport the content +hauler store save + +# <-airgap the haul.tar.zst file generated-> + +# load the content +hauler store load + +# serve the content +hauler store serve +``` + +While the example above fits into a quickstart, it falls short of demonstrating all the capabilities `hauler` has to offer, including taking advantage of its fully declarative nature. Keep reading the [Guided Examples](#Guided-Examples) below for a more thorough walkthrough of `haulers` full capabilities. + +## Guided Examples + +Since `hauler`'s primary objective is to simplify the content collection/distribution airgap process, a lot of the design revolves around the typical airgap workflow: + +```bash +fetch -> save - | | -> validate/load -> distribute +``` + +This is accomplished as follows: + +```bash +# fetch content +hauler store add ... + +# compress and archive content +hauler store save + +# + +# validate/load content +hauler store load ... + +# distribute content +hauler store serve +``` + +At this point you're probably wondering: what is `content`? In `hauler` land, there are a few important terms given to important resources: + +* `artifact`: anything that can be represented as an [`oci artifact`](https://github.com/opencontainers/artifacts) +* `content`: built in "primitive" types of `artifacts` that `hauler` understands + +### Built in content + +As of today, `hauler` understands three types of `content`, one with a strong legacy of community support and consensus ([`image-spec`]()), one with a finalized spec and experimental support ([`chart-spec`]()), and one generic type created just for `hauler`. These `content` types are outlined below: + +__`files`__: + +Generic content that can be represented as a file, either sourced locally or remotely. + +```bash +# local file +hauler store add file path/to/local/file.txt + +# remote file +hauler store add file https://get.k3s.io +``` + +__`images`__: + +Any OCI compatible image can be fetched remotely. + +```bash +# "shorthand" image references +hauler store add image rancher/k3s:v1.22.2-k3s1 + +# fully qualified image references +hauler store add image ghcr.io/fluxcd/flux-cli@sha256:02aa820c3a9c57d67208afcfc4bce9661658c17d15940aea369da259d2b976dd +``` + +__`charts`__: + +Helm charts represented as OCI content. + +```bash +# add a helm chart (defaults to latest version) +hauler store add chart loki --repo "https://grafana.github.io/helm-charts" + +# add a specific version of a helm chart +hauler store add chart loki --repo "https://grafana.github.io/helm-charts" --version 2.8.1 + +# install directly from the oci content +HELM_EXPERIMENTAL_OCI=1 helm install loki oci://localhost:3000/library/loki --version 2.8.1 +``` + +> Note: `hauler` supports the currently experimental format of helm as OCI content, but can also be represented as the usual tarball if necessary + +### Content API + +While imperatively adding `content` to `hauler` is a simple way to get started, the recommended long term approach is to use the provided api that each `content` has, in conjunction with the `sync` command. + +```bash +# create a haul from declaratively defined content +hauler store sync -f testdata/contents.yaml +``` + +> For a commented view of the `contents` api, take a look at the `testdata` folder in the root of the project. + +The API for each type of built-in `content` allows you to easily and declaratively define all the `content` that exist within a `haul`, and ensures a more gitops compatible workflow for managing the lifecycle of your `hauls`. + +### Collections + +Earlier we referred to `content` as "primitives". While the quotes justify the loose definition of that term, we call it that because they can be used to build groups of `content`, which we call `collections`. + +`collections` are groups of 1 or more `contents` that collectively represent something desirable. Just like `content`, there are a handful that are built in to `hauler`. + +Since `collections` usually contain more purposefully crafted `contents`, we restrict their use to the declarative commands (`sync`): + +```bash +# sync a collection +hauler store sync -f my-collection.yaml + +# sync sets of content/collection +hauler store sync -f collection.yaml -f content.yaml +``` + +__`thickcharts`__: + +Thick Charts represent the combination of `charts` and `images`. When storing a thick chart, the chart _and_ the charts dependent images will be fetched and stored by `hauler`. + +```yaml +# thick-chart.yaml +apiVersion: collection.hauler.cattle.io/v1alpha1 +kind: ThickCharts +metadata: + name: loki +spec: + charts: + - name: loki + repoURL: https://grafana.github.io/helm-charts +``` + +When syncing the collection above, `hauler` will identify the images the chart depends on and store those too + +> The method for identifying images is constantly changing, as of today, the chart is rendered and a configurable set of container defining json path's are processed. The most common paths are recognized by hauler, but this can be configured for the more niche CRDs out there. + +__`k3s`__: + +Combining `files` and `images`, full clusters can also be captured by `hauler` for further simplifying the already simple nature of `k3s`. + +```yaml +# k3s.yaml +--- +apiVersion: collection.hauler.cattle.io/v1alpha1 +kind: K3s +metadata: + name: k3s +spec: + version: stable +``` + +Using the collection above, the dependent files (`k3s` executable and `https://get.k3s.io` script) will be fetched, as well as all the dependent images. + +> We know not everyone uses the get.k3s.io script to provision k3s, in the future this may change, but until then you're welcome to mix and match the `collection` with any of your own additional `content` + +#### User defined `collections` + +Although `content` and `collections` can only be used when they are baked in to `hauler`, the goal is to allow these to be securely user-defined, allowing you to define your own desirable `collection` types, and leave the heavy lifting to `hauler`. Check out our [roadmap](../ROADMAP.md) and [milestones]() for more info on that. \ No newline at end of file diff --git a/pkg/store/add.go b/pkg/store/add.go index 27d420e..fb1e94f 100644 --- a/pkg/store/add.go +++ b/pkg/store/add.go @@ -38,11 +38,16 @@ func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference nam } lgr.Debugf("staging %s", reference.Name()) - if err := stg.add(ctx, oci, reference); err != nil { + pdesc, err := stg.add(ctx, oci, reference) + if err != nil { return ocispec.Descriptor{}, err } - return stg.commit(ctx, s) + if err := stg.commit(ctx, s); err != nil { + return ocispec.Descriptor{}, nil + } + + return pdesc, nil } // Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting everything in the underlying store directory @@ -105,25 +110,25 @@ type oci struct { root string } -func (o *oci) add(ctx context.Context, oci artifact.OCI, reference name.Reference) error { +func (o *oci) add(ctx context.Context, oci artifact.OCI, reference name.Reference) (ocispec.Descriptor, error) { mdesc, err := o.layout.WriteOci(oci, reference) - if err != nil { - return err - } - _ = mdesc - return nil -} - -func (o *oci) commit(ctx context.Context, s *Store) (ocispec.Descriptor, error) { - ts, err := layout.NewOCIStore(o.root) if err != nil { return ocispec.Descriptor{}, err } + return mdesc, err +} - err = layout.Copy(ctx, ts, s.Registry()) - +func (o *oci) commit(ctx context.Context, s *Store) error { defer o.close() - return ocispec.Descriptor{}, err + ts, err := layout.NewOCIStore(o.root) + if err != nil { + return err + } + + if err = layout.Copy(ctx, ts, s.Registry()); err != nil { + return err + } + return err } func (o *oci) close() error {