Compare commits

...

31 Commits

Author SHA1 Message Date
Zack Brady
ceb77601d0 merge pull request #252 from zackbradys/main
overhauling github actions and workflows
2024-06-24 22:52:08 -04:00
Zack Brady
d90545a9e4 fixed permissions on testdata 2024-06-17 22:23:03 -04:00
Zack Brady
bef141ab67 updated chart versions (will need to update again) 2024-06-17 19:45:18 -04:00
Zack Brady
385d767c2a last bit of fixes to workflow 2024-06-17 19:42:20 -04:00
Zack Brady
22edc77506 updated unit test workflow 2024-06-14 20:56:37 -04:00
Zack Brady
9058797bbc updated goreleaser deprecations 2024-06-14 20:43:32 -04:00
Zack Brady
35e2f655da added helm chart release job 2024-06-14 20:37:58 -04:00
Zack Brady
f5c0f6f0ae updated github template names 2024-06-14 20:04:01 -04:00
Zack Brady
0ec77b4168 merge pull request #248 from zackbradys/main
formatted all code with `go fmt`
2024-06-14 16:46:21 -04:00
Zack Brady
7a7906b8ea updated imports (and go fmt) 2024-06-13 23:44:06 -04:00
Zack Brady
f4774445f6 merge pull request #240 from pat-earl/doc_updates
added some documentation text to sync command
2024-06-10 18:13:09 -04:00
Zack Brady
d59b29bfce formatted gitignore to match dockerignore 2024-06-05 14:40:50 -04:00
Zack Brady
fd702202ac formatted all code (go fmt) 2024-06-05 08:25:45 -04:00
Adam Martin
9e9565717b Merge pull request #245 from ethanchowell/ehowell/helm-client
Configure chart commands to use helm clients for OCI and private regi…
2024-06-04 13:16:42 -04:00
Zack Brady
bfe47ae141 updated chart tests for new features 2024-06-03 23:31:50 -04:00
Zack Brady
ebab7f38a0 merge pull request #247 from kaminfay/dev/add-fileserver-timeout-flag
adding the timeout flag for fileserver command and setting default timeout to 60 seconds
2024-06-03 22:33:36 -04:00
Kamin Fay
f0cba3c2c6 Adding the timeout flag for fileserver command 2024-05-28 15:28:20 -04:00
Ethan Howell
286120da50 Configure chart commands to use helm clients for OCI and private registry support 2024-05-24 12:06:16 -04:00
Patrick Earl
dcdeb93518 Added some documentation text to sync command 2024-05-02 14:17:38 -04:00
Adam Martin
f7c24f6129 Merge pull request #235 from rancherfederal/dependabot/go_modules/golang.org/x/net-0.23.0
Bump golang.org/x/net from 0.17.0 to 0.23.0
2024-04-23 16:40:45 -04:00
Adam Martin
fe88d7033c Merge pull request #234 from amartin120/dup-digest-bugfix
fix for dup digest smashing in cosign
2024-04-23 15:49:54 -04:00
dependabot[bot]
ef31984c97 Bump golang.org/x/net from 0.17.0 to 0.23.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.17.0 to 0.23.0.
- [Commits](https://github.com/golang/net/compare/v0.17.0...v0.23.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-04-23 19:46:05 +00:00
Adam Martin
2889f30275 fix for dup digest smashing in cosign
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2024-04-23 14:06:40 -04:00
Zack Brady
0674e0ab30 merge pull request #229 from zackbradys/vagrant
removed vagrant scripts
2024-04-15 09:36:24 -04:00
Zack Brady
d645c52135 removed vagrant scripts 2024-04-14 08:45:51 -04:00
Zack Brady
44baab3213 merge pull request #227 from zackbradys/helmifying
helmifying hauler
2024-04-11 22:02:37 -04:00
Zack Brady
1a317b0172 merge pull request #226 from zackbradys/testdata
updated hauler testdata
2024-04-11 21:57:37 -04:00
Zack Brady
128cb3b252 last bit of updates and formatting of chart 2024-04-07 00:23:49 -04:00
Zack Brady
91ff998634 updated hauler testdata 2024-04-06 15:19:53 -04:00
Zack Brady
8ac1ecaf29 adding functionality and cleaning up 2024-04-06 02:06:33 -04:00
Zack Brady
7447aad20a added initial helm chart 2024-04-06 00:28:59 -04:00
68 changed files with 1011 additions and 717 deletions

View File

@@ -1,6 +1,6 @@
---
name: Bug Report
about: Create a report to help us improve!
about: Submit a bug report to help us improve!
title: '[BUG]'
labels: 'bug'
assignees: ''

View File

@@ -1,6 +1,6 @@
---
name: Feature Request
about: Submit a request for us to improve!
about: Submit a feature request for us to improve!
title: '[RFE]'
labels: 'enhancement'
assignees: ''

View File

@@ -23,7 +23,7 @@ jobs:
go-version: 1.21.x
- name: Run Go Releaser
uses: goreleaser/goreleaser-action@v5
uses: goreleaser/goreleaser-action@v6
with:
distribution: goreleaser
version: latest
@@ -36,6 +36,7 @@ jobs:
name: Container Release Job
runs-on: ubuntu-latest
timeout-minutes: 30
continue-on-error: true
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -48,7 +49,7 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
- name: Authenticate to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
@@ -62,3 +63,32 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: ghcr.io/${{ github.repository }}:${{ github.ref_name }}
chart-release:
name: Chart Release Job
runs-on: ubuntu-latest
timeout-minutes: 30
continue-on-error: true
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Authenticate to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set Up Helm
uses: azure/setup-helm@v4
- name: Build Helm Chart
run: |
helm package deploy/kubernetes/helm/charts/hauler --destination deploy/kubernetes/helm/charts/hauler
- name: Push to GitHub Container Registry
run: |
helm push deploy/kubernetes/helm/charts/hauler/hauler-*.tgz oci://ghcr.io/${{ github.repository_owner }}/charts

View File

@@ -29,9 +29,10 @@ jobs:
run: |
mkdir -p cmd/hauler/binaries
touch cmd/hauler/binaries/dummy.txt
go test -race -covermode=atomic -coverprofile=coverage.out ./pkg/... ./internal/... ./cmd/...
go test -race -covermode=atomic -coverprofile=coverage.out ./...
- name: On Failure, Launch Debug Session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
timeout-minutes: 10
- name: Upload Coverage Report
uses: actions/upload-artifact@v4
with:
name: coverage-report
path: coverage.out

16
.gitignore vendored
View File

@@ -1,9 +1,5 @@
.DS_Store
# Vagrant
.vagrant
# Editor directories and files
**/.DS_Store
.idea
.vscode
*.suo
@@ -12,20 +8,12 @@
*.sln
*.sw?
*.dir-locals.el
# old, ad-hoc ignores
artifacts
local-artifacts
airgap-scp.sh
# test artifacts
*.tar*
*.out
# generated
dist/
tmp/
bin/
/store/
/registry/
cmd/hauler/binaries
cmd/hauler/binaries

View File

@@ -10,7 +10,7 @@ release:
env:
- vpkg=github.com/rancherfederal/hauler/internal/version
- cosign_version=v2.2.3+carbide.1
- cosign_version=v2.2.3+carbide.2
builds:
- main: cmd/hauler/main.go
@@ -36,14 +36,14 @@ universal_binaries:
- replace: false
changelog:
skip: false
disable: false
use: git
brews:
- name: hauler
tap:
repository:
owner: rancherfederal
name: homebrew-tap
token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}"
folder: Formula
directory: Formula
description: "Hauler CLI"

View File

@@ -1,7 +1,7 @@
SHELL:=/bin/bash
GO_FILES=$(shell go list ./... | grep -v /vendor/)
COSIGN_VERSION=v2.2.3+carbide.1
COSIGN_VERSION=v2.2.3+carbide.2
.SILENT:

View File

@@ -1,49 +0,0 @@
## Hauler Vagrant machine
A Vagrantfile is provided to allow easy provisioning of a local air-gapped CentOS environment. Some artifacts need to be collected from the internet; below are the steps required for successfully provisioning this machine, downloading all dependencies, and installing k3s (without hauler) into this machine.
### First-time setup
1. Install vagrant, if needed: <https://www.vagrantup.com/downloads>
2. Install `vagrant-vbguest` plugin, as noted in the Vagrantfile:
```shell
vagrant plugin install vagrant-vbguest
```
3. Deploy Vagrant machine, disabling SELinux:
```shell
SELINUX=Disabled vagrant up
```
4. Access the Vagrant machine via SSH:
```shell
vagrant ssh
```
5. Run all prep scripts inside of the Vagrant machine:
> This script temporarily enables internet access from within the VM to allow downloading all dependencies. Even so, the air-gapped network configuration IS restored before completion.
```shell
sudo /opt/hauler/vagrant-scripts/prep-all.sh
```
All dependencies for all `vagrant-scripts/*-install.sh` scripts are now downloaded to the local
repository under `local-artifacts`.
### Installing k3s manually
1. Access the Vagrant machine via SSH:
```bash
vagrant ssh
```
2. Run the k3s install script inside of the Vagrant machine:
```shell
sudo /opt/hauler/vagrant-scripts/k3s-install.sh
```
### Installing RKE2 manually
1. Access the Vagrant machine via SSH:
```shell
vagrant ssh
```
2. Run the RKE2 install script inside of the Vagrant machine:
```shell
sudo /opt/hauler/vagrant-scripts/rke2-install.sh
```

65
Vagrantfile vendored
View File

@@ -1,65 +0,0 @@
##################################
# The vagrant-vbguest plugin is required for CentOS 7.
# Run the following command to install/update this plugin:
# vagrant plugin install vagrant-vbguest
##################################
Vagrant.configure("2") do |config|
config.vm.box = "centos/8"
config.vm.hostname = "airgap"
config.vm.network "private_network", type: "dhcp"
config.vm.synced_folder ".", "/vagrant"
config.vm.provider "virtualbox" do |vb|
vb.memory = "2048"
vb.cpus = "2"
config.vm.provision "airgap", type: "shell", run: "always",
inline: "/vagrant/vagrant-scripts/airgap.sh airgap"
end
# SELinux is Enforcing by default.
# To set SELinux as Disabled on a VM that has already been provisioned:
# SELINUX=Disabled vagrant up --provision-with=selinux
# To set SELinux as Permissive on a VM that has already been provsioned
# SELINUX=Permissive vagrant up --provision-with=selinux
config.vm.provision "selinux", type: "shell", run: "once" do |sh|
sh.upload_path = "/tmp/vagrant-selinux"
sh.env = {
'SELINUX': ENV['SELINUX'] || "Enforcing"
}
sh.inline = <<~SHELL
#!/usr/bin/env bash
set -eux -o pipefail
if ! type -p getenforce setenforce &>/dev/null; then
echo SELinux is Disabled
exit 0
fi
case "${SELINUX}" in
Disabled)
if mountpoint -q /sys/fs/selinux; then
setenforce 0
umount -v /sys/fs/selinux
fi
;;
Enforcing)
mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
setenforce 1
;;
Permissive)
mountpoint -q /sys/fs/selinux || mount -o rw,relatime -t selinuxfs selinuxfs /sys/fs/selinux
setenforce 0
;;
*)
echo "SELinux mode not supported: ${SELINUX}" >&2
exit 1
;;
esac
echo SELinux is $(getenforce)
SHELL
end
end

View File

@@ -3,23 +3,24 @@ package cli
import (
"fmt"
"os"
"github.com/spf13/cobra"
)
func addCompletion(parent *cobra.Command) {
cmd := &cobra.Command{
Use: "completion",
Short: "Generates completion scripts for various shells",
Use: "completion",
Short: "Generates completion scripts for various shells",
Long: `The completion sub-command generates completion scripts for various shells.`,
}
cmd.AddCommand(
addCompletionZsh(),
addCompletionBash(),
addCompletionFish(),
addCompletionPowershell(),
)
parent.AddCommand(cmd)
}
@@ -34,19 +35,19 @@ func addCompletionZsh() *cobra.Command {
Short: "Generates zsh completion scripts",
Long: `The completion sub-command generates completion scripts for zsh.`,
Example: `To load completion run
. <(hauler completion zsh)
To configure your zsh shell to load completions for each session add to your zshrc
# ~/.zshrc or ~/.profile
command -v hauler >/dev/null && . <(hauler completion zsh)
or write a cached file in one of the completion directories in your ${fpath}:
echo "${fpath// /\n}" | grep -i completion
hauler completion zsh > _hauler
mv _hauler ~/.oh-my-zsh/completions # oh-my-zsh
mv _hauler ~/.zprezto/modules/completion/external/src/ # zprezto`,
Run: func(cmd *cobra.Command, args []string) {
@@ -64,11 +65,11 @@ func addCompletionBash() *cobra.Command {
Short: "Generates bash completion scripts",
Long: `The completion sub-command generates completion scripts for bash.`,
Example: `To load completion run
. <(hauler completion bash)
To configure your bash shell to load completions for each session add to your bashrc
# ~/.bashrc or ~/.profile
command -v hauler >/dev/null && . <(hauler completion bash)`,
Run: func(cmd *cobra.Command, args []string) {
@@ -84,9 +85,9 @@ func addCompletionFish() *cobra.Command {
Short: "Generates fish completion scripts",
Long: `The completion sub-command generates completion scripts for fish.`,
Example: `To configure your fish shell to load completions for each session write this script to your completions dir:
hauler completion fish > ~/.config/fish/completions/hauler.fish
See http://fishshell.com/docs/current/index.html#completion-own for more details`,
Run: func(cmd *cobra.Command, args []string) {
cmd.GenFishCompletion(os.Stdout, true)
@@ -101,18 +102,18 @@ func addCompletionPowershell() *cobra.Command {
Short: "Generates powershell completion scripts",
Long: `The completion sub-command generates completion scripts for powershell.`,
Example: `To load completion run
. <(hauler completion powershell)
To configure your powershell shell to load completions for each session add to your powershell profile
Windows:
cd "$env:USERPROFILE\Documents\WindowsPowerShell\Modules"
hauler completion powershell >> hauler-completion.ps1
Linux:
cd "${XDG_CONFIG_HOME:-"$HOME/.config/"}/powershell/modules"
hauler completion powershell >> hauler-completions.ps1`,
Run: func(cmd *cobra.Command, args []string) {
@@ -120,4 +121,4 @@ func addCompletionPowershell() *cobra.Command {
},
}
return cmd
}
}

View File

@@ -2,20 +2,20 @@ package cli
import (
"context"
"strings"
"os"
"io"
"fmt"
"github.com/spf13/cobra"
"io"
"os"
"strings"
"github.com/spf13/cobra"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/pkg/cosign"
)
type Opts struct {
Username string
Password string
Username string
Password string
PasswordStdin bool
}
@@ -35,7 +35,7 @@ func addLogin(parent *cobra.Command) {
Example: `
# Log in to reg.example.com
hauler login reg.example.com -u bob -p haulin`,
Args: cobra.ExactArgs(1),
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, arg []string) error {
ctx := cmd.Context()
@@ -47,7 +47,7 @@ hauler login reg.example.com -u bob -p haulin`,
o.Password = strings.TrimSuffix(string(contents), "\n")
o.Password = strings.TrimSuffix(o.Password, "\r")
}
if o.Username == "" && o.Password == "" {
return fmt.Errorf("username and password required")
}
@@ -62,14 +62,14 @@ hauler login reg.example.com -u bob -p haulin`,
func login(ctx context.Context, o *Opts, registry string) error {
ropts := content.RegistryOptions{
Username: o.Username,
Password: o.Password,
Username: o.Username,
Password: o.Password,
}
err := cosign.RegistryLogin(ctx, nil, registry, ropts)
if err != nil {
return err
}
return nil
}
}

View File

@@ -1,9 +1,10 @@
package cli
import (
"fmt"
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"fmt"
"github.com/rancherfederal/hauler/cmd/hauler/cli/store"
)
@@ -125,11 +126,11 @@ func addStoreServe() *cobra.Command {
// RegistryCmd serves the embedded registry
func addStoreServeRegistry() *cobra.Command {
o := &store.ServeRegistryOpts{RootOpts: rootStoreOpts}
o := &store.ServeRegistryOpts{RootOpts: rootStoreOpts}
cmd := &cobra.Command{
Use: "registry",
Short: "Serve the embedded registry",
RunE: func(cmd *cobra.Command, args []string) error {
Use: "registry",
Short: "Serve the embedded registry",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
@@ -138,21 +139,21 @@ func addStoreServeRegistry() *cobra.Command {
}
return store.ServeRegistryCmd(ctx, o, s)
},
}
},
}
o.AddFlags(cmd)
o.AddFlags(cmd)
return cmd
return cmd
}
// FileServerCmd serves the file server
func addStoreServeFiles() *cobra.Command {
o := &store.ServeFilesOpts{RootOpts: rootStoreOpts}
o := &store.ServeFilesOpts{RootOpts: rootStoreOpts}
cmd := &cobra.Command{
Use: "fileserver",
Short: "Serve the file server",
RunE: func(cmd *cobra.Command, args []string) error {
Use: "fileserver",
Short: "Serve the file server",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
@@ -161,12 +162,12 @@ func addStoreServeFiles() *cobra.Command {
}
return store.ServeFilesCmd(ctx, o, s)
},
}
},
}
o.AddFlags(cmd)
o.AddFlags(cmd)
return cmd
return cmd
}
func addStoreSave() *cobra.Command {
@@ -210,7 +211,7 @@ func addStoreInfo() *cobra.Command {
if err != nil {
return err
}
for _, allowed := range allowedValues {
if o.TypeFilter == allowed {
return store.InfoCmd(ctx, o, s)

View File

@@ -8,15 +8,13 @@ import (
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/content/chart"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
"github.com/rancherfederal/hauler/pkg/store"
)
type AddFileOpts struct {
@@ -103,7 +101,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform
if err != nil {
return err
}
err = cosign.SaveImage(ctx, s, r.Name(), platform)
if err != nil {
return err
@@ -147,7 +145,7 @@ func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Layout, chartNam
func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart, opts *action.ChartPathOptions) error {
l := log.FromContext(ctx)
l.Infof("adding 'chart' [%s] to the store", cfg.Name)
// TODO: This shouldn't be necessary
opts.RepoURL = cfg.RepoURL
opts.Version = cfg.Version

View File

@@ -9,9 +9,8 @@ import (
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
)
type CopyOpts struct {
@@ -55,7 +54,7 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string
Insecure: o.Insecure,
PlainHTTP: o.PlainHTTP,
}
if ropts.Username != "" {
err := cosign.RegistryLogin(ctx, s, components[1], ropts)
if err != nil {

View File

@@ -2,18 +2,17 @@ package store
import (
"context"
"strings"
"encoding/json"
"fmt"
"strings"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
"github.com/rancherfederal/hauler/pkg/store"
)
type ExtractOpts struct {
@@ -37,7 +36,7 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string
found := false
if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
if !strings.Contains(reference, r.Name()) {
return nil
}

View File

@@ -6,10 +6,10 @@ import (
"os"
"path/filepath"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
)
const (

View File

@@ -4,18 +4,16 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/olekukonko/tablewriter"
"os"
"sort"
"github.com/olekukonko/tablewriter"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/reference"
"github.com/rancherfederal/hauler/pkg/store"
)
type InfoOpts struct {
@@ -47,7 +45,7 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
}
defer rc.Close()
// handle multi-arch images
// handle multi-arch images
if desc.MediaType == consts.OCIImageIndexSchema || desc.MediaType == consts.DockerManifestListSchema2 {
var idx ocispec.Index
if err := json.NewDecoder(rc).Decode(&idx); err != nil {
@@ -72,13 +70,13 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
items = append(items, i)
}
}
// handle "non" multi-arch images
// handle "non" multi-arch images
} else if desc.MediaType == consts.DockerManifestSchema2 || desc.MediaType == consts.OCIManifestSchema1 {
var m ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
}
rc, err := s.FetchManifest(ctx, m)
if err != nil {
return err
@@ -90,7 +88,7 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
if err := json.NewDecoder(rc).Decode(&internalManifest); err != nil {
return err
}
if internalManifest.Architecture != "" {
i := newItem(s, desc, m, fmt.Sprintf("%s/%s", internalManifest.OS, internalManifest.Architecture), o)
var emptyItem item
@@ -104,8 +102,8 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
items = append(items, i)
}
}
// handle the rest
} else {
// handle the rest
} else {
var m ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
@@ -144,7 +142,7 @@ func buildTable(items ...item) {
table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
table.SetRowLine(false)
table.SetAutoMergeCellsByColumnIndex([]int{0})
totalSize := int64(0)
for _, i := range items {
if i.Type != "" {
@@ -173,11 +171,11 @@ func buildJson(item ...item) string {
}
type item struct {
Reference string
Type string
Platform string
Layers int
Size int64
Reference string
Type string
Platform string
Layers int
Size int64
}
type byReferenceAndArch []item
@@ -227,7 +225,7 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest, plat
case "dev.cosignproject.cosign/sboms":
ctype = "sbom"
}
ref, err := reference.Parse(desc.Annotations[ocispec.AnnotationRefName])
if err != nil {
return item{}
@@ -238,11 +236,11 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest, plat
}
return item{
Reference: ref.Name(),
Type: ctype,
Platform: plat,
Layers: len(m.Layers),
Size: size,
Reference: ref.Name(),
Type: ctype,
Platform: plat,
Layers: len(m.Layers),
Size: size,
}
}

View File

@@ -5,11 +5,11 @@ import (
"os"
"github.com/mholt/archiver/v3"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
)
type LoadOpts struct {

View File

@@ -14,10 +14,9 @@ import (
"github.com/distribution/distribution/v3/version"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/internal/server"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
)
type ServeRegistryOpts struct {
@@ -80,6 +79,7 @@ type ServeFilesOpts struct {
*RootOpts
Port int
Timeout int
RootDir string
storedir string
@@ -89,6 +89,7 @@ func (o *ServeFilesOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.IntVarP(&o.Port, "port", "p", 8080, "Port to listen on.")
f.IntVarP(&o.Timeout, "timeout", "t", 60, "Set the http request timeout duration in seconds for both reads and write.")
f.StringVar(&o.RootDir, "directory", "fileserver", "Directory to use for backend. Defaults to $PWD/fileserver")
}
@@ -102,8 +103,9 @@ func ServeFilesCmd(ctx context.Context, o *ServeFilesOpts, s *store.Layout) erro
}
cfg := server.FileConfig{
Root: o.RootDir,
Port: o.Port,
Root: o.RootDir,
Port: o.Port,
Timeout: o.Timeout,
}
f, err := server.NewFile(ctx, cfg)

View File

@@ -27,18 +27,18 @@ import (
type SyncOpts struct {
*RootOpts
ContentFiles []string
Key string
Products []string
Platform string
Registry string
ContentFiles []string
Key string
Products []string
Platform string
Registry string
ProductRegistry string
}
func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path(s) to local content files (Manifests). i.e. '--files ./rke2-files.yml")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for signature verification")
f.StringSliceVar(&o.Products, "products", []string{}, "Used for RGS Carbide customers to supply a product and version and Hauler will retrieve the images. i.e. '--product rancher=v2.7.6'")
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specific platform to save. i.e. linux/amd64. Defaults to all if flag is omitted.")
@@ -70,7 +70,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
if err != nil {
return err
}
err = ExtractCmd(ctx, &ExtractOpts{RootOpts: o.RootOpts}, s, fmt.Sprintf("hauler/%s-manifest.yaml:%s", parts[0],tag))
err = ExtractCmd(ctx, &ExtractOpts{RootOpts: o.RootOpts}, s, fmt.Sprintf("hauler/%s-manifest.yaml:%s", parts[0], tag))
if err != nil {
return err
}
@@ -151,19 +151,19 @@ func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layo
}
a := cfg.GetAnnotations()
for _, i := range cfg.Spec.Images {
// Check if the user provided a registry. If a registry is provided in the annotation, use it for the images that don't have a registry in their ref name.
if a[consts.ImageAnnotationRegistry] != "" || o.Registry != ""{
newRef,_ := reference.Parse(i.Name)
if a[consts.ImageAnnotationRegistry] != "" || o.Registry != "" {
newRef, _ := reference.Parse(i.Name)
newReg := o.Registry // cli flag
// if no cli flag but there was an annotation, use the annotation.
if o.Registry == "" && a[consts.ImageAnnotationRegistry] != "" {
newReg = a[consts.ImageAnnotationRegistry]
}
if newRef.Context().RegistryStr() == "" {
newRef,err = reference.Relocate(i.Name, newReg)
newRef, err = reference.Relocate(i.Name, newReg)
if err != nil {
return err
}
@@ -189,7 +189,7 @@ func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layo
}
}
l.Debugf("key for image [%s]", key)
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
@@ -209,7 +209,7 @@ func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layo
if i.Platform != "" {
platform = i.Platform
}
err = storeImage(ctx, s, i, platform)
if err != nil {
return err

View File

@@ -0,0 +1,21 @@
# HELM IGNORE OPTIONS:
# Patterns to ignore when building Helm packages.
# Supports shell glob matching, relative path matching, and negation (prefixed with !)
.DS_Store
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
*.swp
*.bak
*.tmp
*.orig
*~
.project
.idea/
*.tmproj
.vscode/

View File

@@ -0,0 +1,7 @@
apiVersion: v2
name: hauler
description: Hauler Helm Chart - Airgap Swiss Army Knife
icon: https://raw.githubusercontent.com/rancherfederal/hauler/main/static/rgs-hauler-logo-icon.svg
type: application
version: 1.0.3
appVersion: 1.0.3

View File

@@ -0,0 +1,33 @@
# Hauler Helm Chart
### Airgap Swiss Army Knife
`Rancher Government Hauler` simplifies the airgap experience without requiring operators to adopt a specific workflow. **Hauler** simplifies the airgapping process, by representing assets (images, charts, files, etc...) as content and collections to allow operators to easily fetch, store, package, and distribute these assets with declarative manifests or through the command line.
`Hauler` does this by storing contents and collections as OCI Artifacts and allows operators to serve contents and collections with an embedded registry and fileserver. Additionally, `Hauler` has the ability to store and inspect various non-image OCI Artifacts.
**GitHub Repostiory:** https://github.com/rancherfederal/hauler
**Documentation:** http://hauler.dev
---
| Type | Chart Version | App Version |
| ----------- | ------------- | ----------- |
| application | `1.0.3` | `1.0.3` |
## Installing the Chart
```bash
helm install hauler hauler/hauler -n hauler-system -f values.yaml
```
```bash
helm status hauler -n hauler-system
```
## Uninstalling the Chart
```bash
helm uninstall hauler -n hauler-system
```

View File

@@ -0,0 +1,33 @@
# Hauler Helm Chart
### Airgap Swiss Army Knife
`Rancher Government Hauler` simplifies the airgap experience without requiring operators to adopt a specific workflow. **Hauler** simplifies the airgapping process, by representing assets (images, charts, files, etc...) as content and collections to allow operators to easily fetch, store, package, and distribute these assets with declarative manifests or through the command line.
`Hauler` does this by storing contents and collections as OCI Artifacts and allows operators to serve contents and collections with an embedded registry and fileserver. Additionally, `Hauler` has the ability to store and inspect various non-image OCI Artifacts.
**GitHub Repostiory:** https://github.com/rancherfederal/hauler
**Documentation:** http://hauler.dev
---
| Type | Chart Version | App Version |
| ----------- | ------------- | ----------- |
| application | `1.0.3` | `1.0.3` |
## Installing the Chart
```bash
helm install hauler hauler/hauler -n hauler-system -f values.yaml
```
```bash
helm status hauler -n hauler-system
```
## Uninstalling the Chart
```bash
helm uninstall hauler -n hauler-system
```

View File

@@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "hauler.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "hauler.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "hauler.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "hauler.labels" -}}
helm.sh/chart: {{ include "hauler.chart" . }}
{{ include "hauler.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "hauler.selectorLabels" -}}
app.kubernetes.io/name: {{ include "hauler.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "hauler.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "hauler.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

View File

@@ -0,0 +1,76 @@
{{- if .Values.haulerFileserver.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: hauler-fileserver
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.haulerFileserver.replicas }}
selector:
matchLabels:
app: hauler-fileserver
{{- include "hauler.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
app: hauler-fileserver
{{- include "hauler.selectorLabels" . | nindent 8 }}
spec:
{{- if or .Values.haulerJobs.hauls.enabled .Values.haulerJobs.manifests.enabled }}
initContainers:
{{- if .Values.haulerJobs.hauls.enabled }}
- name: wait-for-hauler-hauls-job
image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }}
imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }}
args: ["wait", "--for=condition=complete", "job", "hauler-hauls-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
{{- end }}
{{- if .Values.haulerJobs.manifests.enabled }}
- name: wait-for-hauler-manifests-job
image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }}
imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }}
args: ["wait", "--for=condition=complete", "job", "hauler-manifests-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
{{- end }}
{{- end }}
containers:
- name: hauler-fileserver
image: {{ .Values.hauler.image.repository }}:{{ .Values.hauler.image.tag }}
imagePullPolicy: {{ .Values.hauler.imagePullPolicy }}
args: ["store", "serve", "fileserver", "--port", "{{ .Values.haulerFileserver.port }}"]
ports:
- containerPort: {{ .Values.haulerFileserver.port }}
volumeMounts:
- name: hauler-data
mountPath: /store
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
restartPolicy: Always
serviceAccountName: hauler-service-account
volumes:
- name: hauler-data
persistentVolumeClaim:
claimName: hauler-data
{{- end }}

View File

@@ -0,0 +1,27 @@
{{- if and .Values.haulerFileserver.enabled .Values.haulerFileserver.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: hauler-fileserver
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
rules:
- host: {{ .Values.haulerFileserver.ingress.hostname }}
http:
paths:
- backend:
service:
name: hauler-fileserver
port:
number: {{ .Values.haulerFileserver.service.ports.targetPort }}
path: /
pathType: Prefix
{{- if .Values.haulerFileserver.ingress.tls.enabled }}
tls:
- hosts:
- {{ .Values.haulerFileserver.ingress.hostname }}
secretName: {{ .Values.haulerFileserver.ingress.tls.secretName }}
{{- end }}
{{- end }}

View File

@@ -0,0 +1,18 @@
{{- if and .Values.haulerFileserver.enabled .Values.haulerFileserver.service.enabled }}
apiVersion: v1
kind: Service
metadata:
name: hauler-fileserver
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
selector:
app: hauler-fileserver
ports:
- name: hauler-fileserver
protocol: {{ .Values.haulerFileserver.service.ports.protocol }}
port: {{ .Values.haulerFileserver.service.ports.port }}
targetPort: {{ .Values.haulerFileserver.service.ports.targetPort }}
type: {{ .Values.haulerFileserver.service.type }}
{{- end }}

View File

@@ -0,0 +1,76 @@
{{- if .Values.haulerRegistry.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: hauler-registry
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.haulerRegistry.replicas }}
selector:
matchLabels:
app: hauler-registry
{{- include "hauler.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
app: hauler-registry
{{- include "hauler.selectorLabels" . | nindent 8 }}
spec:
{{- if or .Values.haulerJobs.hauls.enabled .Values.haulerJobs.manifests.enabled }}
initContainers:
{{- if .Values.haulerJobs.hauls.enabled }}
- name: wait-for-hauler-hauls-job
image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }}
imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }}
args: ["wait", "--for=condition=complete", "job", "hauler-hauls-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
{{- end }}
{{- if .Values.haulerJobs.manifests.enabled }}
- name: wait-for-hauler-manifests-job
image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }}
imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }}
args: ["wait", "--for=condition=complete", "job", "hauler-manifests-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
{{- end }}
{{- end }}
containers:
- name: hauler-registry
image: {{ .Values.hauler.image.repository }}:{{ .Values.hauler.image.tag }}
imagePullPolicy: {{ .Values.hauler.imagePullPolicy }}
args: ["store", "serve", "registry", "--port", "{{ .Values.haulerRegistry.port }}"]
ports:
- containerPort: {{ .Values.haulerRegistry.port }}
volumeMounts:
- name: hauler-data
mountPath: /store
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
restartPolicy: Always
serviceAccountName: hauler-service-account
volumes:
- name: hauler-data
persistentVolumeClaim:
claimName: hauler-data
{{- end }}

View File

@@ -0,0 +1,28 @@
{{- if and .Values.haulerRegistry.enabled .Values.haulerRegistry.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: hauler-registry
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
rules:
- host: {{ .Values.haulerRegistry.ingress.hostname }}
http:
paths:
- backend:
service:
name: hauler-registry
port:
number: {{ .Values.haulerRegistry.service.ports.targetPort }}
path: /
pathType: Prefix
{{- if .Values.haulerRegistry.ingress.tls.enabled }}
tls:
- hosts:
- {{ .Values.haulerRegistry.ingress.hostname }}
secretName: {{ .Values.haulerRegistry.ingress.tls.secretName }}
{{- end }}
{{- end }}

View File

@@ -0,0 +1,18 @@
{{- if and .Values.haulerRegistry.enabled .Values.haulerRegistry.service.enabled }}
apiVersion: v1
kind: Service
metadata:
name: hauler-registry
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
selector:
app: hauler-registry
ports:
- name: hauler-registry
protocol: {{ .Values.haulerRegistry.service.ports.protocol }}
port: {{ .Values.haulerRegistry.service.ports.port }}
targetPort: {{ .Values.haulerRegistry.service.ports.targetPort }}
type: {{ .Values.haulerRegistry.service.type }}
{{- end }}

View File

@@ -0,0 +1,126 @@
{{- if .Values.haulerJobs.hauls.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
name: hauler-hauls-job
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
template:
spec:
initContainers:
- name: hauler-fetch-hauls
image: {{ .Values.haulerJobs.image.repository }}:{{ .Values.haulerJobs.image.tag }}
imagePullPolicy: {{ .Values.haulerJobs.imagePullPolicy }}
command: ["/bin/sh", "-c"]
args:
- |
{{- range .Values.haulerJobs.hauls.artifacts }}
curl -o /hauls/{{ .name }} {{ .path }} &&
{{- end }}
echo hauler fetch completed
volumeMounts:
- name: hauler-data
mountPath: /hauls
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
containers:
- name: hauler-load-hauls
image: {{ .Values.hauler.image.repository }}:{{ .Values.hauler.image.tag }}
imagePullPolicy: {{ .Values.hauler.imagePullPolicy }}
args:
- "store"
- "load"
{{- range .Values.haulerJobs.hauls.artifacts }}
- "/hauls/{{ .name }}"
{{- end }}
volumeMounts:
- name: hauler-data
mountPath: /hauls
- name: hauler-data
mountPath: /store
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
restartPolicy: OnFailure
volumes:
- name: hauler-data
persistentVolumeClaim:
claimName: hauler-data
{{- end }}
---
{{- if .Values.haulerJobs.manifests.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
name: hauler-manifests-job
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
template:
spec:
initContainers:
- name: hauler-fetch-manifests
image: {{ .Values.haulerJobs.image.repository }}:{{ .Values.haulerJobs.image.tag }}
imagePullPolicy: {{ .Values.haulerJobs.imagePullPolicy }}
command: ["/bin/sh", "-c"]
args:
- |
{{- range .Values.haulerJobs.manifests.artifacts }}
curl -o /manifests/{{ .name }} {{ .path }} &&
{{- end }}
echo hauler fetch completed
volumeMounts:
- name: hauler-data
mountPath: /manifests
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
containers:
- name: hauler-load-manifests
image: {{ .Values.hauler.image.repository }}:{{ .Values.hauler.image.tag }}
imagePullPolicy: {{ .Values.hauler.imagePullPolicy }}
args:
{{- range .Values.haulerJobs.manifests.artifacts }}
- "store"
- "sync"
- "--files"
- "/manifests/{{ .name }}"
{{- end }}
volumeMounts:
- name: hauler-data
mountPath: /manifests
- name: hauler-data
mountPath: /store
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1001
seccompProfile:
type: RuntimeDefault
restartPolicy: OnFailure
volumes:
- name: hauler-data
persistentVolumeClaim:
claimName: hauler-data
{{- end }}

View File

@@ -0,0 +1,16 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: hauler-data
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
spec:
accessModes:
- {{ .Values.hauler.data.pvc.accessModes }}
resources:
requests:
storage: {{ .Values.hauler.data.pvc.storageRequest }}
{{- if .Values.hauler.data.pvc.storageClass }}
storageClassName: {{ .Values.hauler.data.pvc.storageClass }}
{{- end }}

View File

@@ -0,0 +1,35 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: hauler-service-account
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: hauler-role
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
rules:
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: hauler-role-binding
namespace: {{ .Release.Namespace }}
labels:
{{- include "hauler.labels" . | nindent 4 }}
roleRef:
kind: Role
name: hauler-role
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: hauler-service-account
namespace: {{ .Release.Namespace }}

View File

@@ -0,0 +1,94 @@
# Helm Chart Values for Hauler
# Docs: https://hauler.dev
hauler:
image:
repository: ghcr.io/rancherfederal/hauler
tag: v1.0.3
imagePullPolicy: Always
initContainers:
image:
repository: rancher/kubectl
tag: v1.27.0 # update to your kubernetes version
imagePullPolicy: Always
timeout: 1h
data:
pvc:
accessModes: ReadWriteMany
storageClass: longhorn # optional... will use default storage class
storageRequest: 48Gi # recommended size of 3x the artifact(s)
# Helm Chart Values for the Hauler Jobs
# Docs: https://rancherfederal.github.io/hauler-docs/docs/introduction/quickstart
haulerJobs:
image:
repository: rancher/shell
tag: v0.1.22
imagePullPolicy: Always
hauls:
enabled: true
artifacts:
- path: https://raw.githubusercontent.com/rancherfederal/hauler/main/testdata/haul.tar.zst
name: haul.tar.zst
# - path: /path/to/additional-hauls.tar.zst
# name: additional-hauls.tar.zst
manifests:
enabled: true
artifacts:
- path: https://raw.githubusercontent.com/rancherfederal/hauler/main/testdata/hauler-manifest.yaml
name: hauler-manifest.yaml
# - path: /path/to/additional-manifests.yaml
# name: additional-manifests.yaml
# Helm Chart Values for the Hauler Fileserver
# Docs: https://rancherfederal.github.io/hauler-docs/docs/guides-references/command-line/hauler-store#hauler-store-serve-fileserver
haulerFileserver:
enabled: true
port: 8080 # default port for the fileserver
replicas: 1
ingress:
enabled: true
hostname: fileserver.ranchers.io
tls:
enabled: true
source: secret # only supported source
secretName: tls-certs # must be created outside of this chart
service:
enabled: true
type: ClusterIP
ports:
protocol: TCP
port: 8080 # default port for the fileserver
targetPort: 8080 # default port for the fileserver
# Helm Chart Values for the Hauler Registry
# Docs: https://rancherfederal.github.io/hauler-docs/docs/guides-references/command-line/hauler-store#hauler-store-serve-registry
haulerRegistry:
enabled: true
port: 5000 # default port for the registry
replicas: 1
ingress:
enabled: true
hostname: registry.ranchers.io
tls:
enabled: true
source: secret # only supported source
secretName: tls-certs # must be created outside of this chart
service:
enabled: true
type: ClusterIP
ports:
protocol: TCP
port: 5000 # default port for the registry
targetPort: 5000 # default port for the registry

8
go.mod
View File

@@ -144,11 +144,11 @@ require (
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
golang.org/x/crypto v0.18.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/term v0.16.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect

16
go.sum
View File

@@ -564,8 +564,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -638,8 +638,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -713,14 +713,14 @@ golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

View File

@@ -2,7 +2,7 @@ package mapper
import (
"context"
"io/ioutil"
"io"
"os"
"path/filepath"
"strings"
@@ -15,7 +15,8 @@ import (
)
// NewMapperFileStore creates a new file store that uses mapper functions for each detected descriptor.
// This extends content.File, and differs in that it allows much more functionality into how each descriptor is written.
//
// This extends content.File, and differs in that it allows much more functionality into how each descriptor is written.
func NewMapperFileStore(root string, mapper map[string]Fn) *store {
fs := content.NewFile(root)
return &store{
@@ -58,7 +59,7 @@ func (s *pusher) Push(ctx context.Context, desc ocispec.Descriptor) (ccontent.Wr
// If no custom mapper found, fall back to content.File mapper
if _, ok := s.mapper[desc.MediaType]; !ok {
return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(desc.Digest)), nil
return content.NewIoContentWriter(io.Discard, content.WithOutputHash(desc.Digest)), nil
}
filename, err := s.mapper[desc.MediaType](desc)

View File

@@ -12,9 +12,10 @@ import (
)
type FileConfig struct {
Root string
Host string
Port int
Root string
Host string
Port int
Timeout int
}
// NewFile returns a fileserver
@@ -30,11 +31,15 @@ func NewFile(ctx context.Context, cfg FileConfig) (Server, error) {
cfg.Port = 8080
}
if cfg.Timeout == 0 {
cfg.Timeout = 60
}
srv := &http.Server{
Handler: r,
Addr: fmt.Sprintf(":%d", cfg.Port),
WriteTimeout: 15 * time.Second,
ReadTimeout: 15 * time.Second,
WriteTimeout: time.Duration(cfg.Timeout) * time.Second,
ReadTimeout: time.Duration(cfg.Timeout) * time.Second,
}
return srv, nil

View File

@@ -47,7 +47,7 @@ func NewTempRegistry(ctx context.Context, root string) *tmpRegistryServer {
}
// Add validation configuration
cfg.Validation.Manifests.URLs.Allow = []string{".+"}
cfg.Log.Level = "error"
cfg.HTTP.Headers = http.Header{
"X-Content-Type-Options": []string{"nosniff"},

View File

@@ -226,4 +226,4 @@ func (i *Info) CheckFontName(fontName string) bool {
fmt.Fprintln(os.Stderr, "font not valid, using default")
return false
}
}

View File

@@ -20,7 +20,7 @@ type ImageSpec struct {
type Image struct {
// Name is the full location for the image, can be referenced by tags or digests
Name string `json:"name"`
// Path is the path to the cosign public key used for verifying image signatures
//Key string `json:"key,omitempty"`
Key string `json:"key"`

View File

@@ -1,6 +1,8 @@
package v1alpha1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const K3sCollectionKind = "K3s"

View File

@@ -54,27 +54,27 @@ func NewImage(name string, opts ...remote.Option) (*Image, error) {
}
func IsMultiArchImage(name string, opts ...remote.Option) (bool, error) {
ref, err := gname.ParseReference(name)
if err != nil {
return false, fmt.Errorf("parsing reference %q: %v", name, err)
}
ref, err := gname.ParseReference(name)
if err != nil {
return false, fmt.Errorf("parsing reference %q: %v", name, err)
}
defaultOpts := []remote.Option{
remote.WithAuthFromKeychain(authn.DefaultKeychain),
}
opts = append(opts, defaultOpts...)
desc, err := remote.Get(ref, opts...)
if err != nil {
return false, fmt.Errorf("getting image %q: %v", name, err)
}
desc, err := remote.Get(ref, opts...)
if err != nil {
return false, fmt.Errorf("getting image %q: %v", name, err)
}
_, err = desc.ImageIndex()
if err != nil {
// If the descriptor could not be converted to an image index, it's not a multi-arch image
return false, nil
}
_, err = desc.ImageIndex()
if err != nil {
// If the descriptor could not be converted to an image index, it's not a multi-arch image
return false, nil
}
// If the descriptor could be converted to an image index, it's a multi-arch image
return true, nil
}
// If the descriptor could be converted to an image index, it's a multi-arch image
return true, nil
}

View File

@@ -3,8 +3,9 @@ package artifacts
import "github.com/google/go-containerregistry/pkg/v1"
// OCI is the bare minimum we need to represent an artifact in an oci layout
// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality
// This specific implementation fully encapsulates v1.Layer's within a more generic form
//
// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality
// This specific implementation fully encapsulates v1.Layer's within a more generic form
type OCI interface {
MediaType() string

View File

@@ -1,11 +1,11 @@
package chart
import (
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/content/chart"
"github.com/rancherfederal/hauler/pkg/reference"
)

View File

@@ -9,12 +9,12 @@ import (
"strings"
"sync"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/google/go-containerregistry/pkg/name"
artifact "github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/log"
)
type ImageTxt struct {

View File

@@ -11,12 +11,9 @@ import (
"strings"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/reference"
)

View File

@@ -50,8 +50,8 @@ const (
KindAnnotationName = "kind"
KindAnnotation = "dev.cosignproject.cosign/image"
CarbideRegistry = "rgcrprod.azurecr.us"
ImageAnnotationKey = "hauler.dev/key"
CarbideRegistry = "rgcrprod.azurecr.us"
ImageAnnotationKey = "hauler.dev/key"
ImageAnnotationPlatform = "hauler.dev/platform"
ImageAnnotationRegistry = "hauler.dev/registry"
)

View File

@@ -5,8 +5,10 @@ import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
@@ -15,17 +17,21 @@ import (
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/log"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"github.com/rancherfederal/hauler/pkg/layer"
"helm.sh/helm/v3/pkg/registry"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/layer"
)
var _ artifacts.OCI = (*Chart)(nil)
var (
_ artifacts.OCI = (*Chart)(nil)
settings = cli.New()
)
// Chart implements the OCI interface for Chart API objects. API spec values are
// stored into the Repo, Name, and Version fields.
@@ -36,22 +42,31 @@ type Chart struct {
// NewChart is a helper method that returns NewLocalChart or NewRemoteChart depending on v1alpha1.Chart contents
func NewChart(name string, opts *action.ChartPathOptions) (*Chart, error) {
cpo := action.ChartPathOptions{
RepoURL: opts.RepoURL,
Version: opts.Version,
CaFile: opts.CaFile,
CertFile: opts.CertFile,
KeyFile: opts.KeyFile,
InsecureSkipTLSverify: opts.InsecureSkipTLSverify,
Keyring: opts.Keyring,
Password: opts.Password,
PassCredentialsAll: opts.PassCredentialsAll,
Username: opts.Username,
Verify: opts.Verify,
chartRef := name
actionConfig := new(action.Configuration)
if err := actionConfig.Init(settings.RESTClientGetter(), settings.Namespace(), os.Getenv("HELM_DRIVER"), log.NewLogger(os.Stdout).Debugf); err != nil {
return nil, err
}
chartPath, err := cpo.LocateChart(name, cli.New())
client := action.NewInstall(actionConfig)
client.ChartPathOptions.Version = opts.Version
registryClient, err := newRegistryClient(client.CertFile, client.KeyFile, client.CaFile,
client.InsecureSkipTLSverify, client.PlainHTTP)
if err != nil {
return nil, fmt.Errorf("missing registry client: %w", err)
}
client.SetRegistryClient(registryClient)
if registry.IsOCI(opts.RepoURL) {
chartRef = opts.RepoURL + "/" + name
} else if isUrl(opts.RepoURL) { // OCI Protocol registers as a valid URL
client.ChartPathOptions.RepoURL = opts.RepoURL
} else { // Handles cases like grafana/loki
chartRef = opts.RepoURL + "/" + name
}
chartPath, err := client.ChartPathOptions.LocateChart(chartRef, settings)
if err != nil {
return nil, err
}
@@ -217,3 +232,52 @@ func (h *Chart) chartData() (gv1.Layer, error) {
return chartDataLayer, err
}
func isUrl(name string) bool {
_, err := url.ParseRequestURI(name)
return err == nil
}
func newRegistryClient(certFile, keyFile, caFile string, insecureSkipTLSverify, plainHTTP bool) (*registry.Client, error) {
if certFile != "" && keyFile != "" || caFile != "" || insecureSkipTLSverify {
registryClient, err := newRegistryClientWithTLS(certFile, keyFile, caFile, insecureSkipTLSverify)
if err != nil {
return nil, err
}
return registryClient, nil
}
registryClient, err := newDefaultRegistryClient(plainHTTP)
if err != nil {
return nil, err
}
return registryClient, nil
}
func newDefaultRegistryClient(plainHTTP bool) (*registry.Client, error) {
opts := []registry.ClientOption{
registry.ClientOptDebug(settings.Debug),
registry.ClientOptEnableCache(true),
registry.ClientOptWriter(os.Stderr),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
}
if plainHTTP {
opts = append(opts, registry.ClientOptPlainHTTP())
}
// Create a new registry client
registryClient, err := registry.NewClient(opts...)
if err != nil {
return nil, err
}
return registryClient, nil
}
func newRegistryClientWithTLS(certFile, keyFile, caFile string, insecureSkipTLSverify bool) (*registry.Client, error) {
// Create a new registry client
registryClient, err := registry.NewRegistryClientWithTLS(os.Stderr, certFile, keyFile, caFile, insecureSkipTLSverify,
settings.RegistryConfig, settings.Debug,
)
if err != nil {
return nil, err
}
return registryClient, nil
}

View File

@@ -6,19 +6,13 @@ import (
"testing"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/mholt/archiver/v3"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content/chart"
)
var (
chartpath = "../../../testdata/podinfo-6.0.3.tgz"
)
func TestNewChart(t *testing.T) {
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
@@ -26,10 +20,6 @@ func TestNewChart(t *testing.T) {
}
defer os.RemoveAll(tmpdir)
if err := archiver.Unarchive(chartpath, tmpdir); err != nil {
t.Fatal(err)
}
type args struct {
name string
opts *action.ChartPathOptions
@@ -43,18 +33,18 @@ func TestNewChart(t *testing.T) {
{
name: "should create from a chart archive",
args: args{
name: chartpath,
opts: &action.ChartPathOptions{},
name: "rancher-cluster-templates-0.4.4.tgz",
opts: &action.ChartPathOptions{RepoURL: "../../../testdata"},
},
want: v1.Descriptor{
MediaType: consts.ChartLayerMediaType,
Size: 13524,
Size: 13102,
Digest: v1.Hash{
Algorithm: "sha256",
Hex: "e30b95a08787de69ffdad3c232d65cfb131b5b50c6fd44295f48a078fceaa44e",
Hex: "4b3bb4e474b54bf9057b298f8f11c239bb561396716d8cd5fc369c407fba2965",
},
Annotations: map[string]string{
ocispec.AnnotationTitle: "podinfo-6.0.3.tgz",
ocispec.AnnotationTitle: "rancher-cluster-templates-0.4.4.tgz",
},
},
wantErr: false,
@@ -72,18 +62,18 @@ func TestNewChart(t *testing.T) {
// TODO: Use a mock helm server
name: "should fetch a remote chart",
args: args{
name: "ingress-nginx",
opts: &action.ChartPathOptions{RepoURL: "https://kubernetes.github.io/ingress-nginx", Version: "4.0.16"},
name: "cert-manager",
opts: &action.ChartPathOptions{RepoURL: "https://charts.jetstack.io", Version: "1.14.4"},
},
want: v1.Descriptor{
MediaType: consts.ChartLayerMediaType,
Size: 38591,
Size: 80674,
Digest: v1.Hash{
Algorithm: "sha256",
Hex: "b0ea91f7febc6708ad9971871d2de6e8feb2072110c3add6dd7082d90753caa2",
Hex: "5775fdbc1881d6e510df76d38753af54b86bd14caa8edb28fdbb79527042dede",
},
Annotations: map[string]string{
ocispec.AnnotationTitle: "ingress-nginx-4.0.16.tgz",
ocispec.AnnotationTitle: "cert-manager-v1.14.4.tgz",
},
},
wantErr: false,

View File

@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
@@ -275,7 +274,7 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr
if _, err := os.Stat(blobPath); err == nil {
// file already exists, discard (but validate digest)
return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil
return content.NewIoContentWriter(io.Discard, content.WithOutputHash(d.Digest)), nil
}
f, err := os.Create(blobPath)

View File

@@ -13,10 +13,11 @@ import (
"strings"
"time"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/store"
"oras.land/oras-go/pkg/content"
)
const maxRetries = 3
@@ -276,4 +277,4 @@ func getCosignPath() (string, error) {
binaryPath := filepath.Join(haulerDir, binaryName)
return binaryPath, nil
}
}

View File

@@ -30,13 +30,13 @@ type Fields map[string]string
// NewLogger returns a new Logger
func NewLogger(out io.Writer) Logger {
customTimeFormat := "2006-01-02 15:04:05"
zerolog.TimeFieldFormat = customTimeFormat
output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: customTimeFormat}
l := log.Output(output)
return &logger{
zl: l.With().Timestamp().Logger(),
}
customTimeFormat := "2006-01-02 15:04:05"
zerolog.TimeFieldFormat = customTimeFormat
output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: customTimeFormat}
l := log.Output(output)
return &logger{
zl: l.With().Timestamp().Logger(),
}
}
// FromContext returns a Logger from a context if it exists

View File

@@ -1,7 +1,7 @@
// Package reference provides general types to represent oci content within a registry or local oci layout
// Grammar (stolen mostly from containerd's grammar)
//
// reference :=
// reference :=
package reference
import (

View File

@@ -1,23 +0,0 @@
---
apiVersion: collection.hauler.cattle.io/v1alpha1
kind: ThickCharts
metadata:
name: mythickchart
spec:
charts:
# charts are also fetched and served as OCI content (currently experimental in helm)
# HELM_EXPERIMENTAL_OCI=1 helm chart pull <hauler-registry>/loki:2.6.2
# - name: loki
# repoURL: https://grafana.github.io/helm-charts
# - name: longhorn
# repoURL: https://charts.longhorn.io
# - name: cert-manager
# repoURL: https://charts.jetstack.io
# version: v1.6.1
# extraImages:
# - ref: quay.io/jetstack/cert-manager-cainjector:v1.6.1
- name: podinfo
repoURL: https://stefanprodan.github.io/podinfo

View File

@@ -1,56 +0,0 @@
apiVersion: content.hauler.cattle.io/v1alpha1
kind: Files
metadata:
name: myfile
spec:
files:
# hauler can save/redistribute files on disk (be careful! paths are relative)
- path: testdata/contents.yaml
# when directories are specified, the directory contents will be archived and stored
- path: testdata/
# hauler can also fetch remote content, and will "smartly" identify filenames _when possible_
# filename below = "k3s-images.txt"
- path: "https://github.com/k3s-io/k3s/releases/download/v1.22.2%2Bk3s2/k3s-images.txt"
# when discovered filenames are not desired, a file name can be specified
- path: https://get.k3s.io
name: k3s-init.sh
---
apiVersion: content.hauler.cattle.io/v1alpha1
kind: Images
metadata:
name: myimage
spec:
images:
# images can be referenced shorthanded without a tag
- name: hello-world
# or namespaced with a tag
- name: rancher/cowsay:latest
# or by their digest:
- name: registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f
# or fully qualified from any OCI compliant registry registry
- name: ghcr.io/fluxcd/flux-cli:v0.22.0
---
apiVersion: content.hauler.cattle.io/v1alpha1
kind: Charts
metadata:
name: mychart
spec:
charts:
# charts are also fetched and served as OCI content (currently experimental in helm)
# HELM_EXPERIMENTAL_OCI=1 helm chart pull <hauler-registry>/loki:2.6.2
- name: loki
repoURL: https://grafana.github.io/helm-charts
# version: latest # the latest version will be used when version is empty
# specific versions can also be used
- name: rancher
repoURL: https://releases.rancher.com/server-charts/latest
version: 2.6.2

BIN
testdata/haul.tar.zst vendored Executable file

Binary file not shown.

26
testdata/hauler-manifest.yaml vendored Executable file
View File

@@ -0,0 +1,26 @@
apiVersion: content.hauler.cattle.io/v1alpha1
kind: Images
metadata:
name: hauler-content-images-example
spec:
images:
- name: busybox:latest
---
apiVersion: content.hauler.cattle.io/v1alpha1
kind: Charts
metadata:
name: hauler-content-charts-example
spec:
charts:
- name: rancher
repoURL: https://releases.rancher.com/server-charts/stable
version: 2.8.2
---
apiVersion: content.hauler.cattle.io/v1alpha1
kind: Files
metadata:
name: hauler-content-files-example
spec:
files:
- path: https://get.rke2.io
name: install.sh

View File

@@ -1,13 +0,0 @@
---
apiVersion: collection.hauler.cattle.io/v1alpha1
kind: K3s
metadata:
name: myk3s
spec:
# version can be exact (as listed on https://github.com/k3s-io/k3s/releases)
version: v1.22.2+k3s2
# or can point to a channel, in which case the latest version in the channel is used: https://update.k3s.io/v1-release/channels
# version: stable
# version: latest
# version: v1.22

Binary file not shown.

BIN
testdata/rancher-cluster-templates-0.4.4.tgz vendored Executable file

Binary file not shown.

View File

@@ -1,25 +0,0 @@
#!/bin/sh
set -x
if [ "$#" -ne 1 ] || ( [ "$1" != "internet" ] && [ "$1" != "airgap" ] ); then
echo \
"Enable or disable internet access in hauler's CentOS Vagrant machine.
Usage: $0 internet
$0 airgap" >&2
exit 1
fi
if [ "$1" = "internet" ]; then
# internet: set default gateway to NAT network interface
default_iface="eth0"
gw_ip="10.0.2.2"
else
# airgap: set default gateway to private network interface
default_iface="eth1"
gw_ip=$(ip -f inet a show "${default_iface}" | awk 'match($0, /inet ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})/, arr) { print arr[1] }')
fi
ip r delete default
ip r add default via ${gw_ip} dev ${default_iface} proto dhcp metric 100

View File

@@ -1,41 +0,0 @@
#!/bin/sh
################################################################################
# RUN IN VAGRANT MACHINE
# Install a default, bare k3s cluster into the Vagrant machine
################################################################################
if [ -f "/usr/local/bin/k3s-uninstall.sh" ]; then
/usr/local/bin/k3s-uninstall.sh
else
echo "k3s is not installed"
fi
if pgrep -x "firewalld" >/dev/null
then
echo "[FATAL] disable firewalld first"
fi
SELINUXSTATUS=$(getenforce)
if [ "$SELINUXSTATUS" == "Permissive" ]; then
echo "[FATAL] disable selinux"
exit 1
else
echo "SELINUX disabled. continuing"
fi
LOCAL_IMAGES_FILEPATH=/var/lib/rancher/k3s/agent/images
ARTIFACT_DIR=/opt/hauler/local-artifacts/k3s
mkdir -p ${LOCAL_IMAGES_FILEPATH}
cp ${ARTIFACT_DIR}/images/* ${LOCAL_IMAGES_FILEPATH}
cp ${ARTIFACT_DIR}/bin/k3s /usr/local/bin/k3s
chmod +x /usr/local/bin/k3s
yum install -y ${ARTIFACT_DIR}/rpm/*
INSTALL_K3S_SKIP_DOWNLOAD=true ${ARTIFACT_DIR}/bin/k3s-install.sh
chmod +r /etc/rancher/k3s/k3s.yaml

View File

@@ -1,50 +0,0 @@
#!/bin/sh
################################################################################
# RUN IN VAGRANT MACHINE
# Download all required dependencies for an air-gapped k3s install, saving them
# to the folder shared with the host machine.
################################################################################
BASE_SHARED_DIR="/opt/hauler"
VAGRANT_SCRIPTS_DIR="${BASE_SHARED_DIR}/vagrant-scripts"
ARTIFACTS_DIR="${BASE_SHARED_DIR}/local-artifacts/k3s"
K3S_VERSION='v1.18.8+k3s1'
K3S_VERSION_URL='v1.18.8%2Bk3s1'
LOCAL_IMAGES="${ARTIFACTS_DIR}/images"
LOCAL_BIN="${ARTIFACTS_DIR}/bin"
LOCAL_RPM="${ARTIFACTS_DIR}/rpm"
mkdir -p ${LOCAL_IMAGES}
mkdir -p ${LOCAL_BIN}
mkdir -p ${LOCAL_RPM}
# temporarily allow internet access
${VAGRANT_SCRIPTS_DIR}/airgap.sh internet
pushd ${LOCAL_IMAGES}
curl -LO https://github.com/rancher/k3s/releases/download/${K3S_VERSION_URL}/k3s-airgap-images-amd64.tar
popd
pushd ${LOCAL_BIN}
curl -LO https://github.com/rancher/k3s/releases/download/${K3S_VERSION_URL}/k3s
curl -L https://raw.githubusercontent.com/rancher/k3s/${K3S_VERSION_URL}/install.sh -o k3s-install.sh
chmod +x ./*
popd
pushd ${LOCAL_RPM}
curl -LO https://rpm.rancher.io/k3s-selinux-0.1.1-rc1.el7.noarch.rpm
yum install -y yum-utils
yumdownloader --destdir=. --resolve container-selinux selinux-policy-base
popd
# restore air-gap configuration
${VAGRANT_SCRIPTS_DIR}/airgap.sh airgap

View File

@@ -1,12 +0,0 @@
#!/bin/sh
BASE_SHARED_DIR="/opt/hauler"
VAGRANT_SCRIPTS_DIR="${BASE_SHARED_DIR}/vagrant-scripts"
for script in ${VAGRANT_SCRIPTS_DIR}/*-prep.sh ; do
echo "---"
echo "Running ${script} ..."
echo "---"
sh "${script}"
done

View File

@@ -1,73 +0,0 @@
#!/bin/sh
################################################################################
# RUN IN VAGRANT MACHINE
# Install a default, bare rke2 cluster into the Vagrant machine
################################################################################
BASE_SHARED_DIR="/opt/hauler"
VAGRANT_SCRIPTS_DIR="${BASE_SHARED_DIR}/vagrant-scripts"
RKE2_VERSION_DOCKER='v1.18.4-beta16-rke2'
if pgrep -x "firewalld" >/dev/null
then
echo "[FATAL] disable firewalld first"
fi
mkdir -p /etc/rancher/rke2/
# TODO - allow using selinux
SELINUXSTATUS="$(getenforce)"
if [ "$SELINUXSTATUS" = "Permissive" ] || [ "$SELINUXSTATUS" = "Enforcing" ]
then
echo "selinux: true" | sudo tee -a /etc/rancher/rke2/config.yaml > /dev/null
else
echo "SELINUX disabled. continuing"
fi
LOCAL_IMAGES_FILEPATH=/var/lib/rancher/rke2/agent/images
ARTIFACT_DIR="${BASE_SHARED_DIR}/local-artifacts/rke2"
mkdir -p ${LOCAL_IMAGES_FILEPATH}
cp ${ARTIFACT_DIR}/images/* ${LOCAL_IMAGES_FILEPATH}
# TODO - add ability to use local binary with yum install
# ----------------------------------------------------------
# uncomment to use a specific local binary for the install
# ----------------------------------------------------------
# LOCAL_RKE2_BIN='rke2-beta13-dev'
#if [ -n "${LOCAL_RKE2_BIN}" ] && [ -f "${ARTIFACT_DIR}/bin/${LOCAL_RKE2_BIN}" ] ; then
# echo "Use "${ARTIFACT_DIR}/bin/${LOCAL_RKE2_BIN}" for rke2 binary"
#
# INSTALL_RKE2_SKIP_START=true \
# RKE2_RUNTIME_IMAGE="rancher/rke2-runtime:${RKE2_VERSION_DOCKER}" \
# ${ARTIFACT_DIR}/bin/rke2-installer.run
#
# rm -f /usr/local/bin/rke2
#
# cp "${ARTIFACT_DIR}/bin/${LOCAL_RKE2_BIN}" /usr/local/bin/rke2
#
# systemctl start rke2
#else
# ${ARTIFACT_DIR}/bin/rke2-installer.run
#fi
yum install -y ${ARTIFACT_DIR}/rpm/*
systemctl enable rke2-server && systemctl start rke2-server
while [ -f "/etc/rancher/rke2/rke2.yaml" ] ; do
echo "Waiting for /etc/rancher/rke2/rke2.yaml to exist..."
sleep 10
done
chmod +r /etc/rancher/rke2/rke2.yaml
echo "RKE2 cluster is wrapping up installation, run the following commands to allow kubectl access:
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
export PATH=/var/lib/rancher/rke2/bin/:\${PATH}"

View File

@@ -1,72 +0,0 @@
#!/bin/sh
################################################################################
# RUN IN VAGRANT MACHINE
# Download all required dependencies for an air-gapped rke2 install, saving them
# to the folder shared with the host machine.
################################################################################
BASE_SHARED_DIR="/opt/hauler"
VAGRANT_SCRIPTS_DIR="${BASE_SHARED_DIR}/vagrant-scripts"
ARTIFACTS_DIR="${BASE_SHARED_DIR}/local-artifacts/rke2"
RKE2_VERSION='v1.18.13+rke2r1'
RKE2_VERSION_URL='v1.18.13%2Brke2r1'
RKE2_VERSION_DOCKER='v1.18.13-rke2r1'
LOCAL_IMAGES="${ARTIFACTS_DIR}/images"
LOCAL_BIN="${ARTIFACTS_DIR}/bin"
LOCAL_RPM="${ARTIFACTS_DIR}/rpm"
mkdir -p ${LOCAL_IMAGES}
mkdir -p ${LOCAL_BIN}
mkdir -p ${LOCAL_RPM}
# temporarily allow internet access
${VAGRANT_SCRIPTS_DIR}/airgap.sh internet
pushd ${LOCAL_IMAGES}
curl -LO https://github.com/rancher/rke2/releases/download/${RKE2_VERSION_URL}/rke2-images.linux-amd64.tar.gz
gunzip rke2-images.linux-amd64.tar.gz
popd
#pushd ${LOCAL_BIN}
#
#curl -L https://github.com/rancher/rke2/releases/download/${RKE2_VERSION_URL}/rke2-installer.linux-amd64.run -o rke2-installer.run
#chmod +x ./*
#
#popd
pushd ${LOCAL_RPM}
yum install -y yum-plugin-downloadonly
rke2_rpm_channel='stable'
rpm_site='rpm.rancher.io'
maj_ver='7'
rke2_majmin=$(echo "${RKE2_VERSION}" | sed -E -e "s/^v([0-9]+\.[0-9]+).*/\1/")
cat <<-EOF >"/etc/yum.repos.d/rancher-rke2.repo"
[rancher-rke2-common-${rke2_rpm_channel}]
name=Rancher RKE2 Common (${RKE2_VERSION})
baseurl=https://${rpm_site}/rke2/${rke2_rpm_channel}/common/centos/${maj_ver}/noarch
enabled=1
gpgcheck=1
gpgkey=https://${rpm_site}/public.key
[rancher-rke2-${rke2_majmin}-${rke2_rpm_channel}]
name=Rancher RKE2 ${rke2_majmin} (${RKE2_VERSION})
baseurl=https://${rpm_site}/rke2/${rke2_rpm_channel}/${rke2_majmin}/centos/${maj_ver}/x86_64
enabled=1
gpgcheck=1
gpgkey=https://${rpm_site}/public.key
EOF
yum install --downloadonly --downloaddir=./ rke2-server
popd
# restore air-gap configuration
${VAGRANT_SCRIPTS_DIR}/airgap.sh airgap