Compare commits

...

96 Commits

Author SHA1 Message Date
Adam Martin
ae80b482e4 Merge pull request #168 from amartin120/dep-updates
dependency bumps for security vuln fixes
2024-01-30 16:51:48 -05:00
Adam Martin
1ae496fb8b dep bumps for security vuln fixes
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2024-01-30 14:41:33 -05:00
Adam Martin
7919dccffc Merge pull request #167 from amartin120/prerelease-flag
release process checks tag to determine pre-release
2024-01-30 11:11:13 -05:00
Adam Martin
fc7a19c755 check tag to determine pre-release
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2024-01-30 10:57:40 -05:00
Adam Martin
ade0feccf0 Merge pull request #166 from clemenko/main
Update install.sh for file cleaning
2024-01-30 09:04:41 -05:00
Andy Clemenko
f78fdf5e3d Update install.sh
adding the old hauler binary to the cleanup
2024-01-30 08:55:57 -05:00
Andy Clemenko
85d6bc0233 Update install.sh for file cleaning
removing LICENSE and README.md files.
2024-01-30 08:41:07 -05:00
Adam Martin
d1499b7738 Merge pull request #164 from amartin120/cosign-updates
Add `--platform` flag to image processes and RGS flavored cosign setup improvement.
2024-01-29 14:46:18 -05:00
Adam Martin
27acb239e4 clean up makefile 2024-01-29 13:41:53 -05:00
Adam Martin
e8d084847d remove extra debug statement
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2024-01-28 21:15:27 -05:00
Adam Martin
e70379870f another fix for the unit test gh action
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2024-01-28 19:51:22 -05:00
Adam Martin
a05d21c052 add platform flag for image add and sync
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2024-01-28 19:48:16 -05:00
Adam Martin
8256aa55ce adjust unit test gh action for latest updates 2024-01-28 19:46:55 -05:00
Adam Martin
0e6c3690b1 bump cosign version to v2.2.2+carbide.2 2024-01-28 19:45:05 -05:00
Adam Martin
a977cec50c improve cosign setup
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2024-01-28 12:08:31 -05:00
Adam Martin
5edc96d152 Merge pull request #162 from zackbradys/main
updated archive default name
2024-01-24 09:19:48 -05:00
Zack Hodgson Brady
fbafa60da5 updated archive default name 2024-01-23 22:49:20 -05:00
Adam Martin
cc917af0f2 Merge pull request #159 from amartin120/store-fileserver
Store fileserver
2024-01-22 15:12:45 -05:00
Adam Martin
f76160d8be Merge pull request #160 from amartin120/add-license
add license file
2024-01-22 15:12:05 -05:00
Adam Martin
b24b25d557 add license file 2024-01-22 15:06:09 -05:00
Adam Martin
d9e298b725 adjust to make registry and fileserver subcommands 2024-01-22 13:40:58 -05:00
Adam Martin
e14453f730 add fileserver option for store serve 2024-01-22 11:31:46 -05:00
Zack Brady
990ade9cd0 merge pull request #152 from zackbradys/main
updated readme and hauler `install.sh`
2023-12-20 19:56:57 -05:00
Zack Hodgson Brady
aecd37d192 added homebrew install instructions 2023-12-20 19:46:55 -05:00
Zack Brady
02f4946ead Merge branch 'rancherfederal:main' into main 2023-12-20 00:31:44 -05:00
Zack Hodgson Brady
978dc659f8 updated hauler version and automated default version 2023-12-19 21:24:04 -05:00
Adam Martin
f982f51d57 Merge pull request #150 from amartin120/info-type-filter
add simple type filter to store info
2023-12-19 13:07:46 -05:00
Adam Martin
2174e96f0e add simple type filter to store info
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-19 09:59:06 -05:00
Adam Martin
8cfe4432fc Merge pull request #149 from amartin120/registry-serve-fix
fix for validating foreign blobs
2023-12-18 15:51:38 -05:00
Adam Martin
f129484224 Merge pull request #148 from amartin120/fix-chart-tags
fix for charts with a + in the version
2023-12-18 15:51:20 -05:00
Adam Martin
4dbff83459 fix for validating foreign blobs
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-18 15:27:32 -05:00
Adam Martin
e229c2a1da fix for chart tags with a +
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-15 16:17:34 -05:00
Zack Brady
2a93e74b62 merge pull request #147 from zackbradys/main
updated/fixed install.sh
2023-12-14 23:36:53 -05:00
Zack Hodgson Brady
4d5d9eda7b updated readme for hauler install 2023-12-14 23:05:01 -05:00
Zack Hodgson Brady
a7cbfcb042 updated/fixed hauler install.sh 2023-12-14 23:04:36 -05:00
Adam Martin
7751b12e5e Merge pull request #146 from amartin120/more-updates-0.4.1
Improved logging for store copy / Updated store info to handle multi-arch images
2023-12-14 15:05:24 -05:00
Adam Martin
6e3d3fc7b8 updated store info to handle multi arch images
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-14 11:15:37 -05:00
Adam Martin
0f7f363d6c improved logging for hauler store copy
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-11 18:15:34 -05:00
Adam Martin
ab975a1dc7 Merge pull request #144 from amartin120/add-autocompletion
add autocompletion
2023-12-05 12:19:01 -05:00
Adam Martin
2d92d41245 Merge pull request #142 from amartin120/performance-fix
performance fix / version display improvement
2023-12-05 12:18:34 -05:00
Adam Martin
e2176d211a keep consistent with other subcommands
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-05 11:29:01 -05:00
Adam Martin
93ae968580 add autocompletion
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-05 10:37:29 -05:00
Adam Martin
b0a37d21af performance fix for images
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-04 11:19:57 -05:00
Adam Martin
aa16575c6f cleaned up version command more
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-12-04 11:19:43 -05:00
Adam Martin
2959cfc346 Merge pull request #141 from amartin120/goreleaser-versioning-fix
fix hauler version display
2023-11-30 14:01:14 -05:00
Adam Martin
c04211a55e Merge pull request #140 from amartin120/retry-logic
Retry logic / Auth Flag Fix / Sync Cleanup
2023-11-30 14:00:31 -05:00
Adam Martin
c497f53972 fix hauler version display
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-11-30 13:39:23 -05:00
Adam Martin
f1fbd7e9c2 don't flush store on each sync
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-11-30 10:02:04 -05:00
Adam Martin
f348fb8d4d registry auth fix for copy
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-11-28 22:29:00 -05:00
Adam Martin
fe60b1fd1a add retry logic
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-11-28 10:02:21 -05:00
Zack Brady
756c0171c3 merge pull request #139 from zackbradys/main
added new installation method (`install.sh`)
2023-11-16 14:01:06 -05:00
Zack Hodgson Brady
c394965f88 more improvements to script 2023-11-12 17:18:41 -05:00
Zack Hodgson Brady
43e2dc56ec upgraded install script functionality 2023-11-12 03:50:32 -05:00
Zack Hodgson Brady
795a88218f updated readme for new install script 2023-11-12 02:48:28 -05:00
Zack Hodgson Brady
ec2ada9dcb cleaned up install script variables 2023-11-12 00:26:28 -05:00
Zack Hodgson Brady
45cea89752 added initial install script 2023-11-12 00:06:49 -05:00
Adam Martin
6062c20e02 Merge pull request #138 from rancherfederal/fix-github-path
fix carbide cosign repo path and perms
2023-11-06 09:08:41 -05:00
Adam Martin
be486df762 fix carbide cosign repo path and perms
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-11-06 09:07:13 -05:00
Adam Martin
4d950f7b0a Add OCI hauler manifests. (#136)
* pull carbide flavored hauler manifests from reg
* remove temp constant
* remove temp hardcoding
* add comments for new sync flags
* fixes for version and registry serve
* band-aid for store info... needs love
* add sbom to info logic
* adjust a few text descriptions
* adjust tag names with +
* removed testing file

Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-11-03 12:44:05 -07:00
Adam Martin
f8c16a1a24 Merge pull request #135 from rancherfederal/cosign-verify
Add cosign verify functionality.
2023-11-03 15:27:48 -04:00
Adam Martin
6e8c7db81f Merge branch 'main' of github.com:rancherfederal/hauler into cosign-verify 2023-11-03 13:56:21 -04:00
Adam Martin
4772657548 Add cosign for handling image functionality. (#134)
* pull back in ocil
* updates to OCIL funcs to handle cosign changes
* add cosign logic
* adjust Makefile to be a little more generic
* cli updates to accomodate the cosign additions
* add cosign drop-in funcs
* impl for cosign functions for images & store copy
* fixes and logging for cosign verify <iamge>
* fix cosign verify logging
* update go.mod

Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-11-03 10:43:32 -07:00
Zack Brady
337494cefd merge pull request #132 from rancherfederal/zackbradys-readme-updates
readme and docs updates
2023-10-26 00:43:53 -04:00
Zack Brady
865afb4a2d updated readme for extra info 2023-10-26 00:42:58 -04:00
Zack Brady
d8b0193a92 merge pull request #133 from rancherfederal/zackbradys-github-updates
updated github templates
2023-10-25 18:01:34 -04:00
Zack Brady
b616f54085 updated readme for deprecated commands
Co-authored-by: Jacob Blain Christen <dweomer5@gmail.com>
2023-10-25 17:03:35 -04:00
Zack Brady
870f2ebda8 last typo fixes 2023-10-21 02:37:42 -04:00
Zack Brady
b7a8fc0a60 fixed typos 2023-10-20 12:32:31 -04:00
Zack Brady
04c97b8a97 fixed typos 2023-10-20 12:22:10 -04:00
Zack Brady
d46ccd03a5 updated github templates 2023-10-20 04:59:51 -04:00
Zack Brady
99288f9b9d removed old docs 2023-10-20 03:56:01 -04:00
Zack Brady
2cc5e902ad updated readme 2023-10-20 03:49:43 -04:00
Adam Martin
f2b0c44af3 polish up cosign verify for hauler store sync
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-10-12 12:05:35 -04:00
Adam Martin
356c46fe28 update go.mod
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-10-12 10:34:40 -04:00
Adam Martin
323b93ae20 fix cosign verify logging
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-10-11 13:44:21 -04:00
Adam Martin
bb9a088a84 fixes and logging for cosign verify <iamge>
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-10-11 13:44:21 -04:00
Adam Martin
96d92e3248 impl for cosign functions for images & store copy
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-10-11 13:44:21 -04:00
Adam Martin
220eeedb2c add cosign drop-in funcs
Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
2023-10-11 13:44:21 -04:00
Adam Martin
3049846a46 cli updates to accomodate the cosign additions 2023-10-11 13:44:21 -04:00
Adam Martin
ece463bc1c adjust Makefile to be a little more generic 2023-10-11 13:44:21 -04:00
Adam Martin
58c55d7aeb add cosign logic 2023-10-11 13:44:21 -04:00
Adam Martin
214ed48829 updates to OCIL funcs to handle cosign changes 2023-10-11 13:43:19 -04:00
Adam Martin
7d6bbbc6fc pull back in ocil 2023-10-11 13:40:42 -04:00
Jacob Blain Christen
995477db22 Merge pull request #131 from rancherfederal/dep-updates
dependency updates
2023-10-11 09:36:25 -07:00
Adam Martin
9862e61f23 update github action deps as well 2023-10-06 15:06:27 -04:00
Adam Martin
fe7122da8a update dependencies 2023-10-06 14:53:17 -04:00
Jacob Blain Christen
2999b90e30 Merge pull request #130 from rancherfederal/deprecate-non-store-stuff
deprecation notices for `dl` and the non-store version of `serve`
2023-09-28 11:51:33 -07:00
Adam Martin
4beb4d4200 deprecation notices for dl and non-store serve 2023-09-27 09:07:33 -04:00
Brandon
4ed1b0a1a4 Update walkthrough.md 2022-08-27 10:40:15 -04:00
Brandon
925ce53aeb Merge pull request #127 from neoakris/content_doc_example
Adding example of imperative generation of declarative config file to doc
2022-04-25 15:52:36 -04:00
Chris McGrath
3888e23907 reworded code comment to be more accurate 2022-04-25 15:26:06 -04:00
Chris McGrath
88f482f4af fixed syntax issue 2022-04-25 15:22:27 -04:00
Chris McGrath
425c92e8a6 added missing 'cat contents.yaml' to example 2022-04-25 15:08:08 -04:00
Chris McGrath
011a4d8725 adding imperative generation of declarative config example to doc 2022-04-25 15:03:39 -04:00
Brandon
c60ccc8085 Merge pull request #116 from noslzzp/main
Update README.md
2022-02-03 18:48:54 -05:00
NoSLZZP
6ebcd5088d Update README.md 2022-02-03 17:23:41 -05:00
67 changed files with 4250 additions and 1794 deletions

View File

@@ -1,31 +1,33 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
name: Bug Report
about: Create a report to help us improve!
title: '[BUG]'
labels: 'kind/bug'
assignees: ''
---
<!-- Thanks for helping us to improve Hauler! We welcome all bug reports. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
<!-- Thank you for helping us to improve Hauler! We welcome all bug reports. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
**Environmental Info:**
**Environmental Info:**
*
**Hauler Version:**
*
**System CPU architecture, OS, and Version:**
<!-- Provide the output from "uname -a" on the system where Hauler is installed -->
* <!-- Provide the output from "uname -a" on the system where Hauler is installed -->
**Describe the bug:**
<!-- A clear and concise description of what the bug is. -->
* <!-- A clear and concise description of the bug. -->
**Steps To Reproduce:**
* <!-- A clear and concise way to reproduce the bug. -->
**Expected behavior:**
<!-- A clear and concise description of what you expected to happen. -->
* <!-- A clear and concise description of what you expected to happen, without the bug. -->
**Actual behavior:**
<!-- A clear and concise description of what actually happened. -->
* <!-- A clear and concise description of what actually happened. -->
**Additional context / logs:**
<!-- Add any other context and/or logs about the problem here. -->
* <!-- Add any other context and/or logs about the problem here. -->

View File

@@ -0,0 +1,21 @@
---
name: Feature Request
about: Create a report to help us improve!
title: '[RFE]'
labels: 'kind/rfe'
assignees: ''
---
<!-- Thanks for helping us to improve Hauler! We welcome all requests for enhancements (RFEs). Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
**Is your feature request related to a problem? Please describe.**
* <!-- A clear and concise description of the problem. -->
**Describe the solution you'd like**
* <!-- A clear and concise description of what you want to happen. -->
**Describe alternatives you've considered**
* <!-- A clear and concise description of any alternative solutions or features you've considered. -->
**Additional context**
* <!-- Add any other context or screenshots about the feature request here. -->

View File

@@ -1,23 +1,20 @@
* **Please check if the PR fulfills these requirements**
- [ ] The commit message follows our guidelines
- [ ] Tests for the changes have been added (for bug fixes / features)
- [ ] Docs have been added / updated (for bug fixes / features)
**Please check below, if the PR fulfills these requirements:**
- [ ] The commit message follows the guidelines.
- [ ] Tests for the changes have been added (for bug fixes / features).
- [ ] Docs have been added / updated (for bug fixes / features).
* **What kind of change does this PR introduce?** (Bug fix, feature, docs update, ...)
**What kind of change does this PR introduce?**
* <!-- Bug fix, feature, docs update, ... -->
**What is the current behavior?**
* <!-- You can also link to an open issue here -->
**What is the new behavior (if this is a feature change)?**
* <!-- What changes did this PR introduce? -->
* **What is the current behavior?** (You can also link to an open issue here)
**Does this PR introduce a breaking change?**
* <!-- What changes might users need to make in their application due to this PR? -->
* **What is the new behavior (if this is a feature change)?**
* **Does this PR introduce a breaking change?** (What changes might users need to make in their application due to this PR?)
* **Other information**:
**Other information**:
* <!-- Any additional information -->

View File

@@ -18,13 +18,13 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17.x
go-version: 1.21.x
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v2
with:
distribution: goreleaser
version: latest
args: release --rm-dist
args: release --rm-dist -p 1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}

View File

@@ -24,9 +24,11 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.17.x
go-version: 1.21.x
- name: Run Unit Tests
run: |
mkdir -p cmd/hauler/binaries
touch cmd/hauler/binaries/dummy.txt
go test -race -covermode=atomic -coverprofile=coverage.out ./pkg/... ./internal/... ./cmd/...
- name: On Failure, Launch Debug Session
if: ${{ failure() }}

3
.gitignore vendored
View File

@@ -27,4 +27,5 @@ dist/
tmp/
bin/
/store/
/registry/
/registry/
cmd/hauler/binaries

View File

@@ -3,9 +3,14 @@ before:
hooks:
- go mod tidy
- go mod download
- rm -rf cmd/hauler/binaries
release:
prerelease: auto
env:
- vpkg=github.com/rancherfederal/hauler/pkg/version
- vpkg=github.com/rancherfederal/hauler/internal/version
- cosign_version=v2.2.2+carbide.2
builds:
- main: cmd/hauler/main.go
@@ -17,7 +22,13 @@ builds:
- amd64
- arm64
ldflags:
- -s -w -X {{ .Env.vpkg }}.GitVersion={{ .Version }} -X {{ .Env.vpkg }}.commit={{ .ShortCommit }} -X {{ .Env.vpkg }}.buildDate={{ .Date }}
- -s -w -X {{ .Env.vpkg }}.gitVersion={{ .Version }} -X {{ .Env.vpkg }}.gitCommit={{ .ShortCommit }} -X {{ .Env.vpkg }}.gitTreeState={{if .IsGitDirty}}dirty{{else}}clean{{end}} -X {{ .Env.vpkg }}.buildDate={{ .Date }}
hooks:
pre:
- mkdir -p cmd/hauler/binaries
- wget -P cmd/hauler/binaries/ https://github.com/rancher-government-carbide/cosign/releases/download/{{ .Env.cosign_version }}/cosign-{{ .Os }}-{{ .Arch }}{{ if eq .Os "windows" }}.exe{{ end }}
post:
- rm -rf cmd/hauler/binaries
env:
- CGO_ENABLED=0

177
LICENSE Normal file
View File

@@ -0,0 +1,177 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS

View File

@@ -1,23 +1,27 @@
SHELL:=/bin/bash
GO_BUILD_ENV=GOOS=linux GOARCH=amd64
GO_FILES=$(shell go list ./... | grep -v /vendor/)
BUILD_VERSION=$(shell cat VERSION)
BUILD_TAG=$(BUILD_VERSION)
COSIGN_VERSION=v2.2.2+carbide.2
.SILENT:
all: fmt vet install test
build:
rm -rf cmd/hauler/binaries;\
mkdir -p cmd/hauler/binaries;\
wget -P cmd/hauler/binaries/ https://github.com/rancher-government-carbide/cosign/releases/download/$(COSIGN_VERSION)/cosign-$(shell go env GOOS)-$(shell go env GOARCH);\
mkdir bin;\
$(GO_BUILD_ENV) go build -o bin ./cmd/...;\
CGO_ENABLED=0 go build -o bin ./cmd/...;\
build-all: fmt vet
goreleaser build --rm-dist --snapshot
install:
$(GO_BUILD_ENV) go install
rm -rf cmd/hauler/binaries;\
mkdir -p cmd/hauler/binaries;\
wget -P cmd/hauler/binaries/ https://github.com/rancher-government-carbide/cosign/releases/download/$(COSIGN_VERSION)/cosign-$(shell go env GOOS)-$(shell go env GOARCH);\
CGO_ENABLED=0 go install ./cmd/...;\
vet:
go vet $(GO_FILES)

View File

@@ -1,28 +1,43 @@
# Hauler: Airgap Swiss Army Knife
# Rancher Government Hauler
> ⚠️ This project is still in active development and _not_ GA. While a lot of the core features are ready, we're still adding a _ton_, and we may make breaking api and feature changes version to version.
## Airgap Swiss Army Knife
`hauler` simplifies the airgap experience without forcing you to adopt a specific workflow for your infrastructure or application.
> ⚠️ This project is still in active development and *not* Generally Available (GA). Most of the core functionality and features are ready, but may have breaking changes. Please review the [Release Notes](https://github.com/rancherfederal/hauler/releases) for more information!
To accomplish this, it focuses strictly on two of the biggest airgap pain points:
`Rancher Government Hauler` simplifies the airgap experience without requiring users to adopt a specific workflow. **Hauler** simplifies the airgapping process, by representing assets (images, charts, files, etc...) as content and collections to allow users to easily fetch, store, package, and distribute these assets with declarative manifests or through the command line.
* content collection
* content distribution
`Hauler` does this by storing contents and collections as OCI Artifacts and allows users to serve contents and collections with an embedded registry and fileserver. Additionally, `Hauler` has the ability to store and inspect various non-image OCI Artifacts.
As OCI registries have become ubiquitous nowadays for storing and distributing containers. Their success and widespread adoption has led many projects to expand beyond containers.
For more information, please review the **[Hauler Documentation](https://rancherfederal.github.io/hauler-docs)!**
`hauler` capitalizes on this, and leverages the [`oci`](https://github.com/opencontainers) spec to be a simple, zero dependency tool to collect, transport, and distribute your artifacts.
## Installation
## Getting started
### Linux/Darwin
```bash
# installs latest release
curl -sfL https://get.hauler.dev | bash
```
See the [quickstart](docs/walkthrough.md#Quickstart) for a quick way to get started with some of `haulers` capabilities.
### Homebrew
```bash
# installs latest release
brew tap rancherfederal/homebrew-tap
brew install hauler
```
For a guided example of all of `haulers` capabilities, check out the [guided example](docs/walkthrough.md#guided-examples).
### Windows
```bash
# coming soon
```
## Acknowledgements
`hauler` wouldn't be possible without the open source community, but there are a few dependent projects that stand out:
`Hauler` wouldn't be possible without the open-source community, but there are a few projects that stand out:
* [go-containerregistry](https://github.com/google/go-containerregistry)
* [oras](https://github.com/oras-project/oras)
* [cosign](https://github.com/sigstore/cosign)
* [oras cli](https://github.com/oras-project/oras)
* [cosign](https://github.com/sigstore/cosign)
## Notices
**WARNING - Upcoming Deprecated Command(s):**
`hauler download` (alternatively, `dl`) and `hauler serve` (_not_ `hauler store serve`) commands are deprecated and will be removed in a future release.

View File

@@ -15,7 +15,7 @@ var ro = &rootOpts{}
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "hauler",
Short: "",
Short: "Airgap Swiss Army Knife",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
l := log.FromContext(cmd.Context())
l.SetLevel(ro.logLevel)
@@ -35,6 +35,7 @@ func New() *cobra.Command {
addStore(cmd)
addServe(cmd)
addVersion(cmd)
addCompletion(cmd)
return cmd
}

View File

@@ -0,0 +1,123 @@
package cli
import (
"fmt"
"os"
"github.com/spf13/cobra"
)
func addCompletion(parent *cobra.Command) {
cmd := &cobra.Command{
Use: "completion",
Short: "Generates completion scripts for various shells",
Long: `The completion sub-command generates completion scripts for various shells.`,
}
cmd.AddCommand(
addCompletionZsh(),
addCompletionBash(),
addCompletionFish(),
addCompletionPowershell(),
)
parent.AddCommand(cmd)
}
func completionError(err error) ([]string, cobra.ShellCompDirective) {
cobra.CompError(err.Error())
return nil, cobra.ShellCompDirectiveError
}
func addCompletionZsh() *cobra.Command {
cmd := &cobra.Command{
Use: "zsh",
Short: "Generates zsh completion scripts",
Long: `The completion sub-command generates completion scripts for zsh.`,
Example: `To load completion run
. <(hauler completion zsh)
To configure your zsh shell to load completions for each session add to your zshrc
# ~/.zshrc or ~/.profile
command -v hauler >/dev/null && . <(hauler completion zsh)
or write a cached file in one of the completion directories in your ${fpath}:
echo "${fpath// /\n}" | grep -i completion
hauler completion zsh > _hauler
mv _hauler ~/.oh-my-zsh/completions # oh-my-zsh
mv _hauler ~/.zprezto/modules/completion/external/src/ # zprezto`,
Run: func(cmd *cobra.Command, args []string) {
cmd.GenZshCompletion(os.Stdout)
// Cobra doesn't source zsh completion file, explicitly doing it here
fmt.Println("compdef _hauler hauler")
},
}
return cmd
}
func addCompletionBash() *cobra.Command {
cmd := &cobra.Command{
Use: "bash",
Short: "Generates bash completion scripts",
Long: `The completion sub-command generates completion scripts for bash.`,
Example: `To load completion run
. <(hauler completion bash)
To configure your bash shell to load completions for each session add to your bashrc
# ~/.bashrc or ~/.profile
command -v hauler >/dev/null && . <(hauler completion bash)`,
Run: func(cmd *cobra.Command, args []string) {
cmd.GenBashCompletion(os.Stdout)
},
}
return cmd
}
func addCompletionFish() *cobra.Command {
cmd := &cobra.Command{
Use: "fish",
Short: "Generates fish completion scripts",
Long: `The completion sub-command generates completion scripts for fish.`,
Example: `To configure your fish shell to load completions for each session write this script to your completions dir:
hauler completion fish > ~/.config/fish/completions/hauler.fish
See http://fishshell.com/docs/current/index.html#completion-own for more details`,
Run: func(cmd *cobra.Command, args []string) {
cmd.GenFishCompletion(os.Stdout, true)
},
}
return cmd
}
func addCompletionPowershell() *cobra.Command {
cmd := &cobra.Command{
Use: "powershell",
Short: "Generates powershell completion scripts",
Long: `The completion sub-command generates completion scripts for powershell.`,
Example: `To load completion run
. <(hauler completion powershell)
To configure your powershell shell to load completions for each session add to your powershell profile
Windows:
cd "$env:USERPROFILE\Documents\WindowsPowerShell\Modules"
hauler completion powershell >> hauler-completion.ps1
Linux:
cd "${XDG_CONFIG_HOME:-"$HOME/.config/"}/powershell/modules"
hauler completion powershell >> hauler-completions.ps1`,
Run: func(cmd *cobra.Command, args []string) {
cmd.GenPowerShellCompletion(os.Stdout)
},
}
return cmd
}

View File

@@ -12,7 +12,10 @@ func addDownload(parent *cobra.Command) {
cmd := &cobra.Command{
Use: "download",
Short: "Download OCI content from a registry and populate it on disk",
Long: `Locate OCI content based on it's reference in a compatible registry and download the contents to disk.
Long: `*** WARNING: Deprecated Command ***
The 'download (dl)' command is deprecated and will be removed in a future release of Hauler.
Locate OCI content based on it's reference in a compatible registry and download the contents to disk.
Note that the content type determines it's format on disk. Hauler's built in content types act as follows:

View File

@@ -11,7 +11,7 @@ import (
"oras.land/oras-go/pkg/content"
"oras.land/oras-go/pkg/oras"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/log"

View File

@@ -10,6 +10,8 @@ func addServe(parent *cobra.Command) {
cmd := &cobra.Command{
Use: "serve",
Short: "Run one or more of hauler's embedded servers types",
Long: `*** WARNING: Deprecated Command ***
The 'serve' command is deprecated and will be removed in a future release of Hauler.`,
RunE: func(cmd *cobra.Command, args []string) error {
return cmd.Help()
},

View File

@@ -74,6 +74,7 @@ func (o *RegistryOpts) defaultConfig() *configuration.Configuration {
cfg.HTTP.Addr = fmt.Sprintf(":%d", o.Port)
cfg.HTTP.Headers = http.Header{
"X-Content-Type-Options": []string{"nosniff"},
"Accept": []string{"application/vnd.dsse.envelope.v1+json, application/json"},
}
return cfg

View File

@@ -3,6 +3,7 @@ package cli
import (
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"fmt"
"github.com/rancherfederal/hauler/cmd/hauler/cli/store"
)
@@ -107,12 +108,28 @@ func addStoreLoad() *cobra.Command {
}
func addStoreServe() *cobra.Command {
o := &store.ServeOpts{RootOpts: rootStoreOpts}
cmd := &cobra.Command{
Use: "serve",
Short: "Expose the content of a local store through an OCI compliant server",
Short: "Expose the content of a local store through an OCI compliant registry or file server",
RunE: func(cmd *cobra.Command, args []string) error {
return cmd.Help()
},
}
cmd.AddCommand(
addStoreServeRegistry(),
addStoreServeFiles(),
)
return cmd
}
// RegistryCmd serves the embedded registry
func addStoreServeRegistry() *cobra.Command {
o := &store.ServeRegistryOpts{RootOpts: rootStoreOpts}
cmd := &cobra.Command{
Use: "registry",
Short: "Serve the embedded registry",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
@@ -120,12 +137,36 @@ func addStoreServe() *cobra.Command {
return err
}
return store.ServeCmd(ctx, o, s)
},
}
o.AddFlags(cmd)
return store.ServeRegistryCmd(ctx, o, s)
},
}
return cmd
o.AddFlags(cmd)
return cmd
}
// FileServerCmd serves the file server
func addStoreServeFiles() *cobra.Command {
o := &store.ServeFilesOpts{RootOpts: rootStoreOpts}
cmd := &cobra.Command{
Use: "fileserver",
Short: "Serve the file server",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, err := o.Store(ctx)
if err != nil {
return err
}
return store.ServeFilesCmd(ctx, o, s)
},
}
o.AddFlags(cmd)
return cmd
}
func addStoreSave() *cobra.Command {
@@ -155,6 +196,8 @@ func addStoreSave() *cobra.Command {
func addStoreInfo() *cobra.Command {
o := &store.InfoOpts{RootOpts: rootStoreOpts}
var allowedValues = []string{"image", "chart", "file", "all"}
cmd := &cobra.Command{
Use: "info",
Short: "Print out information about the store",
@@ -167,8 +210,13 @@ func addStoreInfo() *cobra.Command {
if err != nil {
return err
}
return store.InfoCmd(ctx, o, s)
for _, allowed := range allowedValues {
if o.TypeFilter == allowed {
return store.InfoCmd(ctx, o, s)
}
}
return fmt.Errorf("type must be one of %v", allowedValues)
},
}
o.AddFlags(cmd)

View File

@@ -4,17 +4,17 @@ import (
"context"
"github.com/google/go-containerregistry/pkg/name"
"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/ocil/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
"github.com/rancherfederal/hauler/pkg/content/chart"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
)
@@ -61,41 +61,50 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
type AddImageOpts struct {
*RootOpts
Name string
Name string
Key string
Platform string
}
func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
_ = f
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification")
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specific platform to save. i.e. linux/amd64. Defaults to all if flag is omitted.")
}
func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error {
l := log.FromContext(ctx)
cfg := v1alpha1.Image{
Name: reference,
}
return storeImage(ctx, s, cfg)
// Check if the user provided a key.
if o.Key != "" {
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name)
if err != nil {
return err
}
l.Infof("signature verified for image [%s]", cfg.Name)
}
return storeImage(ctx, s, cfg, o.Platform)
}
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error {
func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform string) error {
l := log.FromContext(ctx)
img, err := image.NewImage(i.Name)
if err != nil {
return err
}
r, err := name.ParseReference(i.Name)
if err != nil {
return err
}
desc, err := s.AddOCI(ctx, img, r.Name())
err = cosign.SaveImage(ctx, s, r.Name(), platform)
if err != nil {
return err
}
l.Infof("added 'image' to store at [%s], with digest [%s]", r.Name(), desc.Digest.String())
l.Infof("added 'image' to store at [%s]", r.Name())
return nil
}

View File

@@ -5,14 +5,13 @@ import (
"fmt"
"strings"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/log"
"github.com/rancherfederal/hauler/pkg/reference"
)
type CopyOpts struct {
@@ -36,7 +35,6 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) {
func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error {
l := log.FromContext(ctx)
var descs []ocispec.Descriptor
components := strings.SplitN(targetRef, "://", 2)
switch components[0] {
case "dir":
@@ -44,11 +42,10 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string
fs := content.NewFile(components[1])
defer fs.Close()
ds, err := s.CopyAll(ctx, fs, nil)
_, err := s.CopyAll(ctx, fs, nil)
if err != nil {
return err
}
descs = ds
case "registry":
l.Debugf("identified registry target reference")
@@ -58,29 +55,23 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string
Insecure: o.Insecure,
PlainHTTP: o.PlainHTTP,
}
r, err := content.NewRegistry(ropts)
if err != nil {
return err
}
mapperFn := func(ref string) (string, error) {
r, err := reference.Relocate(ref, components[1])
if ropts.Username != "" {
err := cosign.RegistryLogin(ctx, s, components[1], ropts)
if err != nil {
return "", err
return err
}
return r.Name(), nil
}
ds, err := s.CopyAll(ctx, r, mapperFn)
err := cosign.LoadImages(ctx, s, components[1], ropts)
if err != nil {
return err
}
descs = ds
default:
return fmt.Errorf("detecting protocol from [%s]", targetRef)
}
l.Infof("Copied [%d] artifacts to [%s]", len(descs), components[1])
l.Infof("copied artifacts to [%s]", components[1])
return nil
}

View File

@@ -2,13 +2,14 @@ package store
import (
"context"
"strings"
"encoding/json"
"fmt"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/internal/mapper"
"github.com/rancherfederal/hauler/pkg/log"
@@ -36,7 +37,8 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string
found := false
if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
if reference != r.Name() {
if !strings.Contains(reference, r.Name()) {
return nil
}
found = true
@@ -57,7 +59,7 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string
return err
}
pushedDesc, err := s.Copy(ctx, r.Name(), mapperStore, "")
pushedDesc, err := s.Copy(ctx, reference, mapperStore, "")
if err != nil {
return err
}

View File

@@ -6,8 +6,8 @@ import (
"os"
"path/filepath"
"github.com/rancherfederal/ocil/pkg/layer"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/layer"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/log"

View File

@@ -4,15 +4,16 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"text/tabwriter"
"github.com/olekukonko/tablewriter"
"os"
"sort"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/reference"
)
@@ -21,6 +22,7 @@ type InfoOpts struct {
*RootOpts
OutputFormat string
TypeFilter string
SizeUnit string
}
@@ -28,6 +30,7 @@ func (o *InfoOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringVarP(&o.OutputFormat, "output", "o", "table", "Output format (table, json)")
f.StringVarP(&o.TypeFilter, "type", "t", "all", "Filter on type (image, chart, file)")
// TODO: Regex/globbing
}
@@ -38,52 +41,123 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
return nil
}
rc, err := s.Fetch(ctx, desc)
if err != nil {
return err
}
defer rc.Close()
var m ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
}
// handle multi-arch images
if desc.MediaType == consts.OCIImageIndexSchema || desc.MediaType == consts.DockerManifestListSchema2 {
var idx ocispec.Index
if err := json.NewDecoder(rc).Decode(&idx); err != nil {
return err
}
i := newItem(s, desc, m)
items = append(items, i)
for _, internalDesc := range idx.Manifests {
rc, err := s.Fetch(ctx, internalDesc)
if err != nil {
return err
}
defer rc.Close()
var internalManifest ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&internalManifest); err != nil {
return err
}
i := newItem(s, desc, internalManifest, fmt.Sprintf("%s/%s", internalDesc.Platform.OS, internalDesc.Platform.Architecture), o)
var emptyItem item
if i != emptyItem {
items = append(items, i)
}
}
// handle "non" multi-arch images
} else if desc.MediaType == consts.DockerManifestSchema2 || desc.MediaType == consts.OCIManifestSchema1 {
var m ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
}
rc, err := s.FetchManifest(ctx, m)
if err != nil {
return err
}
defer rc.Close()
// Unmarshal the OCI image content
var internalManifest ocispec.Image
if err := json.NewDecoder(rc).Decode(&internalManifest); err != nil {
return err
}
if internalManifest.Architecture != "" {
i := newItem(s, desc, m, fmt.Sprintf("%s/%s", internalManifest.OS, internalManifest.Architecture), o)
var emptyItem item
if i != emptyItem {
items = append(items, i)
}
} else {
i := newItem(s, desc, m, "-", o)
var emptyItem item
if i != emptyItem {
items = append(items, i)
}
}
// handle the rest
} else {
var m ocispec.Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
}
i := newItem(s, desc, m, "-", o)
var emptyItem item
if i != emptyItem {
items = append(items, i)
}
}
return nil
}); err != nil {
return err
}
// sort items by ref and arch
sort.Sort(byReferenceAndArch(items))
var msg string
switch o.OutputFormat {
case "json":
msg = buildJson(items...)
fmt.Println(msg)
default:
msg = buildTable(items...)
buildTable(items...)
}
fmt.Println(msg)
return nil
}
func buildTable(items ...item) string {
b := strings.Builder{}
tw := tabwriter.NewWriter(&b, 1, 1, 3, ' ', 0)
fmt.Fprintf(tw, "Reference\tType\t# Layers\tSize\n")
fmt.Fprintf(tw, "---------\t----\t--------\t----\n")
func buildTable(items ...item) {
// Create a table for the results
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Reference", "Type", "Platform", "# Layers", "Size"})
table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
table.SetRowLine(false)
table.SetAutoMergeCellsByColumnIndex([]int{0})
for _, i := range items {
fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n",
i.Reference, i.Type, i.Layers, i.Size,
)
if i.Type != "" {
row := []string{
i.Reference,
i.Type,
i.Platform,
fmt.Sprintf("%d", i.Layers),
i.Size,
}
table.Append(row)
}
}
tw.Flush()
return b.String()
table.Render()
}
func buildJson(item ...item) string {
@@ -95,13 +169,32 @@ func buildJson(item ...item) string {
}
type item struct {
Reference string
Type string
Layers int
Size string
Reference string
Type string
Platform string
Layers int
Size string
}
func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item {
type byReferenceAndArch []item
func (a byReferenceAndArch) Len() int { return len(a) }
func (a byReferenceAndArch) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byReferenceAndArch) Less(i, j int) bool {
if a[i].Reference == a[j].Reference {
return a[i].Platform < a[j].Platform
}
return a[i].Reference < a[j].Reference
}
func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest, plat string, o *InfoOpts) item {
// skip listing cosign items
if desc.Annotations["kind"] == "dev.cosignproject.cosign/atts" ||
desc.Annotations["kind"] == "dev.cosignproject.cosign/sigs" ||
desc.Annotations["kind"] == "dev.cosignproject.cosign/sboms" {
return item{}
}
var size int64 = 0
for _, l := range m.Layers {
size = +l.Size
@@ -117,7 +210,7 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item
case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType:
ctype = "file"
default:
ctype = "unknown"
ctype = "image"
}
ref, err := reference.Parse(desc.Annotations[ocispec.AnnotationRefName])
@@ -125,11 +218,16 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item
return item{}
}
if o.TypeFilter != "all" && ctype != o.TypeFilter {
return item{}
}
return item{
Reference: ref.Name(),
Type: ctype,
Layers: len(m.Layers),
Size: byteCountSI(size),
Reference: ref.Name(),
Type: ctype,
Platform: plat,
Layers: len(m.Layers),
Size: byteCountSI(size),
}
}

View File

@@ -5,8 +5,8 @@ import (
"os"
"github.com/mholt/archiver/v3"
"github.com/rancherfederal/ocil/pkg/content"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/spf13/cobra"
"github.com/rancherfederal/hauler/pkg/log"

View File

@@ -19,7 +19,7 @@ type SaveOpts struct {
func (o *SaveOpts) AddArgs(cmd *cobra.Command) {
f := cmd.Flags()
f.StringVarP(&o.FileName, "filename", "f", "pkg.tar.zst", "Name of archive")
f.StringVarP(&o.FileName, "filename", "f", "haul.tar.zst", "Name of archive")
}
// SaveCmd

View File

@@ -14,33 +14,32 @@ import (
"github.com/distribution/distribution/v3/version"
"github.com/spf13/cobra"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/internal/server"
"github.com/rancherfederal/hauler/pkg/log"
)
type ServeOpts struct {
type ServeRegistryOpts struct {
*RootOpts
Port int
RootDir string
ConfigFile string
Daemon bool
storedir string
}
func (o *ServeOpts) AddFlags(cmd *cobra.Command) {
func (o *ServeRegistryOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.IntVarP(&o.Port, "port", "p", 5000, "Port to listen on")
f.StringVar(&o.RootDir, "directory", "registry", "Directory to use for registry backend (defaults to '$PWD/registry')")
f.IntVarP(&o.Port, "port", "p", 5000, "Port to listen on.")
f.StringVar(&o.RootDir, "directory", "registry", "Directory to use for backend. Defaults to $PWD/registry")
f.StringVarP(&o.ConfigFile, "config", "c", "", "Path to a config file, will override all other configs")
f.BoolVarP(&o.Daemon, "daemon", "d", false, "Toggle serving as a daemon")
}
// ServeCmd serves the embedded registry almost identically to how distribution/v3 does it
func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Layout) error {
func ServeRegistryCmd(ctx context.Context, o *ServeRegistryOpts, s *store.Layout) error {
l := log.FromContext(ctx)
ctx = dcontext.WithVersion(ctx, version.Version)
tr := server.NewTempRegistry(ctx, o.RootDir)
@@ -55,7 +54,7 @@ func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Layout) error {
tr.Close()
cfg := o.defaultConfig()
cfg := o.defaultRegistryConfig()
if o.ConfigFile != "" {
ucfg, err := loadConfig(o.ConfigFile)
if err != nil {
@@ -64,14 +63,59 @@ func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Layout) error {
cfg = ucfg
}
l.Infof("starting registry on port [%d]", o.Port)
r, err := server.NewRegistry(ctx, cfg)
if err != nil {
return err
}
if err = r.ListenAndServe(); err != nil {
return err
}
return nil
}
type ServeFilesOpts struct {
*RootOpts
Port int
RootDir string
storedir string
}
func (o *ServeFilesOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.IntVarP(&o.Port, "port", "p", 8080, "Port to listen on.")
f.StringVar(&o.RootDir, "directory", "store-files", "Directory to use for backend. Defaults to $PWD/store-files")
}
func ServeFilesCmd(ctx context.Context, o *ServeFilesOpts, s *store.Layout) error {
l := log.FromContext(ctx)
ctx = dcontext.WithVersion(ctx, version.Version)
opts := &CopyOpts{}
if err := CopyCmd(ctx, opts, s, "dir://"+o.RootDir); err != nil {
return err
}
cfg := server.FileConfig{
Root: o.RootDir,
Port: o.Port,
}
f, err := server.NewFile(ctx, cfg)
if err != nil {
return err
}
l.Infof("starting file server on port [%d]", o.Port)
if err := f.ListenAndServe(); err != nil {
return err
}
return nil
}
@@ -84,7 +128,7 @@ func loadConfig(filename string) (*configuration.Configuration, error) {
return configuration.Parse(f)
}
func (o *ServeOpts) defaultConfig() *configuration.Configuration {
func (o *ServeRegistryOpts) defaultRegistryConfig() *configuration.Configuration {
cfg := &configuration.Configuration{
Version: "0.1",
Storage: configuration.Storage{
@@ -95,6 +139,10 @@ func (o *ServeOpts) defaultConfig() *configuration.Configuration {
// "maintenance": configuration.Parameters{"readonly.enabled": false},
},
}
// Add validation configuration
cfg.Validation.Manifests.URLs.Allow = []string{".+"}
cfg.Log.Level = "info"
cfg.HTTP.Addr = fmt.Sprintf(":%d", o.Port)
cfg.HTTP.Headers = http.Header{

View File

@@ -6,174 +6,243 @@ import (
"fmt"
"io"
"os"
"strings"
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"k8s.io/apimachinery/pkg/util/yaml"
"github.com/mitchellh/go-homedir"
"github.com/rancherfederal/ocil/pkg/store"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
"github.com/rancherfederal/hauler/pkg/collection/k3s"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
)
type SyncOpts struct {
*RootOpts
ContentFiles []string
Key string
Products []string
Platform string
}
func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()
f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for signature verification")
f.StringSliceVar(&o.Products, "products", []string{}, "Used for RGS Carbide customers to supply a product and version and Hauler will retrieve the images. i.e. '--product rancher=v2.7.6'")
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specific platform to save. i.e. linux/amd64. Defaults to all if flag is omitted.")
}
func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
l := log.FromContext(ctx)
// Start from an empty store (contents are cached elsewhere)
l.Debugf("flushing content store")
if err := s.Flush(ctx); err != nil {
return err
// if passed products, check for a remote manifest to retrieve and use.
for _, product := range o.Products {
l.Infof("processing content file for product: '%s'", product)
parts := strings.Split(product, "=")
tag := strings.ReplaceAll(parts[1], "+", "-")
manifestLoc := fmt.Sprintf("%s/hauler/%s-manifest.yaml:%s", consts.CarbideRegistry, parts[0], tag)
l.Infof("retrieving product manifest from: '%s'", manifestLoc)
img := v1alpha1.Image{
Name: manifestLoc,
}
err := storeImage(ctx, s, img, o.Platform)
if err != nil {
return err
}
err = ExtractCmd(ctx, &ExtractOpts{RootOpts: o.RootOpts}, s, fmt.Sprintf("hauler/%s-manifest.yaml:%s", parts[0],tag))
if err != nil {
return err
}
filename := fmt.Sprintf("%s-manifest.yaml", parts[0])
fi, err := os.Open(filename)
if err != nil {
return err
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}
// if passed a local manifest, process it
for _, filename := range o.ContentFiles {
l.Debugf("processing content file: '%s'", filename)
fi, err := os.Open(filename)
if err != nil {
return err
}
reader := yaml.NewYAMLReader(bufio.NewReader(fi))
var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
docs = append(docs, raw)
}
for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}
l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())
// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}
case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, i := range cfg.Spec.Images {
err := storeImage(ctx, s, i)
if err != nil {
return err
}
}
case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}
case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
k, err := k3s.NewK3s(cfg.Spec.Version)
if err != nil {
return err
}
if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}
case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}
if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
}
case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}
if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
}
default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}
return nil
}
func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layout) error {
l := log.FromContext(ctx)
reader := yaml.NewYAMLReader(bufio.NewReader(fi))
var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
docs = append(docs, raw)
}
for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}
l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())
// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}
case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, i := range cfg.Spec.Images {
// Check if the user provided a key.
if o.Key != "" || i.Key != "" {
key := o.Key
if i.Key != "" {
key, err = homedir.Expand(i.Key)
}
l.Debugf("key for image [%s]", key)
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
}
l.Infof("signature verified for image [%s]", i.Name)
}
// Check if the user provided a platform.
platform := o.Platform
if i.Platform != "" {
platform = i.Platform
}
err = storeImage(ctx, s, i, platform)
if err != nil {
return err
}
}
// sync with local index
s.CopyAll(ctx, s.OCI, nil)
case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}
case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
k, err := k3s.NewK3s(cfg.Spec.Version)
if err != nil {
return err
}
if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}
case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}
if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
}
case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}
if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
}
default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
}
}
return nil
}

View File

@@ -17,15 +17,20 @@ func addVersion(parent *cobra.Command) {
Aliases: []string{"v"},
RunE: func(cmd *cobra.Command, args []string) error {
v := version.GetVersionInfo()
response := v.String()
v.Name = cmd.Root().Name()
v.Description = cmd.Root().Short
v.FontName = "starwars"
cmd.SetOut(cmd.OutOrStdout())
if json {
data, err := v.JSONString()
out, err := v.JSONString()
if err != nil {
return err
return fmt.Errorf("unable to generate JSON from version info: %w", err)
}
response = data
cmd.Println(out)
} else {
cmd.Println(v.String())
}
fmt.Print(response)
return nil
},
}

View File

@@ -3,11 +3,16 @@ package main
import (
"context"
"os"
"embed"
"github.com/rancherfederal/hauler/cmd/hauler/cli"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
)
//go:embed binaries/*
var binaries embed.FS
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -15,6 +20,11 @@ func main() {
logger := log.NewLogger(os.Stdout)
ctx = logger.WithContext(ctx)
// ensure cosign binary is available
if err := cosign.EnsureBinaryExists(ctx, binaries); err != nil {
logger.Errorf("%v", err)
}
if err := cli.New().ExecuteContext(ctx); err != nil {
logger.Errorf("%v", err)
}

View File

@@ -1,177 +0,0 @@
# Walkthrough
## Installation
The latest version of `hauler` is available as statically compiled binaries for most combinations of operating systems and architectures on the GitHub [releases](https://github.com/rancherfederal/hauler/releases) page.
## Quickstart
The tl;dr for how to use `hauler` to fetch, transport, and distribute `content`:
```bash
# fetch some content
hauler store add file "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
hauler store add chart longhorn --repo "https://charts.longhorn.io"
hauler store add image "rancher/cowsay"
# transport the content
hauler store save
# <-airgap the haul.tar.zst file generated->
# load the content
hauler store load
# serve the content
hauler store serve
```
While the example above fits into a quickstart, it falls short of demonstrating all the capabilities `hauler` has to offer, including taking advantage of its fully declarative nature. Keep reading the [Guided Examples](#Guided-Examples) below for a more thorough walkthrough of `haulers` full capabilities.
## Guided Examples
Since `hauler`'s primary objective is to simplify the content collection/distribution airgap process, a lot of the design revolves around the typical airgap workflow:
```bash
fetch -> save - | <airgap> | -> validate/load -> distribute
```
This is accomplished as follows:
```bash
# fetch content
hauler store add ...
# compress and archive content
hauler store save
# <airgap>
# validate/load content
hauler store load ...
# distribute content
hauler store serve
```
At this point you're probably wondering: what is `content`? In `hauler` land, there are a few important terms given to important resources:
* `artifact`: anything that can be represented as an [`oci artifact`](https://github.com/opencontainers/artifacts)
* `content`: built in "primitive" types of `artifacts` that `hauler` understands
### Built in content
As of today, `hauler` understands three types of `content`, one with a strong legacy of community support and consensus ([`image-spec`]()), one with a finalized spec and experimental support ([`chart-spec`]()), and one generic type created just for `hauler`. These `content` types are outlined below:
__`files`__:
Generic content that can be represented as a file, either sourced locally or remotely.
```bash
# local file
hauler store add file path/to/local/file.txt
# remote file
hauler store add file https://get.k3s.io
```
__`images`__:
Any OCI compatible image can be fetched remotely.
```bash
# "shorthand" image references
hauler store add image rancher/k3s:v1.22.2-k3s1
# fully qualified image references
hauler store add image ghcr.io/fluxcd/flux-cli@sha256:02aa820c3a9c57d67208afcfc4bce9661658c17d15940aea369da259d2b976dd
```
__`charts`__:
Helm charts represented as OCI content.
```bash
# add a helm chart (defaults to latest version)
hauler store add chart loki --repo "https://grafana.github.io/helm-charts"
# add a specific version of a helm chart
hauler store add chart loki --repo "https://grafana.github.io/helm-charts" --version 2.8.1
# install directly from the oci content
HELM_EXPERIMENTAL_OCI=1 helm install loki oci://localhost:3000/library/loki --version 2.8.1
```
> Note: `hauler` supports the currently experimental format of helm as OCI content, but can also be represented as the usual tarball if necessary
### Content API
While imperatively adding `content` to `hauler` is a simple way to get started, the recommended long term approach is to use the provided api that each `content` has, in conjunction with the `sync` command.
```bash
# create a haul from declaratively defined content
hauler store sync -f testdata/contents.yaml
```
> For a commented view of the `contents` api, take a look at the `testdata` folder in the root of the project.
The API for each type of built-in `content` allows you to easily and declaratively define all the `content` that exist within a `haul`, and ensures a more gitops compatible workflow for managing the lifecycle of your `hauls`.
### Collections
Earlier we referred to `content` as "primitives". While the quotes justify the loose definition of that term, we call it that because they can be used to build groups of `content`, which we call `collections`.
`collections` are groups of 1 or more `contents` that collectively represent something desirable. Just like `content`, there are a handful that are built in to `hauler`.
Since `collections` usually contain more purposefully crafted `contents`, we restrict their use to the declarative commands (`sync`):
```bash
# sync a collection
hauler store sync -f my-collection.yaml
# sync sets of content/collection
hauler store sync -f collection.yaml -f content.yaml
```
__`thickcharts`__:
Thick Charts represent the combination of `charts` and `images`. When storing a thick chart, the chart _and_ the charts dependent images will be fetched and stored by `hauler`.
```yaml
# thick-chart.yaml
apiVersion: collection.hauler.cattle.io/v1alpha1
kind: ThickCharts
metadata:
name: loki
spec:
charts:
- name: loki
repoURL: https://grafana.github.io/helm-charts
```
When syncing the collection above, `hauler` will identify the images the chart depends on and store those too
> The method for identifying images is constantly changing, as of today, the chart is rendered and a configurable set of container defining json path's are processed. The most common paths are recognized by hauler, but this can be configured for the more niche CRDs out there.
__`k3s`__:
Combining `files` and `images`, full clusters can also be captured by `hauler` for further simplifying the already simple nature of `k3s`.
```yaml
# k3s.yaml
---
apiVersion: collection.hauler.cattle.io/v1alpha1
kind: K3s
metadata:
name: k3s
spec:
version: stable
```
Using the collection above, the dependent files (`k3s` executable and `https://get.k3s.io` script) will be fetched, as well as all the dependent images.
> We know not everyone uses the get.k3s.io script to provision k3s, in the future this may change, but until then you're welcome to mix and match the `collection` with any of your own additional `content`
#### User defined `collections`
Although `content` and `collections` can only be used when they are baked in to `hauler`, the goal is to allow these to be securely user-defined, allowing you to define your own desirable `collection` types, and leave the heavy lifting to `hauler`. Check out our [roadmap](../ROADMAP.md) and [milestones]() for more info on that.

212
go.mod
View File

@@ -1,42 +1,42 @@
module github.com/rancherfederal/hauler
go 1.17
go 1.21
require (
github.com/containerd/containerd v1.5.9
github.com/distribution/distribution/v3 v3.0.0-20211125133600-cc4627fc6e5f
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
github.com/containerd/containerd v1.7.11
github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2
github.com/docker/go-metrics v0.0.1
github.com/google/go-containerregistry v0.7.0
github.com/google/go-containerregistry v0.16.1
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/mholt/archiver/v3 v3.5.1
github.com/opencontainers/image-spec v1.0.2
github.com/mitchellh/go-homedir v1.1.0
github.com/olekukonko/tablewriter v0.0.5
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc6
github.com/pkg/errors v0.9.1
github.com/rancherfederal/ocil v0.1.9
github.com/rs/zerolog v1.26.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.3.0
helm.sh/helm/v3 v3.8.0
k8s.io/apimachinery v0.23.1
k8s.io/client-go v0.23.1
oras.land/oras-go v1.1.0
)
replace (
github.com/go-logr/logr v1.2.0 => github.com/go-logr/logr v0.4.0
k8s.io/klog/v2 v2.30.0 => k8s.io/klog/v2 v2.9.0
github.com/rs/zerolog v1.31.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/afero v1.10.0
github.com/spf13/cobra v1.8.0
golang.org/x/sync v0.6.0
helm.sh/helm/v3 v3.14.0
k8s.io/apimachinery v0.29.0
k8s.io/client-go v0.29.0
oras.land/oras-go v1.2.5
)
require (
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.2 // indirect
github.com/Masterminds/squirrel v1.5.2 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/Microsoft/hcsshim v0.11.4 // indirect
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
github.com/andybalholm/brotli v1.0.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
@@ -45,123 +45,131 @@ require (
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd // indirect
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.11+incompatible // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.12+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/docker/cli v25.0.1+incompatible // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker v25.0.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v5.7.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/go-errors/errors v1.0.1 // indirect
github.com/go-logr/logr v1.2.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.3 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.2 // indirect
github.com/gomodule/redigo v1.8.2 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmoiron/sqlx v1.3.4 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lib/pq v1.10.4 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/nwaples/rardecode v1.1.0 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.15.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.30.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rubenv/sql-migrate v1.5.2 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/ulikunitz/xz v0.5.9 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
github.com/xlab/treeprint v1.2.0 // indirect
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 // indirect
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 // indirect
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
golang.org/x/crypto v0.18.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/term v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/gorp.v1 v1.7.2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.58.3 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.23.1 // indirect
k8s.io/apiextensions-apiserver v0.23.1 // indirect
k8s.io/apiserver v0.23.1 // indirect
k8s.io/cli-runtime v0.23.1 // indirect
k8s.io/component-base v0.23.1 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/kubectl v0.23.1 // indirect
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/kustomize/api v0.10.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.29.0 // indirect
k8s.io/apiextensions-apiserver v0.29.0 // indirect
k8s.io/apiserver v0.29.0 // indirect
k8s.io/cli-runtime v0.29.0 // indirect
k8s.io/component-base v0.29.0 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
k8s.io/kubectl v0.29.0 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

1490
go.sum

File diff suppressed because it is too large Load Diff

156
install.sh Executable file
View File

@@ -0,0 +1,156 @@
#!/bin/bash
# Usage:
# - curl -sfL... | ENV_VAR=... bash
# - ENV_VAR=... bash ./install.sh
# - ./install.sh ENV_VAR=...
# Example:
# Install Latest Release
# - curl -sfL https://get.hauler.dev | bash
# Install Specific Release
# - curl -sfL https://get.hauler.dev | HAULER_VERSION=0.4.2 bash
# Documentation:
# - https://hauler.dev
# - https://github.com/rancherfederal/hauler
# set functions for debugging/logging
function info {
echo && echo "[INFO] Hauler: $1"
}
function verbose {
echo "$1"
}
function warn {
echo && echo "[WARN] Hauler: $1"
}
function fatal {
echo && echo "[ERROR] Hauler: $1"
exit 1
}
# check for required dependencies
for cmd in curl sed awk openssl tar rm; do
if ! command -v "$cmd" &> /dev/null; then
fatal "$cmd is not installed"
fi
done
# start hauler installation
info "Starting Installation..."
# set version with an environment variable
version=${HAULER_VERSION:-$(curl -s https://api.github.com/repos/rancherfederal/hauler/releases/latest | grep '"tag_name":' | sed 's/.*"v\([^"]*\)".*/\1/')}
# set verision with an argument
while [[ $# -gt 0 ]]; do
case "$1" in
HAULER_VERSION=*)
version="${1#*=}"
shift
;;
*)
shift
;;
esac
done
# detect the operating system
platform=$(uname -s | tr '[:upper:]' '[:lower:]')
case $platform in
linux)
platform="linux"
;;
darwin)
platform="darwin"
;;
*)
fatal "Unsupported Platform: $platform"
;;
esac
# detect the architecture
arch=$(uname -m)
case $arch in
x86_64 | x86-32 | x64 | x32 | amd64)
arch="amd64"
;;
aarch64 | arm64)
arch="arm64"
;;
*)
fatal "Unsupported Architecture: $arch"
;;
esac
# display the version, platform, and architecture
verbose "- Version: v$version"
verbose "- Platform: $platform"
verbose "- Architecture: $arch"
# download the checksum file
if ! curl -sOL "https://github.com/rancherfederal/hauler/releases/download/v${version}/hauler_${version}_checksums.txt"; then
fatal "Failed to Download: hauler_${version}_checksums.txt"
fi
# download the archive file
if ! curl -sOL "https://github.com/rancherfederal/hauler/releases/download/v${version}/hauler_${version}_${platform}_${arch}.tar.gz"; then
fatal "Failed to Download: hauler_${version}_${platform}_${arch}.tar.gz"
fi
# start hauler checksum verification
info "Starting Checksum Verification..."
# Verify the Hauler checksum
expected_checksum=$(awk -v version="$version" -v platform="$platform" -v arch="$arch" '$2 == "hauler_"version"_"platform"_"arch".tar.gz" {print $1}' "hauler_${version}_checksums.txt")
determined_checksum=$(openssl dgst -sha256 "hauler_${version}_${platform}_${arch}.tar.gz" | awk '{print $2}')
if [ -z "$expected_checksum" ]; then
fatal "Failed to Locate Checksum: hauler_${version}_${platform}_${arch}.tar.gz"
elif [ "$determined_checksum" = "$expected_checksum" ]; then
verbose "- Expected Checksum: $expected_checksum"
verbose "- Determined Checksum: $determined_checksum"
verbose "- Successfully Verified Checksum: hauler_${version}_${platform}_${arch}.tar.gz"
else
verbose "- Expected: $expected_checksum"
verbose "- Determined: $determined_checksum"
fatal "Failed Checksum Verification: hauler_${version}_${platform}_${arch}.tar.gz"
fi
# uncompress the archive
tar -xzf "hauler_${version}_${platform}_${arch}.tar.gz" || fatal "Failed to Extract: hauler_${version}_${platform}_${arch}.tar.gz"
# install the binary
case "$platform" in
linux)
install hauler /usr/local/bin || fatal "Failed to Install Hauler to /usr/local/bin"
;;
darwin)
install hauler /usr/local/bin || fatal "Failed to Install Hauler to /usr/local/bin"
;;
*)
fatal "Unsupported Platform or Architecture: $platform/$arch"
;;
esac
# clean up checksum(s)
rm -rf "hauler_${version}_checksums.txt" || warn "Failed to Remove: hauler_${version}_checksums.txt"
# clean up archive file(s)
rm -rf "hauler_${version}_${platform}_${arch}.tar.gz" || warn "Failed to Remove: hauler_${version}_${platform}_${arch}.tar.gz"
# clean up other files
rm -rf LICENSE README.md hauler
# display success message
info "Successfully Installed at /usr/local/bin/hauler"
# display availability message
verbose "- Hauler v${version} is now available for use!"
# display hauler docs message
verbose "- Documentation: https://hauler.dev" && echo

View File

@@ -6,7 +6,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Fn func(desc ocispec.Descriptor) (string, error)
@@ -39,7 +39,7 @@ func Images() map[string]Fn {
return "manifest.json", nil
})
for _, l := range []string{consts.DockerManifestSchema2, consts.OCIManifestSchema1} {
for _, l := range []string{consts.DockerManifestSchema2, consts.DockerManifestListSchema2, consts.OCIManifestSchema1} {
m[l] = manifestMapperFn
}

View File

@@ -21,8 +21,7 @@ type FileConfig struct {
// TODO: Better configs
func NewFile(ctx context.Context, cfg FileConfig) (Server, error) {
r := mux.NewRouter()
r.Handle("/", handlers.LoggingHandler(os.Stdout, http.FileServer(http.Dir(cfg.Root))))
r.PathPrefix("/").Handler(handlers.LoggingHandler(os.Stdout, http.StripPrefix("/", http.FileServer(http.Dir(cfg.Root)))))
if cfg.Root == "" {
cfg.Root = "."
}

View File

@@ -45,6 +45,9 @@ func NewTempRegistry(ctx context.Context, root string) *tmpRegistryServer {
"filesystem": configuration.Parameters{"rootdirectory": root},
},
}
// Add validation configuration
cfg.Validation.Manifests.URLs.Allow = []string{".+"}
cfg.Log.Level = "error"
cfg.HTTP.Headers = http.Header{
"X-Content-Type-Options": []string{"nosniff"},

View File

@@ -1,61 +1,229 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"encoding/json"
"fmt"
"path"
"os"
"runtime"
"runtime/debug"
"strings"
"sync"
"text/tabwriter"
"time"
"github.com/common-nighthawk/go-figure"
)
const unknown = "unknown"
// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags.
var (
GitVersion = "devel"
commit = "unknown"
buildDate = "unknown"
// Output of "git describe". The prerequisite is that the
// branch should be tagged using the correct versioning strategy.
gitVersion = "devel"
// SHA1 from git, output of $(git rev-parse HEAD)
gitCommit = unknown
// State of git tree, either "clean" or "dirty"
gitTreeState = unknown
// Build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
buildDate = unknown
// flag to print the ascii name banner
asciiName = "true"
// goVersion is the used golang version.
goVersion = unknown
// compiler is the used golang compiler.
compiler = unknown
// platform is the used os/arch identifier.
platform = unknown
once sync.Once
info = Info{}
)
type Info struct {
GitVersion string
GitCommit string
BuildDate string
GitVersion string `json:"gitVersion"`
GitCommit string `json:"gitCommit"`
GitTreeState string `json:"gitTreeState"`
BuildDate string `json:"buildDate"`
GoVersion string `json:"goVersion"`
Compiler string `json:"compiler"`
Platform string `json:"platform"`
GoVersion string
Compiler string
Platform string
ASCIIName string `json:"-"`
FontName string `json:"-"`
Name string `json:"-"`
Description string `json:"-"`
}
func GetVersionInfo() Info {
return Info{
GitVersion: GitVersion,
GitCommit: commit,
BuildDate: buildDate,
GoVersion: runtime.Version(),
Compiler: runtime.Compiler,
Platform: path.Join(runtime.GOOS, runtime.GOARCH),
func getBuildInfo() *debug.BuildInfo {
bi, ok := debug.ReadBuildInfo()
if !ok {
return nil
}
return bi
}
func (i Info) String() string {
func getGitVersion(bi *debug.BuildInfo) string {
if bi == nil {
return unknown
}
// TODO: remove this when the issue https://github.com/golang/go/issues/29228 is fixed
if bi.Main.Version == "(devel)" || bi.Main.Version == "" {
return gitVersion
}
return bi.Main.Version
}
func getCommit(bi *debug.BuildInfo) string {
return getKey(bi, "vcs.revision")
}
func getDirty(bi *debug.BuildInfo) string {
modified := getKey(bi, "vcs.modified")
if modified == "true" {
return "dirty"
}
if modified == "false" {
return "clean"
}
return unknown
}
func getBuildDate(bi *debug.BuildInfo) string {
buildTime := getKey(bi, "vcs.time")
t, err := time.Parse("2006-01-02T15:04:05Z", buildTime)
if err != nil {
return unknown
}
return t.Format("2006-01-02T15:04:05")
}
func getKey(bi *debug.BuildInfo, key string) string {
if bi == nil {
return unknown
}
for _, iter := range bi.Settings {
if iter.Key == key {
return iter.Value
}
}
return unknown
}
// GetVersionInfo represents known information on how this binary was built.
func GetVersionInfo() Info {
once.Do(func() {
buildInfo := getBuildInfo()
gitVersion = getGitVersion(buildInfo)
if gitCommit == unknown {
gitCommit = getCommit(buildInfo)
}
if gitTreeState == unknown {
gitTreeState = getDirty(buildInfo)
}
if buildDate == unknown {
buildDate = getBuildDate(buildInfo)
}
if goVersion == unknown {
goVersion = runtime.Version()
}
if compiler == unknown {
compiler = runtime.Compiler
}
if platform == unknown {
platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)
}
info = Info{
ASCIIName: asciiName,
GitVersion: gitVersion,
GitCommit: gitCommit,
GitTreeState: gitTreeState,
BuildDate: buildDate,
GoVersion: goVersion,
Compiler: compiler,
Platform: platform,
}
})
return info
}
// String returns the string representation of the version info
func (i *Info) String() string {
b := strings.Builder{}
w := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0)
fmt.Fprintf(w, "GitVersion:\t%s\n", i.GitVersion)
fmt.Fprintf(w, "GitCommit:\t%s\n", i.GitCommit)
fmt.Fprintf(w, "BuildDate:\t%s\n", i.BuildDate)
fmt.Fprintf(w, "GoVersion:\t%s\n", i.GoVersion)
fmt.Fprintf(w, "Compiler:\t%s\n", i.Compiler)
fmt.Fprintf(w, "Platform:\t%s\n", i.Platform)
// name and description are optional.
if i.Name != "" {
if i.ASCIIName == "true" {
f := figure.NewFigure(strings.ToUpper(i.Name), i.FontName, true)
_, _ = fmt.Fprint(w, f.String())
}
_, _ = fmt.Fprint(w, i.Name)
if i.Description != "" {
_, _ = fmt.Fprintf(w, ": %s", i.Description)
}
_, _ = fmt.Fprint(w, "\n\n")
}
w.Flush()
_, _ = fmt.Fprintf(w, "GitVersion:\t%s\n", i.GitVersion)
_, _ = fmt.Fprintf(w, "GitCommit:\t%s\n", i.GitCommit)
_, _ = fmt.Fprintf(w, "GitTreeState:\t%s\n", i.GitTreeState)
_, _ = fmt.Fprintf(w, "BuildDate:\t%s\n", i.BuildDate)
_, _ = fmt.Fprintf(w, "GoVersion:\t%s\n", i.GoVersion)
_, _ = fmt.Fprintf(w, "Compiler:\t%s\n", i.Compiler)
_, _ = fmt.Fprintf(w, "Platform:\t%s\n", i.Platform)
_ = w.Flush()
return b.String()
}
func (i Info) JSONString() (string, error) {
// JSONString returns the JSON representation of the version info
func (i *Info) JSONString() (string, error) {
b, err := json.MarshalIndent(i, "", " ")
if err != nil {
return "", err
}
return string(b), nil
}
func (i *Info) CheckFontName(fontName string) bool {
assetNames := figure.AssetNames()
for _, font := range assetNames {
if strings.Contains(font, fontName) {
return true
}
}
fmt.Fprintln(os.Stderr, "font not valid, using default")
return false
}

View File

@@ -20,4 +20,12 @@ type ImageSpec struct {
type Image struct {
// Name is the full location for the image, can be referenced by tags or digests
Name string `json:"name"`
// Path is the path to the cosign public key used for verifying image signatures
//Key string `json:"key,omitempty"`
Key string `json:"key"`
// Platform of the image to be pulled. If not specified, all platforms will be pulled.
//Platform string `json:"key,omitempty"`
Platform string `json:"platform"`
}

92
pkg/artifacts/config.go Normal file
View File

@@ -0,0 +1,92 @@
package artifacts
import (
"bytes"
"encoding/json"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ partial.Describable = (*marshallableConfig)(nil)
type Config interface {
// Raw returns the config bytes
Raw() ([]byte, error)
Digest() (v1.Hash, error)
MediaType() (types.MediaType, error)
Size() (int64, error)
}
type Marshallable interface{}
type ConfigOption func(*marshallableConfig)
// ToConfig takes anything that is marshallabe and converts it into a Config
func ToConfig(i Marshallable, opts ...ConfigOption) Config {
mc := &marshallableConfig{Marshallable: i}
for _, o := range opts {
o(mc)
}
return mc
}
func WithConfigMediaType(mediaType string) ConfigOption {
return func(config *marshallableConfig) {
config.mediaType = mediaType
}
}
// marshallableConfig implements Config using helper methods
type marshallableConfig struct {
Marshallable
mediaType string
}
func (c *marshallableConfig) MediaType() (types.MediaType, error) {
mt := c.mediaType
if mt == "" {
mt = consts.UnknownManifest
}
return types.MediaType(mt), nil
}
func (c *marshallableConfig) Raw() ([]byte, error) {
return json.Marshal(c.Marshallable)
}
func (c *marshallableConfig) Digest() (v1.Hash, error) {
return Digest(c)
}
func (c *marshallableConfig) Size() (int64, error) {
return Size(c)
}
type WithRawConfig interface {
Raw() ([]byte, error)
}
func Digest(c WithRawConfig) (v1.Hash, error) {
b, err := c.Raw()
if err != nil {
return v1.Hash{}, err
}
digest, _, err := v1.SHA256(bytes.NewReader(b))
return digest, err
}
func Size(c WithRawConfig) (int64, error) {
b, err := c.Raw()
if err != nil {
return -1, err
}
return int64(len(b)), nil
}

116
pkg/artifacts/file/file.go Normal file
View File

@@ -0,0 +1,116 @@
package file
import (
"context"
gv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/consts"
)
// interface guard
var _ artifacts.OCI = (*File)(nil)
// File implements the OCI interface for File API objects. API spec information is
// stored into the Path field.
type File struct {
Path string
computed bool
client *getter.Client
config artifacts.Config
blob gv1.Layer
manifest *gv1.Manifest
annotations map[string]string
}
func NewFile(path string, opts ...Option) *File {
client := getter.NewClient(getter.ClientOptions{})
f := &File{
client: client,
Path: path,
}
for _, opt := range opts {
opt(f)
}
return f
}
// Name is the name of the file's reference
func (f *File) Name(path string) string {
return f.client.Name(path)
}
func (f *File) MediaType() string {
return consts.OCIManifestSchema1
}
func (f *File) RawConfig() ([]byte, error) {
if err := f.compute(); err != nil {
return nil, err
}
return f.config.Raw()
}
func (f *File) Layers() ([]gv1.Layer, error) {
if err := f.compute(); err != nil {
return nil, err
}
var layers []gv1.Layer
layers = append(layers, f.blob)
return layers, nil
}
func (f *File) Manifest() (*gv1.Manifest, error) {
if err := f.compute(); err != nil {
return nil, err
}
return f.manifest, nil
}
func (f *File) compute() error {
if f.computed {
return nil
}
ctx := context.TODO()
blob, err := f.client.LayerFrom(ctx, f.Path)
if err != nil {
return err
}
layer, err := partial.Descriptor(blob)
if err != nil {
return err
}
cfg := f.client.Config(f.Path)
if cfg == nil {
cfg = f.client.Config(f.Path)
}
cfgDesc, err := partial.Descriptor(cfg)
if err != nil {
return err
}
m := &gv1.Manifest{
SchemaVersion: 2,
MediaType: gtypes.MediaType(f.MediaType()),
Config: *cfgDesc,
Layers: []gv1.Descriptor{*layer},
Annotations: f.annotations,
}
f.manifest = m
f.config = cfg
f.blob = blob
f.computed = true
return nil
}

View File

@@ -0,0 +1,166 @@
package file_test
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/consts"
)
var (
filename = "myfile.yaml"
data = []byte(`data`)
ts *httptest.Server
tfs afero.Fs
mc *getter.Client
)
func TestMain(m *testing.M) {
teardown := setup()
defer teardown()
code := m.Run()
os.Exit(code)
}
func Test_file_Config(t *testing.T) {
tests := []struct {
name string
ref string
want string
wantErr bool
}{
{
name: "should properly type local file",
ref: filename,
want: consts.FileLocalConfigMediaType,
wantErr: false,
},
{
name: "should properly type remote file",
ref: ts.URL + "/" + filename,
want: consts.FileHttpConfigMediaType,
wantErr: false,
},
// TODO: Add directory test
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := file.NewFile(tt.ref, file.WithClient(mc))
f.MediaType()
m, err := f.Manifest()
if err != nil {
t.Fatal(err)
}
got := string(m.Config.MediaType)
if got != tt.want {
t.Errorf("unxpected mediatype; got %s, want %s", got, tt.want)
}
})
}
}
func Test_file_Layers(t *testing.T) {
tests := []struct {
name string
ref string
want []byte
wantErr bool
}{
{
name: "should load a local file and preserve contents",
ref: filename,
want: data,
wantErr: false,
},
{
name: "should load a remote file and preserve contents",
ref: ts.URL + "/" + filename,
want: data,
wantErr: false,
},
// TODO: Add directory test
}
for _, tt := range tests {
t.Run(tt.name, func(it *testing.T) {
f := file.NewFile(tt.ref, file.WithClient(mc))
layers, err := f.Layers()
if (err != nil) != tt.wantErr {
it.Fatalf("unexpected Layers() error: got %v, want %v", err, tt.wantErr)
}
rc, err := layers[0].Compressed()
if err != nil {
it.Fatal(err)
}
got, err := io.ReadAll(rc)
if err != nil {
it.Fatal(err)
}
if !bytes.Equal(got, tt.want) {
it.Fatalf("unexpected Layers(): got %v, want %v", layers, tt.want)
}
})
}
}
func setup() func() {
tfs = afero.NewMemMapFs()
afero.WriteFile(tfs, filename, data, 0644)
mf := &mockFile{File: getter.NewFile(), fs: tfs}
mockHttp := getter.NewHttp()
mhttp := afero.NewHttpFs(tfs)
fileserver := http.FileServer(mhttp.Dir("."))
http.Handle("/", fileserver)
ts = httptest.NewServer(fileserver)
mc = &getter.Client{
Options: getter.ClientOptions{},
Getters: map[string]getter.Getter{
"file": mf,
"http": mockHttp,
},
}
teardown := func() {
defer ts.Close()
}
return teardown
}
type mockFile struct {
*getter.File
fs afero.Fs
}
func (m mockFile) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return m.fs.Open(filepath.Join(u.Host, u.Path))
}
func (m mockFile) Detect(u *url.URL) bool {
fi, err := m.fs.Stat(filepath.Join(u.Host, u.Path))
if err != nil {
return false
}
return !fi.IsDir()
}

View File

@@ -0,0 +1,165 @@
package getter
import (
"archive/tar"
"compress/gzip"
"context"
"io"
"net/url"
"os"
"path/filepath"
"time"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type directory struct {
*File
}
func NewDirectory() *directory {
return &directory{File: NewFile()}
}
func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
tmpfile, err := os.CreateTemp("", "hauler")
if err != nil {
return nil, err
}
digester := digest.Canonical.Digester()
zw := gzip.NewWriter(io.MultiWriter(tmpfile, digester.Hash()))
defer zw.Close()
tarDigester := digest.Canonical.Digester()
if err := tarDir(d.path(u), d.Name(u), io.MultiWriter(zw, tarDigester.Hash()), false); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
if err := tmpfile.Sync(); err != nil {
return nil, err
}
fi, err := os.Open(tmpfile.Name())
if err != nil {
return nil, err
}
// rc := &closer{
// t: io.TeeReader(tmpfile, fi),
// closes: []func() error{fi.Close, tmpfile.Close, zw.Close},
// }
return fi, nil
}
func (d directory) Detect(u *url.URL) bool {
if len(d.path(u)) == 0 {
return false
}
fi, err := os.Stat(d.path(u))
if err != nil {
return false
}
return fi.IsDir()
}
func (d directory) Config(u *url.URL) artifacts.Config {
c := &directoryConfig{
config{Reference: u.String()},
}
return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileDirectoryConfigMediaType))
}
type directoryConfig struct {
config `json:",inline,omitempty"`
}
func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error {
tw := tar.NewWriter(w)
defer tw.Close()
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Rename path
name, err := filepath.Rel(root, path)
if err != nil {
return err
}
name = filepath.Join(prefix, name)
name = filepath.ToSlash(name)
// Generate header
var link string
mode := info.Mode()
if mode&os.ModeSymlink != 0 {
if link, err = os.Readlink(path); err != nil {
return err
}
}
header, err := tar.FileInfoHeader(info, link)
if err != nil {
return errors.Wrap(err, path)
}
header.Name = name
header.Uid = 0
header.Gid = 0
header.Uname = ""
header.Gname = ""
if stripTimes {
header.ModTime = time.Time{}
header.AccessTime = time.Time{}
header.ChangeTime = time.Time{}
}
// Write file
if err := tw.WriteHeader(header); err != nil {
return errors.Wrap(err, "tar")
}
if mode.IsRegular() {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err := io.Copy(tw, file); err != nil {
return errors.Wrap(err, path)
}
}
return nil
}); err != nil {
return err
}
return nil
}
type closer struct {
t io.Reader
closes []func() error
}
func (c *closer) Read(p []byte) (n int, err error) {
return c.t.Read(p)
}
func (c *closer) Close() error {
var err error
for _, c := range c.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}

View File

@@ -0,0 +1,53 @@
package getter
import (
"context"
"io"
"net/url"
"os"
"path/filepath"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type File struct{}
func NewFile() *File {
return &File{}
}
func (f File) Name(u *url.URL) string {
return filepath.Base(f.path(u))
}
func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
return os.Open(f.path(u))
}
func (f File) Detect(u *url.URL) bool {
if len(f.path(u)) == 0 {
return false
}
fi, err := os.Stat(f.path(u))
if err != nil {
return false
}
return !fi.IsDir()
}
func (f File) path(u *url.URL) string {
return filepath.Join(u.Host, u.Path)
}
func (f File) Config(u *url.URL) artifacts.Config {
c := &fileConfig{
config{Reference: u.String()},
}
return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileLocalConfigMediaType))
}
type fileConfig struct {
config `json:",inline,omitempty"`
}

View File

@@ -0,0 +1,148 @@
package getter
import (
"context"
"fmt"
"io"
"net/url"
v1 "github.com/google/go-containerregistry/pkg/v1"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"oras.land/oras-go/pkg/content"
content2 "github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/layer"
)
type Client struct {
Getters map[string]Getter
Options ClientOptions
}
// ClientOptions provides options for the client
type ClientOptions struct {
NameOverride string
}
var (
ErrGetterTypeUnknown = errors.New("no getter type found matching reference")
)
type Getter interface {
Open(context.Context, *url.URL) (io.ReadCloser, error)
Detect(*url.URL) bool
Name(*url.URL) string
Config(*url.URL) content2.Config
}
func NewClient(opts ClientOptions) *Client {
defaults := map[string]Getter{
"file": NewFile(),
"directory": NewDirectory(),
"http": NewHttp(),
}
c := &Client{
Getters: defaults,
Options: opts,
}
return c
}
func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) {
u, err := url.Parse(source)
if err != nil {
return nil, err
}
g, err := c.getterFrom(u)
if err != nil {
if errors.Is(err, ErrGetterTypeUnknown) {
return nil, err
}
return nil, fmt.Errorf("create getter: %w", err)
}
opener := func() (io.ReadCloser, error) {
return g.Open(ctx, u)
}
annotations := make(map[string]string)
annotations[ocispec.AnnotationTitle] = c.Name(source)
switch g.(type) {
case *directory:
annotations[content.AnnotationUnpack] = "true"
}
l, err := layer.FromOpener(opener,
layer.WithMediaType(consts.FileLayerMediaType),
layer.WithAnnotations(annotations))
if err != nil {
return nil, err
}
return l, nil
}
func (c *Client) ContentFrom(ctx context.Context, source string) (io.ReadCloser, error) {
u, err := url.Parse(source)
if err != nil {
return nil, fmt.Errorf("parse source %s: %w", source, err)
}
g, err := c.getterFrom(u)
if err != nil {
if errors.Is(err, ErrGetterTypeUnknown) {
return nil, err
}
return nil, fmt.Errorf("create getter: %w", err)
}
return g.Open(ctx, u)
}
func (c *Client) getterFrom(srcUrl *url.URL) (Getter, error) {
for _, g := range c.Getters {
if g.Detect(srcUrl) {
return g, nil
}
}
return nil, errors.Wrapf(ErrGetterTypeUnknown, "source %s", srcUrl.String())
}
func (c *Client) Name(source string) string {
if c.Options.NameOverride != "" {
return c.Options.NameOverride
}
u, err := url.Parse(source)
if err != nil {
return source
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Name(u)
}
}
return source
}
func (c *Client) Config(source string) content2.Config {
u, err := url.Parse(source)
if err != nil {
return nil
}
for _, g := range c.Getters {
if g.Detect(u) {
return g.Config(u)
}
}
return nil
}
type config struct {
Reference string `json:"reference"`
Annotations map[string]string `json:"annotations,omitempty"`
}

View File

@@ -0,0 +1,139 @@
package getter_test
import (
"net/url"
"os"
"path/filepath"
"testing"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
)
func TestClient_Detect(t *testing.T) {
teardown := setup(t)
defer teardown()
c := getter.NewClient(getter.ClientOptions{})
type args struct {
source string
}
tests := []struct {
name string
args args
want string
}{
{
name: "should identify a file",
args: args{
source: fileWithExt,
},
want: "file",
},
{
name: "should identify a directory",
args: args{
source: rootDir,
},
want: "directory",
},
{
name: "should identify an http fqdn",
args: args{
source: "http://my.cool.website",
},
want: "http",
},
{
name: "should identify an http fqdn",
args: args{
source: "https://my.cool.website",
},
want: "http",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := identify(c, tt.args.source); got != tt.want {
t.Errorf("identify() = %v, want %v", got, tt.want)
}
})
}
}
func identify(c *getter.Client, source string) string {
u, _ := url.Parse(source)
for t, g := range c.Getters {
if g.Detect(u) {
return t
}
}
return ""
}
func TestClient_Name(t *testing.T) {
teardown := setup(t)
defer teardown()
type args struct {
source string
opts getter.ClientOptions
}
tests := []struct {
name string
args args
want string
}{
{
name: "should correctly name a file with an extension",
args: args{
source: fileWithExt,
opts: getter.ClientOptions{},
},
want: "file.yaml",
},
{
name: "should correctly name a directory",
args: args{
source: rootDir,
opts: getter.ClientOptions{},
},
want: rootDir,
},
{
name: "should correctly override a files name",
args: args{
source: fileWithExt,
opts: getter.ClientOptions{NameOverride: "myfile"},
},
want: "myfile",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := getter.NewClient(tt.args.opts)
if got := c.Name(tt.args.source); got != tt.want {
t.Errorf("Name() = %v, want %v", got, tt.want)
}
})
}
}
var (
rootDir = "gettertests"
fileWithExt = filepath.Join(rootDir, "file.yaml")
)
func setup(t *testing.T) func() {
if err := os.MkdirAll(rootDir, os.ModePerm); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil {
t.Fatal(err)
}
return func() {
os.RemoveAll(rootDir)
}
}

View File

@@ -0,0 +1,67 @@
package getter
import (
"context"
"io"
"mime"
"net/http"
"net/url"
"path/filepath"
"strings"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Http struct{}
func NewHttp() *Http {
return &Http{}
}
func (h Http) Name(u *url.URL) string {
resp, err := http.Head(u.String())
if err != nil {
return ""
}
contentType := resp.Header.Get("Content-Type")
for _, v := range strings.Split(contentType, ",") {
t, _, err := mime.ParseMediaType(v)
if err != nil {
break
}
// TODO: Identify known mimetypes for hints at a filename
_ = t
}
// TODO: Not this
return filepath.Base(u.String())
}
func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) {
resp, err := http.Get(u.String())
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (h Http) Detect(u *url.URL) bool {
switch u.Scheme {
case "http", "https":
return true
}
return false
}
func (h *Http) Config(u *url.URL) artifacts.Config {
c := &httpConfig{
config{Reference: u.String()},
}
return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileHttpConfigMediaType))
}
type httpConfig struct {
config `json:",inline,omitempty"`
}

View File

@@ -0,0 +1,26 @@
package file
import (
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
)
type Option func(*File)
func WithClient(c *getter.Client) Option {
return func(f *File) {
f.client = c
}
}
func WithConfig(obj interface{}, mediaType string) Option {
return func(f *File) {
f.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType))
}
}
func WithAnnotations(m map[string]string) Option {
return func(f *File) {
f.annotations = m
}
}

View File

@@ -0,0 +1,53 @@
package image
import (
"github.com/google/go-containerregistry/pkg/authn"
gname "github.com/google/go-containerregistry/pkg/name"
gv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/rancherfederal/hauler/pkg/artifacts"
)
var _ artifacts.OCI = (*Image)(nil)
func (i *Image) MediaType() string {
mt, err := i.Image.MediaType()
if err != nil {
return ""
}
return string(mt)
}
func (i *Image) RawConfig() ([]byte, error) {
return i.RawConfigFile()
}
// Image implements the OCI interface for Image API objects. API spec information
// is stored into the Name field.
type Image struct {
Name string
gv1.Image
}
func NewImage(name string, opts ...remote.Option) (*Image, error) {
r, err := gname.ParseReference(name)
if err != nil {
return nil, err
}
defaultOpts := []remote.Option{
remote.WithAuthFromKeychain(authn.DefaultKeychain),
}
opts = append(opts, defaultOpts...)
img, err := remote.Image(r, opts...)
if err != nil {
return nil, err
}
return &Image{
Name: name,
Image: img,
}, nil
}

View File

@@ -0,0 +1 @@
package image_test

View File

@@ -0,0 +1,78 @@
package memory
import (
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ artifacts.OCI = (*Memory)(nil)
// Memory implements the OCI interface for a generic set of bytes stored in memory.
type Memory struct {
blob v1.Layer
annotations map[string]string
config artifacts.Config
}
type defaultConfig struct {
MediaType string `json:"mediaType,omitempty"`
}
func NewMemory(data []byte, mt string, opts ...Option) *Memory {
blob := static.NewLayer(data, types.MediaType(mt))
cfg := defaultConfig{MediaType: consts.MemoryConfigMediaType}
m := &Memory{
blob: blob,
config: artifacts.ToConfig(cfg),
}
for _, opt := range opts {
opt(m)
}
return m
}
func (m *Memory) MediaType() string {
return consts.OCIManifestSchema1
}
func (m *Memory) Manifest() (*v1.Manifest, error) {
layer, err := partial.Descriptor(m.blob)
if err != nil {
return nil, err
}
cfgDesc, err := partial.Descriptor(m.config)
if err != nil {
return nil, err
}
manifest := &v1.Manifest{
SchemaVersion: 2,
MediaType: types.MediaType(m.MediaType()),
Config: *cfgDesc,
Layers: []v1.Descriptor{*layer},
Annotations: m.annotations,
}
return manifest, nil
}
func (m *Memory) RawConfig() ([]byte, error) {
if m.config == nil {
return []byte(`{}`), nil
}
return m.config.Raw()
}
func (m *Memory) Layers() ([]v1.Layer, error) {
var layers []v1.Layer
layers = append(layers, m.blob)
return layers, nil
}

View File

@@ -0,0 +1,61 @@
package memory_test
import (
"math/rand"
"testing"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/opencontainers/go-digest"
"github.com/rancherfederal/hauler/pkg/artifacts/memory"
)
func TestMemory_Layers(t *testing.T) {
tests := []struct {
name string
want *v1.Manifest
wantErr bool
}{
{
name: "should preserve content",
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data, m := setup(t)
layers, err := m.Layers()
if err != nil {
t.Fatal(err)
}
if len(layers) != 1 {
t.Fatalf("Expected 1 layer, got %d", len(layers))
}
h, err := layers[0].Digest()
if err != nil {
t.Fatal(err)
}
d := digest.FromBytes(data)
if d.String() != h.String() {
t.Fatalf("bytes do not match, got %s, expected %s", h.String(), d.String())
}
})
}
}
func setup(t *testing.T) ([]byte, *memory.Memory) {
block := make([]byte, 2048)
_, err := rand.Read(block)
if err != nil {
t.Fatal(err)
}
mem := memory.NewMemory(block, "random")
return block, mem
}

View File

@@ -0,0 +1,17 @@
package memory
import "github.com/rancherfederal/hauler/pkg/artifacts"
type Option func(*Memory)
func WithConfig(obj interface{}, mediaType string) Option {
return func(m *Memory) {
m.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType))
}
}
func WithAnnotations(annotations map[string]string) Option {
return func(m *Memory) {
m.annotations = annotations
}
}

21
pkg/artifacts/ocis.go Normal file
View File

@@ -0,0 +1,21 @@
package artifacts
import "github.com/google/go-containerregistry/pkg/v1"
// OCI is the bare minimum we need to represent an artifact in an oci layout
// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality
// This specific implementation fully encapsulates v1.Layer's within a more generic form
type OCI interface {
MediaType() string
Manifest() (*v1.Manifest, error)
RawConfig() ([]byte, error)
Layers() ([]v1.Layer, error)
}
type OCICollection interface {
// Contents returns the list of contents in the collection
Contents() (map[string]OCI, error)
}

View File

@@ -1,8 +1,8 @@
package chart
import (
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"

View File

@@ -12,9 +12,9 @@ import (
"github.com/rancherfederal/hauler/pkg/log"
"github.com/google/go-containerregistry/pkg/name"
artifact "github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
artifact "github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
)
type ImageTxt struct {

View File

@@ -8,8 +8,8 @@ import (
"os"
"testing"
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
)
var (

View File

@@ -10,12 +10,12 @@ import (
"path"
"strings"
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/ocil/pkg/artifacts/image"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts/image"
"github.com/rancherfederal/ocil/pkg/artifacts/file"
"github.com/rancherfederal/hauler/pkg/artifacts/file"
"github.com/rancherfederal/ocil/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/artifacts/file/getter"
"github.com/rancherfederal/hauler/pkg/reference"
)

54
pkg/consts/consts.go Normal file
View File

@@ -0,0 +1,54 @@
package consts
const (
OCIManifestSchema1 = "application/vnd.oci.image.manifest.v1+json"
DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json"
DockerManifestListSchema2 = "application/vnd.docker.distribution.manifest.list.v2+json"
OCIImageIndexSchema = "application/vnd.oci.image.index.v1+json"
DockerConfigJSON = "application/vnd.docker.container.image.v1+json"
DockerLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip"
DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar"
OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip"
OCIArtifact = "application/vnd.oci.empty.v1+json"
// ChartConfigMediaType is the reserved media type for the Helm chart manifest config
ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
// ChartLayerMediaType is the reserved media type for Helm chart package content
ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"
// ProvLayerMediaType is the reserved media type for Helm chart provenance files
ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov"
// FileLayerMediaType is the reserved media type for File content layers
FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1"
// FileLocalConfigMediaType is the reserved media type for File config
FileLocalConfigMediaType = "application/vnd.content.hauler.file.local.config.v1+json"
FileDirectoryConfigMediaType = "application/vnd.content.hauler.file.directory.config.v1+json"
FileHttpConfigMediaType = "application/vnd.content.hauler.file.http.config.v1+json"
// MemoryConfigMediaType is the reserved media type for Memory config for a generic set of bytes stored in memory
MemoryConfigMediaType = "application/vnd.content.hauler.memory.config.v1+json"
// WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers
WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm"
// WasmConfigMediaType is the reserved media type for WASM configs
WasmConfigMediaType = "application/vnd.wasm.config.v1+json"
UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json"
UnknownLayer = "application/vnd.content.hauler.unknown.layer"
OCIVendorPrefix = "vnd.oci"
DockerVendorPrefix = "vnd.docker"
HaulerVendorPrefix = "vnd.hauler"
OCIImageIndexFile = "index.json"
KindAnnotationName = "kind"
KindAnnotation = "dev.cosignproject.cosign/image"
CarbideRegistry = "rgcrprod.azurecr.us"
)

View File

@@ -14,15 +14,15 @@ import (
"github.com/google/go-containerregistry/pkg/v1/partial"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rancherfederal/ocil/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/artifacts"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"github.com/rancherfederal/ocil/pkg/layer"
"github.com/rancherfederal/hauler/pkg/layer"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ artifacts.OCI = (*Chart)(nil)
@@ -137,7 +137,8 @@ func (h *Chart) RawChartData() ([]byte, error) {
}
// chartData loads the chart contents into memory and returns a NopCloser for the contents
// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB
//
// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB
func (h *Chart) chartData() (gv1.Layer, error) {
info, err := os.Stat(h.path)
if err != nil {

View File

@@ -10,7 +10,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"helm.sh/helm/v3/pkg/action"
"github.com/rancherfederal/ocil/pkg/consts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content/chart"
)

288
pkg/content/oci.go Normal file
View File

@@ -0,0 +1,288 @@
package content
import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"sync"
ccontent "github.com/containerd/containerd/content"
"github.com/containerd/containerd/remotes"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/pkg/content"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/pkg/consts"
)
var _ target.Target = (*OCI)(nil)
type OCI struct {
root string
index *ocispec.Index
nameMap *sync.Map // map[string]ocispec.Descriptor
}
func NewOCI(root string) (*OCI, error) {
o := &OCI{
root: root,
nameMap: &sync.Map{},
}
return o, nil
}
// AddIndex adds a descriptor to the index and updates it
//
// The descriptor must use AnnotationRefName to identify itself
func (o *OCI) AddIndex(desc ocispec.Descriptor) error {
if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
return fmt.Errorf("descriptor must contain a reference from the annotation: %s", ocispec.AnnotationRefName)
}
key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName])
o.nameMap.Store(key, desc)
return o.SaveIndex()
}
// LoadIndex will load the index from disk
func (o *OCI) LoadIndex() error {
path := o.path(consts.OCIImageIndexFile)
idx, err := os.Open(path)
if err != nil {
if !os.IsNotExist(err) {
return err
}
o.index = &ocispec.Index{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
}
return nil
}
defer idx.Close()
if err := json.NewDecoder(idx).Decode(&o.index); err != nil {
return err
}
for _, desc := range o.index.Manifests {
key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName])
if strings.TrimSpace(key) != "--" {
o.nameMap.Store(key, desc)
}
}
return nil
}
// SaveIndex will update the index on disk
func (o *OCI) SaveIndex() error {
var descs []ocispec.Descriptor
o.nameMap.Range(func(name, desc interface{}) bool {
n := desc.(ocispec.Descriptor).Annotations[ocispec.AnnotationRefName]
d := desc.(ocispec.Descriptor)
if d.Annotations == nil {
d.Annotations = make(map[string]string)
}
d.Annotations[ocispec.AnnotationRefName] = n
descs = append(descs, d)
return true
})
// sort index to ensure that images come before any signatures and attestations.
sort.SliceStable(descs, func(i, j int) bool {
kindI := descs[i].Annotations["kind"]
kindJ := descs[j].Annotations["kind"]
// Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top.
if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) {
return true
} else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) {
return false
}
return false // Default: maintain the order.
})
o.index.Manifests = descs
data, err := json.Marshal(o.index)
if err != nil {
return err
}
return os.WriteFile(o.path(consts.OCIImageIndexFile), data, 0644)
}
// Resolve attempts to resolve the reference into a name and descriptor.
//
// The argument `ref` should be a scheme-less URI representing the remote.
// Structurally, it has a host and path. The "host" can be used to directly
// reference a specific host or be matched against a specific handler.
//
// The returned name should be used to identify the referenced entity.
// Dependending on the remote namespace, this may be immutable or mutable.
// While the name may differ from ref, it should itself be a valid ref.
//
// If the resolution fails, an error will be returned.
func (o *OCI) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
if err := o.LoadIndex(); err != nil {
return "", ocispec.Descriptor{}, err
}
d, ok := o.nameMap.Load(ref)
if !ok {
return "", ocispec.Descriptor{}, err
}
desc = d.(ocispec.Descriptor)
return ref, desc, nil
}
// Fetcher returns a new fetcher for the provided reference.
// All content fetched from the returned fetcher will be
// from the namespace referred to by ref.
func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) {
if err := o.LoadIndex(); err != nil {
return nil, err
}
if _, ok := o.nameMap.Load(ref); !ok {
return nil, nil
}
return o, nil
}
func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
readerAt, err := o.blobReaderAt(desc)
if err != nil {
return nil, err
}
return readerAt, nil
}
func (o *OCI) FetchManifest(ctx context.Context, manifest ocispec.Manifest) (io.ReadCloser, error) {
readerAt, err := o.manifestBlobReaderAt(manifest)
if err != nil {
return nil, err
}
return readerAt, nil
}
// Pusher returns a new pusher for the provided reference
// The returned Pusher should satisfy content.Ingester and concurrent attempts
// to push the same blob using the Ingester API should result in ErrUnavailable.
func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) {
if err := o.LoadIndex(); err != nil {
return nil, err
}
var baseRef, hash string
parts := strings.SplitN(ref, "@", 2)
baseRef = parts[0]
if len(parts) > 1 {
hash = parts[1]
}
return &ociPusher{
oci: o,
ref: baseRef,
digest: hash,
}, nil
}
func (o *OCI) Walk(fn func(reference string, desc ocispec.Descriptor) error) error {
if err := o.LoadIndex(); err != nil {
return err
}
var errst []string
o.nameMap.Range(func(key, value interface{}) bool {
if err := fn(key.(string), value.(ocispec.Descriptor)); err != nil {
errst = append(errst, err.Error())
}
return true
})
if errst != nil {
return fmt.Errorf(strings.Join(errst, "; "))
}
return nil
}
func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) {
blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex())
if err != nil {
return nil, err
}
return os.Open(blobPath)
}
func (o *OCI) manifestBlobReaderAt(manifest ocispec.Manifest) (*os.File, error) {
blobPath, err := o.ensureBlob(string(manifest.Config.Digest.Algorithm().String()), manifest.Config.Digest.Hex())
if err != nil {
return nil, err
}
return os.Open(blobPath)
}
func (o *OCI) blobWriterAt(desc ocispec.Descriptor) (*os.File, error) {
blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex())
if err != nil {
return nil, err
}
return os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
}
func (o *OCI) ensureBlob(alg string, hex string) (string, error) {
dir := o.path("blobs", alg)
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return "", err
}
return filepath.Join(dir, hex), nil
}
func (o *OCI) path(elem ...string) string {
complete := []string{string(o.root)}
return filepath.Join(append(complete, elem...)...)
}
type ociPusher struct {
oci *OCI
ref string
digest string
}
// Push returns a content writer for the given resource identified
// by the descriptor.
func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Writer, error) {
switch d.MediaType {
case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex, consts.DockerManifestSchema2, consts.DockerManifestListSchema2:
// if the hash of the content matches that which was provided as the hash for the root, mark it
if p.digest != "" && p.digest == d.Digest.String() {
if err := p.oci.LoadIndex(); err != nil {
return nil, err
}
p.oci.nameMap.Store(p.ref, d)
if err := p.oci.SaveIndex(); err != nil {
return nil, err
}
}
}
blobPath, err := p.oci.ensureBlob(d.Digest.Algorithm().String(), d.Digest.Hex())
if err != nil {
return nil, err
}
if _, err := os.Stat(blobPath); err == nil {
// file already exists, discard (but validate digest)
return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil
}
f, err := os.Create(blobPath)
if err != nil {
return nil, err
}
w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest))
return w, nil
}

245
pkg/cosign/cosign.go Normal file
View File

@@ -0,0 +1,245 @@
package cosign
import (
"fmt"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"context"
"time"
"bufio"
"embed"
"strings"
"oras.land/oras-go/pkg/content"
"github.com/rancherfederal/hauler/pkg/store"
"github.com/rancherfederal/hauler/pkg/log"
)
const maxRetries = 3
const retryDelay = time.Second * 5
// VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign.
func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string) error {
operation := func() error {
cosignBinaryPath, err := getCosignPath(ctx)
if err != nil {
return err
}
cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, ref)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error verifying signature: %v, output: %s", err, output)
}
return nil
}
return RetryOperation(ctx, operation)
}
// SaveImage saves image and any signatures/attestations to the store.
func SaveImage(ctx context.Context, s *store.Layout, ref string, platform string) error {
operation := func() error {
cosignBinaryPath, err := getCosignPath(ctx)
if err != nil {
return err
}
cmd := exec.Command(cosignBinaryPath, "save", ref, "--dir", s.Root)
// Conditionally add platform.
if platform != "" {
cmd.Args = append(cmd.Args, "--platform", platform)
}
output, err := cmd.CombinedOutput()
if err != nil {
if strings.Contains(string(output), "specified reference is not a multiarch image") {
// Rerun the command without the platform flag
cmd = exec.Command(cosignBinaryPath, "save", ref, "--dir", s.Root)
output, err = cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error adding image to store: %v, output: %s", err, output)
}
} else {
return fmt.Errorf("error adding image to store: %v, output: %s", err, output)
}
}
return nil
}
return RetryOperation(ctx, operation)
}
// LoadImage loads store to a remote registry.
func LoadImages(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error {
l := log.FromContext(ctx)
cosignBinaryPath, err := getCosignPath(ctx)
if err != nil {
return err
}
cmd := exec.Command(cosignBinaryPath, "load", "--registry", registry, "--dir", s.Root)
// Conditionally add extra registry flags.
if ropts.Insecure {
cmd.Args = append(cmd.Args, "--allow-insecure-registry=true")
}
if ropts.PlainHTTP {
cmd.Args = append(cmd.Args, "--allow-http-registry=true")
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
// start the command after having set up the pipe
if err := cmd.Start(); err != nil {
return err
}
// read command's stdout line by line
output := bufio.NewScanner(stdout)
for output.Scan() {
l.Infof(output.Text()) // write each line to your log, or anything you need
}
if err := output.Err(); err != nil {
cmd.Wait()
return err
}
// read command's stderr line by line
errors := bufio.NewScanner(stderr)
for errors.Scan() {
l.Errorf(errors.Text()) // write each line to your log, or anything you need
}
if err := errors.Err(); err != nil {
cmd.Wait()
return err
}
// Wait for the command to finish
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
// RegistryLogin - performs cosign login
func RegistryLogin(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error {
cosignBinaryPath, err := getCosignPath(ctx)
if err != nil {
return err
}
cmd := exec.Command(cosignBinaryPath, "login", registry, "-u", ropts.Username, "-p", ropts.Password)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error logging into registry: %v, output: %s", err, output)
}
return nil
}
func RetryOperation(ctx context.Context, operation func() error) error {
l := log.FromContext(ctx)
for attempt := 1; attempt <= maxRetries; attempt++ {
err := operation()
if err == nil {
// If the operation succeeds, return nil (no error).
return nil
}
// Log the error for the current attempt.
l.Errorf("Error (attempt %d/%d): %v", attempt, maxRetries, err)
// If this is not the last attempt, wait before retrying.
if attempt < maxRetries {
time.Sleep(retryDelay)
}
}
// If all attempts fail, return an error.
return fmt.Errorf("operation failed after %d attempts", maxRetries)
}
func EnsureBinaryExists(ctx context.Context, bin embed.FS) (error) {
// Set up a path for the binary to be copied.
binaryPath, err := getCosignPath(ctx)
if err != nil {
return fmt.Errorf("Error: %v\n", err)
}
// Determine the architecture so that we pull the correct embedded binary.
arch := runtime.GOARCH
rOS := runtime.GOOS
binaryName := "cosign"
if rOS == "windows" {
binaryName = fmt.Sprintf("cosign-%s-%s.exe", rOS, arch)
} else {
binaryName = fmt.Sprintf("cosign-%s-%s", rOS, arch)
}
// retrieve the embedded binary
f, err := bin.ReadFile(fmt.Sprintf("binaries/%s", binaryName))
if err != nil {
return fmt.Errorf("Error: %v\n", err)
}
// write the binary to the filesystem
err = os.WriteFile(binaryPath, f, 0755)
if err != nil {
return fmt.Errorf("Error: %v\n", err)
}
return nil
}
// getCosignPath returns the binary path
func getCosignPath(ctx context.Context) (string, error) {
// Get the current user's information
currentUser, err := user.Current()
if err != nil {
return "", fmt.Errorf("Error: %v\n", err)
}
// Get the user's home directory
homeDir := currentUser.HomeDir
// Construct the path to the .hauler directory
haulerDir := filepath.Join(homeDir, ".hauler")
// Create the .hauler directory if it doesn't exist
if _, err := os.Stat(haulerDir); os.IsNotExist(err) {
// .hauler directory does not exist, create it
if err := os.MkdirAll(haulerDir, 0755); err != nil {
return "", fmt.Errorf("Error creating .hauler directory: %v\n", err)
}
}
// Determine the binary name.
rOS := runtime.GOOS
binaryName := "cosign"
if rOS == "windows" {
binaryName = "cosign.exe"
}
// construct path to binary
binaryPath := filepath.Join(haulerDir, binaryName)
return binaryPath, nil
}

106
pkg/layer/cache.go Normal file
View File

@@ -0,0 +1,106 @@
package layer
import (
"errors"
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/artifacts"
)
/*
This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache
*/
type Cache interface {
Put(v1.Layer) (v1.Layer, error)
Get(v1.Hash) (v1.Layer, error)
}
var ErrLayerNotFound = errors.New("layer not found")
type oci struct {
artifacts.OCI
c Cache
}
func OCICache(o artifacts.OCI, c Cache) artifacts.OCI {
return &oci{
OCI: o,
c: c,
}
}
func (o *oci) Layers() ([]v1.Layer, error) {
ls, err := o.OCI.Layers()
if err != nil {
return nil, err
}
var out []v1.Layer
for _, l := range ls {
out = append(out, &lazyLayer{inner: l, c: o.c})
}
return out, nil
}
type lazyLayer struct {
inner v1.Layer
c Cache
}
func (l *lazyLayer) Compressed() (io.ReadCloser, error) {
digest, err := l.inner.Digest()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(digest)
if err != nil {
return nil, err
}
return layer.Compressed()
}
func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) {
diffID, err := l.inner.DiffID()
if err != nil {
return nil, err
}
layer, err := l.getOrPut(diffID)
if err != nil {
return nil, err
}
return layer.Uncompressed()
}
func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) {
var layer v1.Layer
if cl, err := l.c.Get(h); err == nil {
layer = cl
} else if err == ErrLayerNotFound {
rl, err := l.c.Put(l.inner)
if err != nil {
return nil, err
}
layer = rl
} else {
return nil, err
}
return layer, nil
}
func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() }
func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() }
func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() }

118
pkg/layer/filesystem.go Normal file
View File

@@ -0,0 +1,118 @@
package layer
import (
"io"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
type fs struct {
root string
}
func NewFilesystemCache(root string) Cache {
return &fs{root: root}
}
func (f *fs) Put(l v1.Layer) (v1.Layer, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
diffID, err := l.DiffID()
if err != nil {
return nil, err
}
return &cachedLayer{
Layer: l,
root: f.root,
digest: digest,
diffID: diffID,
}, nil
}
func (f *fs) Get(h v1.Hash) (v1.Layer, error) {
opener := f.open(h)
l, err := FromOpener(opener)
if os.IsNotExist(err) {
return nil, ErrLayerNotFound
}
return l, err
}
func (f *fs) open(h v1.Hash) Opener {
return func() (io.ReadCloser, error) {
return os.Open(layerpath(f.root, h))
}
}
type cachedLayer struct {
v1.Layer
root string
digest, diffID v1.Hash
}
func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) {
lp := layerpath(l.root, h)
if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil {
return nil, err
}
return os.Create(lp)
}
func (l *cachedLayer) Compressed() (io.ReadCloser, error) {
f, err := l.create(l.digest)
if err != nil {
return nil, nil
}
rc, err := l.Layer.Compressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) {
f, err := l.create(l.diffID)
if err != nil {
return nil, err
}
rc, err := l.Layer.Uncompressed()
if err != nil {
return nil, err
}
return &readcloser{
t: io.TeeReader(rc, f),
closes: []func() error{rc.Close, f.Close},
}, nil
}
func layerpath(root string, h v1.Hash) string {
return filepath.Join(root, h.Algorithm, h.Hex)
}
type readcloser struct {
t io.Reader
closes []func() error
}
func (rc *readcloser) Read(b []byte) (int, error) {
return rc.t.Read(b)
}
func (rc *readcloser) Close() error {
var err error
for _, c := range rc.closes {
lastErr := c()
if err == nil {
err = lastErr
}
}
return err
}

127
pkg/layer/layer.go Normal file
View File

@@ -0,0 +1,127 @@
package layer
import (
"io"
v1 "github.com/google/go-containerregistry/pkg/v1"
gtypes "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/rancherfederal/hauler/pkg/consts"
)
type Opener func() (io.ReadCloser, error)
func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) {
var err error
layer := &layer{
mediaType: consts.UnknownLayer,
annotations: make(map[string]string, 1),
}
layer.uncompressedOpener = opener
layer.compressedOpener = func() (io.ReadCloser, error) {
rc, err := opener()
if err != nil {
return nil, err
}
return rc, nil
}
for _, opt := range opts {
opt(layer)
}
if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil {
return nil, err
}
if layer.diffID, _, err = compute(layer.compressedOpener); err != nil {
return nil, err
}
return layer, nil
}
func compute(opener Opener) (v1.Hash, int64, error) {
rc, err := opener()
if err != nil {
return v1.Hash{}, 0, err
}
defer rc.Close()
return v1.SHA256(rc)
}
type Option func(*layer)
func WithMediaType(mt string) Option {
return func(l *layer) {
l.mediaType = mt
}
}
func WithAnnotations(annotations map[string]string) Option {
return func(l *layer) {
if l.annotations == nil {
l.annotations = make(map[string]string)
}
l.annotations = annotations
}
}
type layer struct {
digest v1.Hash
diffID v1.Hash
size int64
compressedOpener Opener
uncompressedOpener Opener
mediaType string
annotations map[string]string
urls []string
}
func (l layer) Descriptor() (*v1.Descriptor, error) {
digest, err := l.Digest()
if err != nil {
return nil, err
}
mt, err := l.MediaType()
if err != nil {
return nil, err
}
return &v1.Descriptor{
MediaType: mt,
Size: l.size,
Digest: digest,
Annotations: l.annotations,
URLs: l.urls,
// TODO: Allow platforms
Platform: nil,
}, nil
}
func (l layer) Digest() (v1.Hash, error) {
return l.digest, nil
}
func (l layer) DiffID() (v1.Hash, error) {
return l.diffID, nil
}
func (l layer) Compressed() (io.ReadCloser, error) {
return l.compressedOpener()
}
func (l layer) Uncompressed() (io.ReadCloser, error) {
return l.uncompressedOpener()
}
func (l layer) Size() (int64, error) {
return l.size, nil
}
func (l layer) MediaType() (gtypes.MediaType, error) {
return gtypes.MediaType(l.mediaType), nil
}

View File

@@ -29,7 +29,7 @@ func NewTagged(n string, tag string) (gname.Reference, error) {
if err != nil {
return nil, err
}
tag = strings.Replace(tag, "+", "-", -1)
return repo.Context().Tag(tag), nil
}

262
pkg/store/store.go Normal file
View File

@@ -0,0 +1,262 @@
package store
import (
"context"
"encoding/json"
"io"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"oras.land/oras-go/pkg/oras"
"oras.land/oras-go/pkg/target"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/layer"
)
type Layout struct {
*content.OCI
Root string
cache layer.Cache
}
type Options func(*Layout)
func WithCache(c layer.Cache) Options {
return func(l *Layout) {
l.cache = c
}
}
func NewLayout(rootdir string, opts ...Options) (*Layout, error) {
ociStore, err := content.NewOCI(rootdir)
if err != nil {
return nil, err
}
if err := ociStore.LoadIndex(); err != nil {
return nil, err
}
l := &Layout{
Root: rootdir,
OCI: ociStore,
}
for _, opt := range opts {
opt(l)
}
return l, nil
}
// AddOCI adds an artifacts.OCI to the store
//
// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once
// saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use
// strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the
// future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout.
func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (ocispec.Descriptor, error) {
if l.cache != nil {
cached := layer.OCICache(oci, l.cache)
oci = cached
}
// Write manifest blob
m, err := oci.Manifest()
if err != nil {
return ocispec.Descriptor{}, err
}
mdata, err := json.Marshal(m)
if err != nil {
return ocispec.Descriptor{}, err
}
if err := l.writeBlobData(mdata); err != nil {
return ocispec.Descriptor{}, err
}
// Write config blob
cdata, err := oci.RawConfig()
if err != nil {
return ocispec.Descriptor{}, err
}
static.NewLayer(cdata, "")
if err := l.writeBlobData(cdata); err != nil {
return ocispec.Descriptor{}, err
}
// write blob layers concurrently
layers, err := oci.Layers()
if err != nil {
return ocispec.Descriptor{}, err
}
var g errgroup.Group
for _, lyr := range layers {
lyr := lyr
g.Go(func() error {
return l.writeLayer(lyr)
})
}
if err := g.Wait(); err != nil {
return ocispec.Descriptor{}, err
}
// Build index
idx := ocispec.Descriptor{
MediaType: string(m.MediaType),
Digest: digest.FromBytes(mdata),
Size: int64(len(mdata)),
Annotations: map[string]string{
consts.KindAnnotationName: consts.KindAnnotation,
ocispec.AnnotationRefName: ref,
},
URLs: nil,
Platform: nil,
}
return idx, l.OCI.AddIndex(idx)
}
// AddOCICollection .
func (l *Layout) AddOCICollection(ctx context.Context, collection artifacts.OCICollection) ([]ocispec.Descriptor, error) {
cnts, err := collection.Contents()
if err != nil {
return nil, err
}
var descs []ocispec.Descriptor
for ref, oci := range cnts {
desc, err := l.AddOCI(ctx, oci, ref)
if err != nil {
return nil, err
}
descs = append(descs, desc)
}
return descs, nil
}
// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting oci-layout content
//
// This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents
// To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly deletes oci-layout content only
func (l *Layout) Flush(ctx context.Context) error {
blobs := filepath.Join(l.Root, "blobs")
if err := os.RemoveAll(blobs); err != nil {
return err
}
index := filepath.Join(l.Root, "index.json")
if err := os.RemoveAll(index); err != nil {
return err
}
layout := filepath.Join(l.Root, "oci-layout")
if err := os.RemoveAll(layout); err != nil {
return err
}
return nil
}
// Copy will copy a given reference to a given target.Target
//
// This is essentially a wrapper around oras.Copy, but locked to this content store
func (l *Layout) Copy(ctx context.Context, ref string, to target.Target, toRef string) (ocispec.Descriptor, error) {
return oras.Copy(ctx, l.OCI, ref, to, toRef,
oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2, consts.DockerManifestListSchema2))
}
// CopyAll performs bulk copy operations on the stores oci layout to a provided target.Target
func (l *Layout) CopyAll(ctx context.Context, to target.Target, toMapper func(string) (string, error)) ([]ocispec.Descriptor, error) {
var descs []ocispec.Descriptor
err := l.OCI.Walk(func(reference string, desc ocispec.Descriptor) error {
toRef := ""
if toMapper != nil {
tr, err := toMapper(reference)
if err != nil {
return err
}
toRef = tr
}
desc, err := l.Copy(ctx, reference, to, toRef)
if err != nil {
return err
}
descs = append(descs, desc)
return nil
})
if err != nil {
return nil, err
}
return descs, nil
}
// Identify is a helper function that will identify a human-readable content type given a descriptor
func (l *Layout) Identify(ctx context.Context, desc ocispec.Descriptor) string {
rc, err := l.OCI.Fetch(ctx, desc)
if err != nil {
return ""
}
defer rc.Close()
m := struct {
Config struct {
MediaType string `json:"mediaType"`
} `json:"config"`
}{}
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return ""
}
return m.Config.MediaType
}
func (l *Layout) writeBlobData(data []byte) error {
blob := static.NewLayer(data, "") // NOTE: MediaType isn't actually used in the writing
return l.writeLayer(blob)
}
func (l *Layout) writeLayer(layer v1.Layer) error {
d, err := layer.Digest()
if err != nil {
return err
}
r, err := layer.Compressed()
if err != nil {
return err
}
dir := filepath.Join(l.Root, "blobs", d.Algorithm)
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return err
}
blobPath := filepath.Join(dir, d.Hex)
// Skip entirely if something exists, assume layer is present already
if _, err := os.Stat(blobPath); err == nil {
return nil
}
w, err := os.Create(blobPath)
if err != nil {
return err
}
defer w.Close()
_, err = io.Copy(w, r)
return err
}

105
pkg/store/store_test.go Normal file
View File

@@ -0,0 +1,105 @@
package store_test
import (
"context"
"os"
"testing"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/random"
"github.com/rancherfederal/hauler/pkg/artifacts"
"github.com/rancherfederal/hauler/pkg/store"
)
var (
ctx context.Context
root string
)
func TestLayout_AddOCI(t *testing.T) {
teardown := setup(t)
defer teardown()
type args struct {
ref string
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "",
args: args{
ref: "hello/world:v1",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, err := store.NewLayout(root)
if (err != nil) != tt.wantErr {
t.Errorf("NewOCI() error = %v, wantErr %v", err, tt.wantErr)
return
}
moci := genArtifact(t, tt.args.ref)
got, err := s.AddOCI(ctx, moci, tt.args.ref)
if (err != nil) != tt.wantErr {
t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr)
return
}
_ = got
_, err = s.AddOCI(ctx, moci, tt.args.ref)
if err != nil {
t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr)
return
}
})
}
}
func setup(t *testing.T) func() error {
tmpdir, err := os.MkdirTemp("", "hauler")
if err != nil {
t.Fatal(err)
}
root = tmpdir
ctx = context.Background()
return func() error {
os.RemoveAll(tmpdir)
return nil
}
}
type mockArtifact struct {
v1.Image
}
func (m mockArtifact) MediaType() string {
mt, err := m.Image.MediaType()
if err != nil {
return ""
}
return string(mt)
}
func (m mockArtifact) RawConfig() ([]byte, error) {
return m.RawConfigFile()
}
func genArtifact(t *testing.T, ref string) artifacts.OCI {
img, err := random.Image(1024, 3)
if err != nil {
t.Fatal(err)
}
return &mockArtifact{
img,
}
}