feat: v1.9.0 regression tests (#696)

Add regression tests for BYOB releae.

---------

Signed-off-by: laurentsimon <laurentsimon@google.com>
This commit is contained in:
laurentsimon
2023-08-24 09:20:57 -07:00
committed by GitHub
parent 58eede7e66
commit 80c7d86183
72 changed files with 418 additions and 9 deletions

View File

@@ -36,7 +36,7 @@ func pString(s string) *string {
const TEST_DIR = "./testdata"
var (
GHA_ARTIFACT_PATH_BUILDERS = []string{"gha_go", "gha_generic"}
GHA_ARTIFACT_PATH_BUILDERS = []string{"gha_go", "gha_generic", "gha_delegator", "gha_maven", "gha_gradle"}
// TODO(https://github.com/slsa-framework/slsa-verifier/issues/485): Merge this with
// GHA_ARTIFACT_PATH_BUILDERS.
GHA_ARTIFACT_CONTAINER_BUILDERS = []string{"gha_container-based"}
@@ -80,6 +80,9 @@ func Test_runVerifyGHAArtifactPath(t *testing.T) {
t.Parallel()
goBuilder := "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/builder_go_slsa3.yml"
genericBuilder := "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml"
delegatorBuilder := "https://github.com/slsa-framework/example-trw/.github/workflows/builder_high-perms_slsa3.yml"
mavenBuilder := "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/builder_maven_slsa3.yml"
gradleBuilder := "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/builder_gradle_slsa3.yml"
tests := []struct {
name string
@@ -532,9 +535,15 @@ func Test_runVerifyGHAArtifactPath(t *testing.T) {
for _, v := range checkVersions {
var provenancePath string
var byob bool
if tt.provenancePath == "" {
testPath := filepath.Clean(filepath.Join(TEST_DIR, v, tt.artifacts[0]))
provenancePath = fmt.Sprintf("%s.intoto.jsonl", testPath)
if strings.Contains(testPath, "delegator") || strings.Contains(testPath, "maven") || strings.Contains(testPath, "gradle") {
provenancePath = fmt.Sprintf("%s.build.slsa", testPath)
byob = true
} else {
provenancePath = fmt.Sprintf("%s.intoto.jsonl", testPath)
}
} else {
provenancePath = filepath.Clean(filepath.Join(TEST_DIR, v, tt.provenancePath))
}
@@ -564,6 +573,12 @@ func Test_runVerifyGHAArtifactPath(t *testing.T) {
builder = goBuilder
case strings.HasSuffix(name, "_generic"):
builder = genericBuilder
case strings.HasSuffix(name, "_delegator"):
builder = delegatorBuilder
case strings.HasSuffix(name, "_maven"):
builder = mavenBuilder
case strings.HasSuffix(name, "_gradle"):
builder = gradleBuilder
default:
builder = genericBuilder
}
@@ -571,7 +586,12 @@ func Test_runVerifyGHAArtifactPath(t *testing.T) {
// Default builders to test.
builderIDs := []*string{
pString(builder),
nil,
}
// Do not run without explicit builder ID for the delegator,
// because it's hosted on a different repo slsa-framework/example-package.
if builder != delegatorBuilder {
builderIDs = append(builderIDs, nil)
}
// We only add the tags to tests for versions >= 1,
@@ -600,6 +620,10 @@ func Test_runVerifyGHAArtifactPath(t *testing.T) {
BuildWorkflowInputs: tt.inputs,
}
// BYOB-based builders ignore the reusable workflow.
if errCmp(tt.err, serrors.ErrorUntrustedReusableWorkflow) && byob {
tt.err = serrors.ErrorMismatchBuilderID
}
// The outBuilderID is the actual builder ID from the provenance.
// This is always long form for the GHA builders.
outBuilderID, err := cmd.Exec(context.Background(), artifacts)
@@ -699,6 +723,10 @@ func Test_runVerifyGHAArtifactImage(t *testing.T) {
// or testdata from malicious untrusted builders.
// When true, this does not iterate over all builder versions.
noversion bool
// minversion is a special case to test a newly added feature into a builder.
minversion string
// maxversion is a special case to handle incompatible error changes in the builder.
maxversion string
}{
{
name: "valid main branch default",
@@ -718,7 +746,6 @@ func Test_runVerifyGHAArtifactImage(t *testing.T) {
source: "github.com/slsa-framework/example-package",
pbranch: pString("main"),
},
{
name: "wrong branch master",
artifact: "container_workflow_dispatch",
@@ -745,19 +772,37 @@ func Test_runVerifyGHAArtifactImage(t *testing.T) {
err: serrors.ErrorMismatchSource,
},
{
name: "tag no match empty tag workflow_dispatch",
artifact: "container_workflow_dispatch",
source: "github.com/slsa-framework/example-package",
ptag: pString("v1.2.3"),
err: serrors.ErrorInvalidRef,
name: "tag no match empty tag workflow_dispatch",
artifact: "container_workflow_dispatch",
source: "github.com/slsa-framework/example-package",
ptag: pString("v1.2.3"),
maxversion: "v1.8.0",
err: serrors.ErrorInvalidRef,
},
{
name: "versioned tag no match empty tag workflow_dispatch",
artifact: "container_workflow_dispatch",
source: "github.com/slsa-framework/example-package",
pversiontag: pString("v1"),
maxversion: "v1.8.0",
err: serrors.ErrorInvalidRef,
},
{
name: "tag no match empty tag workflow_dispatch > v1.9.0",
artifact: "container_workflow_dispatch",
source: "github.com/slsa-framework/example-package",
ptag: pString("v1.2.3"),
minversion: "v1.9.0",
err: serrors.ErrorMismatchTag,
},
{
name: "versioned tag no match empty tag workflow_dispatch > v1.9.0",
artifact: "container_workflow_dispatch",
source: "github.com/slsa-framework/example-package",
pversiontag: pString("v1"),
minversion: "v1.9.0",
err: serrors.ErrorMismatchTag,
},
}
for _, tt := range tests {
tt := tt // Re-initializing variable so it is not changed while executing the closure below
@@ -770,6 +815,19 @@ func Test_runVerifyGHAArtifactImage(t *testing.T) {
}
for _, v := range checkVersions {
parts := strings.Split(v, "/")
version := ""
if len(parts) > 1 {
version = parts[1]
}
if version != "" && tt.minversion != "" && semver.Compare(version, tt.minversion) <= 0 {
fmt.Println("skiping due to min:", version)
continue
}
if version != "" && tt.maxversion != "" && semver.Compare(version, tt.maxversion) > 0 {
fmt.Println("skiping due to max:", version)
continue
}
image := filepath.Clean(filepath.Join(TEST_DIR, v, tt.artifact))
// TODO(#258): test for tagged builder.
sv := filepath.Base(v)

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
hello

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
hello

View File

@@ -0,0 +1 @@
hello

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
hello

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
sha256:5b146084201148bf151871fa779767f013c47706d031106af0d727fb5b88f5ac

View File

@@ -0,0 +1,19 @@
{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"digest": "sha256:9e0ead4f1d9a7486c64047ec5aa74cfd7ca8892e4319e8b985cadf4163fe44b8",
"size": 167
},
"layers": [
{
"mediaType": "application/vnd.in-toto+json",
"digest": "sha256:10dc279526ec7dec3af88852fd3b0db3e130792b2e2f23f7a7db99285e792b98",
"size": 14271,
"annotations": {
"in-toto.io/predicate-type": "https://slsa.dev/provenance/v0.2"
}
}
]
}

View File

@@ -0,0 +1,28 @@
{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.index.v1+json",
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"digest": "sha256:7606c85a2e0bc3af20a2e0de046f4c97c275ba689a60fcc0140186acf852ffd3",
"size": 2183,
"platform": {
"architecture": "amd64",
"os": "linux"
}
},
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"digest": "sha256:280a265e62bd3fb7b54f60712b585eec918bd747ad31dd9d96c02014f5329842",
"size": 567,
"annotations": {
"vnd.docker.reference.digest": "sha256:7606c85a2e0bc3af20a2e0de046f4c97c275ba689a60fcc0140186acf852ffd3",
"vnd.docker.reference.type": "attestation-manifest"
},
"platform": {
"architecture": "unknown",
"os": "unknown"
}
}
]
}

View File

@@ -0,0 +1,61 @@
{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"digest": "sha256:a067ba5dae3a4c56003cf3fa9aef6c9d6618c92f585a2e112aa1cbc4d7efcd19",
"size": 2160
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639",
"size": 103732
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58",
"size": 21202
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db",
"size": 716491
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265",
"size": 317
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0",
"size": 198
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c",
"size": 113
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f",
"size": 385
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c",
"size": 355
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a",
"size": 130562
},
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"digest": "sha256:fae15c0b6b42691083a94fd78ce64bd27c6378539d8dd3fb383cf024e1aac260",
"size": 648815
}
]
}

View File

@@ -0,0 +1 @@
{"architecture":"","created":"2023-08-23T05:22:09.250238557Z","history":[{"created":"0001-01-01T00:00:00Z"}],"os":"","rootfs":{"type":"layers","diff_ids":["sha256:a5e176610c4305f4068a2d2e2884c5d57b3e28d46ccbb825a85212dc09010814"]},"config":{}}

View File

@@ -0,0 +1 @@
{"architecture":"unknown","os":"unknown","config":{},"rootfs":{"type":"layers","diff_ids":["sha256:10dc279526ec7dec3af88852fd3b0db3e130792b2e2f23f7a7db99285e792b98"]}}

View File

@@ -0,0 +1 @@
{"architecture":"amd64","config":{"User":"0","Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"],"Entrypoint":["/app"],"WorkingDir":"/","Labels":{"org.opencontainers.image.created":"2023-08-23T05:20:21.969Z","org.opencontainers.image.description":"","org.opencontainers.image.licenses":"Apache-2.0","org.opencontainers.image.revision":"2bcaa7495e1cbd11fbd4f598d857b3a6f18df933","org.opencontainers.image.source":"https://github.com/slsa-framework/example-package","org.opencontainers.image.title":"example-package","org.opencontainers.image.url":"https://github.com/slsa-framework/example-package","org.opencontainers.image.version":"v13.0.30"},"OnBuild":null},"created":"2023-08-23T05:20:53.897507136Z","history":[{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"0001-01-01T00:00:00Z"},{"created":"2023-08-23T05:20:53.897507136Z","created_by":"COPY /app/app /app # buildkit","comment":"buildkit.dockerfile.v0"},{"created":"2023-08-23T05:20:53.897507136Z","created_by":"ENTRYPOINT [\"/app\"]","comment":"buildkit.dockerfile.v0","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:e023e0e48e6e29e90e519f4dd356d058ff2bffbd16e28b802f3b8f93aa4ccb17","sha256:6fbdf253bbc2490dcfede5bdb58ca0db63ee8aff565f6ea9f918f3bce9e2d5aa","sha256:7bea6b893187b14fc0a759fe5f8972d1292a9c0554c87cbf485f0947c26b8a05","sha256:ff5700ec54186528cbae40f54c24b1a34fb7c01527beaa1232868c16e2353f52","sha256:d52f02c6501c9c4410568f0bf6ff30d30d8290f57794c308fe36ea78393afac2","sha256:e624a5370eca2b8266e74d179326e2a8767d361db14d13edd9fb57e408731784","sha256:1a73b54f556b477f0a8b939d13c504a3b4f4db71f7a09c63afbc10acb3de5849","sha256:d2d7ec0f6756eb51cf1602c6f8ac4dd811d3d052661142e0110357bf0b581457","sha256:4cb10dd2545bd173858450b80853b850e49608260f1a0789e0d0b39edf12f500","sha256:ef80ed09dad1a86267c8587f5a446b9db6954da5040171a1e9fd129b823596ef"]}}

View File

@@ -0,0 +1,21 @@
{
"schemaVersion": 2,
"manifests": [
{
"mediaType": "application/vnd.oci.image.index.v1+json",
"size": 856,
"digest": "sha256:5b146084201148bf151871fa779767f013c47706d031106af0d727fb5b88f5ac",
"annotations": {
"kind": "dev.cosignproject.cosign/imageIndex"
}
},
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 10388,
"digest": "sha256:d86a56f046e85503c3e54a6fb12fade50c53eb39bcdddc8fd624cc71ecf3b88c",
"annotations": {
"kind": "dev.cosignproject.cosign/atts"
}
}
]
}

View File

@@ -0,0 +1,3 @@
{
"imageLayoutVersion": "1.0.0"
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

184
download-artifacts.sh Normal file
View File

@@ -0,0 +1,184 @@
#!/bin/bash
set -euo pipefail
# USAGE: mkdir -p tmp/v14 tmp/v14.2 tmp/v13.0.30 tmp/dispatch
# cd in each folder, and run `bash ../../download-artifacts.sh run_id builder_tag
# example: bash ../../download-artifacts.sh 5947345583 v1.9.0
# TODO: get the run id automatically thru GitHub APIs.
if [ "$#" -ne 2 ]; then
echo "Usage: $0 run_id version"
exit 1
fi
# Verify GH_TOKEN is set.
if [[ -z "${GH_TOKEN:-}" ]]; then
echo "GH_TOKEN is unset"
exit 1
fi
# Set the gh CLI.
if [[ -z "${GH:-}" ]]; then
GH="gh"
fi
unzip_files() {
local zip_path="$1"
local output_path="$2"
case "${zip_path}" in
# Ignore some files.
./slsa-builder-go-linux-amd64*)
echo "Ignoring ${zip_path}"
;;
# Container-based artifact and provenance.
./build-outputs-*.zip | ./slsa-outputs-*.zip)
unzip -o "${zip_path}" -d "${output_path}"
;;
# See partern marching https://stackoverflow.com/questions/4554718/how-to-use-patterns-in-a-case-statement.
./gha_*)
unzip -o "${zip_path}" -d "${output_path}"
;;
# Low-perm delegator artifact.
./*-artifacts.zip)
tmp_dir=$(mktemp -d)
unzip -o "${zip_path}" -d "${tmp_dir}"
cd "${tmp_dir}"
tar xvzf folder.tgz
cd -
cp "${tmp_dir}/artifacts/"* "${output_path}"
rm -rf "${tmp_dir}"
;;
# delegator attestations.
./*-slsa-attestations.zip)
tmp_dir=$(mktemp -d)
unzip -o "${zip_path}" -d "${tmp_dir}"
cd "${tmp_dir}"
tar xvzf folder.tgz
cd -
cp "${tmp_dir}/${zip_path%.*}/"* "${output_path}"
rm -rf "${tmp_dir}"
;;
# Maven artifacts.
./*-target.zip)
tmp_dir=$(mktemp -d)
unzip -o "${zip_path}" -d "${tmp_dir}"
cd "${tmp_dir}"
tar xvzf folder.tgz
cd -
cp "${tmp_dir}/target/test-java-project-"*.jar "${output_path}"
rm -rf "${tmp_dir}"
;;
# Gradle artifacts.
./*-build.zip)
tmp_dir=$(mktemp -d)
unzip -o "${zip_path}" -d "${tmp_dir}"
cd "${tmp_dir}"
tar xvzf folder.tgz
cd -
cp "${tmp_dir}/build/libs/workflow_dispatch-"*.jar "${output_path}"
rm -rf "${tmp_dir}"
;;
*)
echo "unexpected file path: ${zip_path}"
exit 1
;;
esac
# Cleanup
rm *sources.jar* *javadoc.jar* folder.tgz original-test-java-project* 2>/dev/null || true
rm "${zip_path}"
}
copy_files() {
local binary="$1"
local path="$2"
mkdir -p "${path}"
for fn in $(ls | grep "${binary}"); do
# The prefix is what precedes "binary-linux".
# May be 'gha_container-based-', etc.
prefix=${fn%%binary-linux*}
is_dispatch=$(echo "${fn}" | grep "dispatch" || true)
if [[ "${is_dispatch}" != "" ]]; then
cp "${fn}" "${path}/${fn#"${prefix}"}"
else
f="${fn/amd64/amd64-push}"
cp "${fn}" "${path}/${f#"${prefix}"}"
fi
done;
}
# Rename jar files and their attestations.
rename_java_files() {
local path="$1"
local name="$2"
v=$(ls | grep gha_delegator-binary-linux-amd64- | grep -v slsa | cut -d- -f5)
if [[ "${v}" == "" ]]; then
return
fi
artifact=$(ls | grep "${path}" | grep -v slsa || true)
if [[ "${artifact}" == "" ]]; then
return
fi
mv "${artifact}" "gha_${name}-binary-linux-amd64-${v}"
mv "${artifact}.build.slsa" "gha_${name}-binary-linux-amd64-${v}.build.slsa"
}
# Script inputs
run_id="$1"
version="$2"
output_path="."
repo=slsa-framework/example-package
artifacts=$($GH api \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"/repos/${repo}/actions/runs/${run_id}/artifacts" |
jq -r -c '.artifacts')
arr=$(echo "$artifacts" | jq -c '.[]')
for item in ${arr}; do
artifact_id=$(echo "${item}" | jq -r '.id')
artifact_name=$(echo "${item}" | jq -r '.name')
zip_path="${output_path}/${artifact_name}.zip"
$GH api \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"/repos/${repo}/actions/artifacts/${artifact_id}/zip" \
>"${zip_path}"
echo "Downloaded ${zip_path}"
unzip_files "${zip_path}" "${output_path}"
done
rename_java_files "test-java-project-" "maven"
rename_java_files "workflow_dispatch-" "gradle"
# Files downloaded. Now copy them
repo_path="../.."
# Go builder files.
copy_files "gha_go-binary-linux-amd64-" "${repo_path}/cli/slsa-verifier/testdata/gha_go/${version}"
# Generic generator.
copy_files "gha_generic-binary-linux-amd64-" "${repo_path}/cli/slsa-verifier/testdata/gha_generic/${version}"
# Container based.
copy_files "gha_container-based-binary-linux-amd64-" "${repo_path}/cli/slsa-verifier/testdata/gha_container-based/${version}"
# TODO: generic container
# Delegator
copy_files "gha_delegator-binary-linux-amd64-" "${repo_path}/cli/slsa-verifier/testdata/gha_delegator/${version}"
# Maven builder
copy_files "gha_maven-binary-linux-amd64-" "${repo_path}/cli/slsa-verifier/testdata/gha_maven/${version}"
# gradle builder
copy_files "gha_gradle-binary-linux-amd64-" "${repo_path}/cli/slsa-verifier/testdata/gha_gradle/${version}"