feat: scheduled tests for installer Action (#398)

* update Signed-off-by: laurentsimon <laurentsimon@google.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> * Update .github/workflows/schedule.installer.yml Co-authored-by: Ian Lewis <ianlewis@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> * Update .github/workflows/schedule.installer.yml Co-authored-by: Ian Lewis <ianlewis@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> * Update .github/workflows/schedule.installer.yml Co-authored-by: Ian Lewis <ianlewis@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> * Update .github/workflows/schedule.installer.yml Co-authored-by: Ian Lewis <ianlewis@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> * Update .github/workflows/schedule.installer.yml Co-authored-by: Ian Lewis <ianlewis@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> * Update .github/workflows/schedule.installer.yml Co-authored-by: Ian Lewis <ianlewis@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> * Update .github/workflows/schedule.installer.yml Co-authored-by: Ian Lewis <ianlewis@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> * update Signed-off-by: laurentsimon <laurentsimon@google.com> Signed-off-by: laurentsimon <laurentsimon@google.com> Signed-off-by: laurentsimon <64505099+laurentsimon@users.noreply.github.com> Co-authored-by: Ian Lewis <ianlewis@google.com>
2026-08-18 19:18:16 +00:00 · 2022-12-14 01:37:23 +00:00
parent a43888265e
commit 53b3aebdb9
5 changed files with 252 additions and 6 deletions
@@ -0,0 +1,212 @@
+name: verifier action
+on:
+  # Daily run.
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+    inputs:
+      version:
+        type: string
+        description: The version to to test for pre-release.
+        required: true
+
+permissions: read-all
+
+env:
+  GH_TOKEN: ${{ secrets.CREATE_ISSUES }}
+  ISSUE_REPOSITORY: ${{ github.repository }}
+  MINIMUM_INSTALLER_VERSION: v2.0.1
+
+jobs:
+  list-verifiers:
+    runs-on: ubuntu-latest
+    outputs:
+      # NOTE: version output is a JSON list of version numbers.
+      # https://github.blog/changelog/2020-04-15-github-actions-new-workflow-features/#new-fromjson-method-in-expressions
+      # https://docs.github.com/en/actions/learn-github-actions/expressions#fromjson
+      version: ${{ steps.generate-versions.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.1.2
+        with:
+          # NOTE: the example-package needs to be checked out in the default workspace.
+          repository: slsa-framework/example-package
+          ref: main
+
+      - name: Checkout
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.1.2
+        with:
+          path: __THIS_REPO__
+
+      - name: Generate verifier list
+        if: inputs.version == ''
+        id: generate-list
+        run: ./__THIS_REPO__/.github/workflows/scripts/schedule.actions/verifier-installer.sh
+
+      - name: Generate pre-release list
+        if: inputs.version != ''
+        id: generate-prerelease
+        env:
+          PRE_RELEASE_VERSION: ${{ inputs.version }}
+        run: echo "version=[\"$PRE_RELEASE_VERSION\"]" >> "$GITHUB_OUTPUT"
+
+      - name: Generate pre-release list
+        id: generate-versions
+        env:
+          PRE_RELEASE_VERSION: ${{ steps.generate-prerelease.outputs.version }}
+          LIST_VERSION: ${{ steps.generate-list.outputs.version }}
+        run: |
+          if [[ -n $PRE_RELEASE_VERSION ]]; then
+            echo "version=$PRE_RELEASE_VERSION" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=$LIST_VERSION" >> "$GITHUB_OUTPUT"
+          fi
+
+  verifier-run:
+    needs: list-verifiers
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ${{ fromJson(needs.list-verifiers.outputs.version) }}
+    steps:
+      - name: Debug
+        env:
+          VERSION: ${{ matrix.version }}
+        run: echo "version is '$VERSION'"
+
+      - name: Checkout this repository
+        # Skip release candidates unless specified explicitly.
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.1.2
+        with:
+          ref: ${{ matrix.version }}
+
+      # Install at tag.
+      # ==============
+      - name: Run the Action at tag
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF: ${{ matrix.version }}
+        uses: ./actions/installer
+
+      - name: Verify the version
+        env:
+          VERSION: ${{ matrix.version }}
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          version=$(slsa-verifier version 2>&1 | grep GitVersion | cut -d ':' -f2 | tr -d "[:space:]")
+          slsa-verifier version
+          echo "version: $version"
+          echo "VERSION: $VERSION"
+          # NOTE: the version reported by the slsa-verifier does not contain the leading `v`.
+          [ "$version" == "${VERSION:1}" ]
+
+      - name: Delete the binary
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          binary_path=$(which slsa-verifier)
+          echo "binary_path: $binary_path"
+          rm -rf "$binary_path"
+
+      # Install at commit sha.
+      # =====================
+      - name: Get sha1
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        id: commit
+        env:
+          VERSION: ${{ matrix.version }}
+        run: |
+          commit_sha=$(gh api -H "Accept: application/vnd.github+json" "/repos/$GITHUB_REPOSITORY/git/ref/tags/$VERSION" | jq -r '.object.sha')
+          echo "commit_sha=$commit_sha" >> "$GITHUB_OUTPUT"
+
+      - name: Run the Action at commit
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF: ${{ steps.commit.outputs.commit_sha }}
+        uses: ./actions/installer
+
+      - name: Verify the version
+        env:
+          VERSION: ${{ matrix.version }}
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          version=$(slsa-verifier version 2>&1 | grep GitVersion | cut -d ':' -f2 | tr -d "[:space:]")
+          slsa-verifier version
+          echo "version: $version"
+          echo "VERSION: $VERSION"
+          # NOTE: the version reported by the slsa-verifier does not contain the leading `v`.
+          [ "$version" == "${VERSION:1}" ]
+
+      - name: Delete the binary
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          binary_path=$(which slsa-verifier)
+          echo "binary_path: $binary_path"
+          rm -rf "$binary_path"
+
+      # Install at invalid commit.
+      # =========================
+      - name: Install invalid commit
+        id: invalid-commit
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        continue-on-error: true
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF: 55ca6286e3e4f4fba5d0448333fa99fc5a404a73
+        uses: ./actions/installer
+      - env:
+          SUCCESS: ${{ steps.invalid-commit.outcome == 'failure' }}
+        run: |
+          [ "$SUCCESS" == "true" ]
+
+      # Install at non-existent tag.
+      # =========================
+      - name: Install non-existent tag
+        id: nonexistent-tag
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        continue-on-error: true
+        env:
+          # NOTE: actions/installer checks for valid semantic version numbers.
+          SLSA_VERIFIER_CI_ACTION_REF: v100.3.5
+        uses: ./actions/installer
+      - env:
+          SUCCESS: ${{ steps.nonexistent-tag.outcome == 'failure' }}
+        run: |
+          [ "$SUCCESS" == "true" ]
+
+      # Install at empty tag.
+      # =====================
+      - name: Install empty tag
+        id: empty-tag
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        continue-on-error: true
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF:
+        uses: ./actions/installer
+      - env:
+          SUCCESS: ${{ steps.empty-tag.outcome == 'failure' }}
+        run: |
+          [ "$SUCCESS" == "true" ]
+
+  if-succeed:
+    needs: [verifier-run]
+    runs-on: ubuntu-latest
+    # We use `== 'failure'` instead of ` != 'success'` because we want to ignore skipped jobs, if there are any.
+    if: always() && inputs.version == '' && needs.verifier-run.result != 'failure'
+    steps:
+      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # tag=v2.4.0
+        with:
+          repository: slsa-framework/example-package
+          ref: main
+      - run: ./.github/workflows/scripts/e2e-report-success.sh
+
+  if-failed:
+
+    needs: [verifier-run]
+    runs-on: ubuntu-latest
+    if: always() && inputs.version == '' && needs.verifier-run.result == 'failure'
+    steps:
+      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # tag=v2.4.0
+        with:
+          repository: slsa-framework/example-package
+          ref: main
+      - run: ./.github/workflows/scripts/e2e-report-failure.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=/dev/null
+source "./.github/workflows/scripts/e2e-utils.sh"
+
+minimum_version="$MINIMUM_INSTALLER_VERSION"
+list=""
+# Check the releases.
+echo "Listing releases"
+# Note: can remove -R option.
+release_list=$(gh -R slsa-framework/slsa-verifier release list)
+while read -r line; do
+    tag=$(echo "$line" | cut -f1)
+    if version_ge "$tag" "$minimum_version"; then
+        echo " INFO: found version to test: $tag"
+        if [[ -n $list ]]; then
+            list="$list, \"$tag\""
+        else
+            list="\"$tag\""
+        fi
+    fi
+done <<<"$release_list"
+
+versions="[$list]"
+echo "version=$versions" >> "$GITHUB_OUTPUT"
@@ -41,8 +41,10 @@ Check the following:
 3. Ensure that the latest release can be installed via a `go install`.
 4. Verify that the version reported by the `version` command is correct:
 ```shell
-$ ./slsa-verifier version 2>&1 | grep GitVersion 
+$ ./slsa-verifier version 2>&1 | grep GitVersion
 ```
+5. Ensure the installer Action works by manually running the [schedule.installer.yml](https://github.com/slsa-framework/slsa-verifier/actions/workflows/pre-submit.actions.yml). 
+

 If both of these steps succeed, then move on to the [Final Release](#final-release).

@@ -117,6 +119,8 @@ $ sed -i "s/v1.0.0/v1.1.1/g" ./README.md
 4. Send a pull request with the changes. In the description, explain the steps to verify the hash update, i.e., reviewers shoud LGTM only if the provenance verification succeeds
 and the hash in the pull request matches the one computed on the binary. You can use [#slsa-framework/slsa-github-generator#113](https://github.com/slsa-framework/slsa-github-generator/pull/113) as example.

+5. Replace all version / commit references to the slsa-verifier repo with references to the newly released version [e2e.installer-action.yml](https://github.com/slsa-framework/example-package/blob/main/.github/workflows/e2e.installer-action.yml). Each reference has the comment `# UPDATE ON RELEASE`.
+
 ## Update builders

 Send a similar pull request to update the hash and version of the verifier for the workflow [slsa-framework/slsa-github-generator/blob/main/.github/workflows/builder_go_slsa3.yml#L30-L31](https://github.com/slsa-framework/slsa-github-generator/blob/main/.github/workflows/builder_go_slsa3.yml#L30-L31). Explain the steps to verify the hash. If the pull request for the verifier is already merged, you can simply point to it instead.
@@ -179,10 +179,14 @@ function run() {
        // Validate binary provenance
        try {
            const { exitCode, stdout, stderr } = yield exec.getExecOutput(`${bootstrapVerifierPath}`, [
-                "verify-artifact", downloadedBinaryPath,
-                "--provenance-path", downloadedProvenancePath,
-                "--source-uri", "github.com/slsa-framework/slsa-verifier",
-                "--source-tag", version,
+                "verify-artifact",
+                downloadedBinaryPath,
+                "--provenance-path",
+                downloadedProvenancePath,
+                "--source-uri",
+                "github.com/slsa-framework/slsa-verifier",
+                "--source-tag",
+                version,
            ]);
            if (exitCode !== 0) {
                throw new Error(`Unable to verify binary provenance. Aborting installation. stdout: ${stdout}; stderr: ${stderr}`);