From 2a2b0e0f301465e9a9e2661e54e397e7720d60d3 Mon Sep 17 00:00:00 2001 From: Noah Campbell Date: Tue, 14 Oct 2025 11:30:05 -0500 Subject: [PATCH] reverted regression tests to how they were before This was turning into something worthy of its own PR so I put it back to how it was before to work on next/separately --- .github/workflows/regression-test.yaml | 357 ++++++++++++++++--------- 1 file changed, 226 insertions(+), 131 deletions(-) diff --git a/.github/workflows/regression-test.yaml b/.github/workflows/regression-test.yaml index 893559e8..991c803d 100644 --- a/.github/workflows/regression-test.yaml +++ b/.github/workflows/regression-test.yaml @@ -4,7 +4,7 @@ on: push: branches: [main] pull_request: - types: [opened, reopened, synchronize, ready_for_review] + types: [opened, synchronize, reopened] workflow_dispatch: inputs: update_baselines: @@ -13,16 +13,22 @@ on: default: false jobs: - create-cluster: - # Create a shared cluster for the matrix; skip on fork PRs due to secrets - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false + regression-test: runs-on: ubuntu-22.04 - outputs: - kubeconfig: ${{ steps.create.outputs.cluster-kubeconfig }} - cluster-id: ${{ steps.create.outputs.cluster-id }} + timeout-minutes: 25 + steps: + # 1. SETUP + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for git describe to work + + - name: Create output directory + run: mkdir -p test/output + - name: Create k3s cluster - id: create + id: create-cluster uses: replicatedhq/compatibility-actions/create-cluster@v1 with: api-token: ${{ secrets.REPLICATED_API_TOKEN }} @@ -31,21 +37,13 @@ jobs: ttl: 25m timeout-minutes: 5 - regression: - # Run on push, manual dispatch, and internal PRs (skip fork PRs) - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false - needs: create-cluster - runs-on: ubuntu-22.04 - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - suite: [preflight-v1beta3, preflight-v1beta2, support-bundle] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - name: Configure kubeconfig + run: | + echo "${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > $GITHUB_WORKSPACE/kubeconfig.yaml + echo "KUBECONFIG=$GITHUB_WORKSPACE/kubeconfig.yaml" >> $GITHUB_ENV + + - name: Verify cluster access + run: kubectl get nodes -o wide - name: Setup Go uses: actions/setup-go@v5 @@ -54,6 +52,13 @@ jobs: cache: true cache-dependency-path: go.sum + - name: Build binaries + run: | + echo "Building preflight and support-bundle binaries..." + make bin/preflight bin/support-bundle + ./bin/preflight version + ./bin/support-bundle version + - name: Setup Python for comparison uses: actions/setup-python@v5 with: @@ -63,135 +68,225 @@ jobs: run: | pip install pyyaml deepdiff - - name: Configure kubeconfig + # 2. EXECUTE SPECS (in parallel) + - name: Run all specs in parallel + continue-on-error: true run: | - echo "${{ needs.create-cluster.outputs.kubeconfig }}" > $GITHUB_WORKSPACE/kubeconfig.yaml - echo "KUBECONFIG=$GITHUB_WORKSPACE/kubeconfig.yaml" >> $GITHUB_ENV + echo "Running all 3 specs in parallel..." - - name: Verify cluster access - run: kubectl get nodes -o wide + # Run v1beta3 in background + ( + echo "Starting preflight v1beta3..." + ./bin/preflight \ + examples/preflight/complex-v1beta3.yaml \ + --values examples/preflight/values-complex-full.yaml \ + --interactive=false \ + --format=json \ + --output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true - - name: Prepare namespace (preflight only) - if: startsWith(matrix.suite, 'preflight-') + BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) + if [ -n "$BUNDLE" ]; then + mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz + echo "✓ v1beta3 bundle saved" + fi + ) & + PID_V1BETA3=$! + + # Run v1beta2 in background + ( + echo "Starting preflight v1beta2..." + ./bin/preflight \ + examples/preflight/all-analyzers-v1beta2.yaml \ + --interactive=false \ + --format=json \ + --output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true + + BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) + if [ -n "$BUNDLE" ]; then + mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz + echo "✓ v1beta2 bundle saved" + fi + ) & + PID_V1BETA2=$! + + # Run support bundle in background + ( + echo "Starting support bundle..." + ./bin/support-bundle \ + examples/collect/host/all-kubernetes-collectors.yaml \ + --interactive=false \ + --output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true + + if [ -f test/output/supportbundle.tar.gz ]; then + echo "✓ Support bundle saved" + fi + ) & + PID_SUPPORTBUNDLE=$! + + # Wait for all to complete + echo "Waiting for all specs to complete..." + wait $PID_V1BETA3 + wait $PID_V1BETA2 + wait $PID_SUPPORTBUNDLE + + echo "All specs completed!" + + # Verify bundles exist + ls -lh test/output/*.tar.gz || echo "Warning: Some bundles may be missing" + + # 3. COMPARE BUNDLES + - name: Compare preflight v1beta3 bundle + id: compare-v1beta3 + continue-on-error: true run: | - ns="ts-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}" - echo "E2E_NS=$ns" >> $GITHUB_ENV - kubectl create namespace "$ns" + echo "Comparing v1beta3 preflight bundle against baseline..." + if [ ! -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then + echo "⚠ No baseline found for v1beta3 - skipping comparison" + echo "baseline_missing=true" >> $GITHUB_OUTPUT + exit 0 + fi - - name: Build binary for suite + python3 scripts/compare_bundles.py \ + --baseline test/baselines/preflight-v1beta3/baseline.tar.gz \ + --current test/output/preflight-v1beta3-bundle.tar.gz \ + --rules scripts/compare_rules.yaml \ + --report test/output/diff-report-v1beta3.json \ + --spec-type preflight + + - name: Compare preflight v1beta2 bundle + id: compare-v1beta2 + continue-on-error: true run: | - case "${{ matrix.suite }}" in - preflight-*) - make bin/preflight - ./bin/preflight version - ;; - support-bundle) - make bin/support-bundle - ./bin/support-bundle version - ;; - esac + echo "Comparing v1beta2 preflight bundle against baseline..." + if [ ! -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then + echo "⚠ No baseline found for v1beta2 - skipping comparison" + echo "baseline_missing=true" >> $GITHUB_OUTPUT + exit 0 + fi - - name: Run suite + python3 scripts/compare_bundles.py \ + --baseline test/baselines/preflight-v1beta2/baseline.tar.gz \ + --current test/output/preflight-v1beta2-bundle.tar.gz \ + --rules scripts/compare_rules.yaml \ + --report test/output/diff-report-v1beta2.json \ + --spec-type preflight + + - name: Compare support bundle + id: compare-supportbundle + continue-on-error: true run: | - mkdir -p test/output - case "${{ matrix.suite }}" in - preflight-v1beta3) - ./bin/preflight \ - examples/preflight/complex-v1beta3.yaml \ - --values examples/preflight/values-complex-full.yaml \ - --interactive=false \ - --format=json \ - --auto-update=false \ - --kubeconfig=$GITHUB_WORKSPACE/kubeconfig.yaml \ - -n "$E2E_NS" \ - --output=test/output/preflight-results-v1beta3.json - BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) - if [ -n "$BUNDLE" ]; then - mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz - fi - ;; - preflight-v1beta2) - ./bin/preflight \ - examples/preflight/all-analyzers-v1beta2.yaml \ - --interactive=false \ - --format=json \ - --auto-update=false \ - --kubeconfig=$GITHUB_WORKSPACE/kubeconfig.yaml \ - -n "$E2E_NS" \ - --output=test/output/preflight-results-v1beta2.json - BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) - if [ -n "$BUNDLE" ]; then - mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz - fi - ;; - support-bundle) - ./bin/support-bundle \ - examples/collect/host/all-kubernetes-collectors.yaml \ - --interactive=false \ - --kubeconfig=$GITHUB_WORKSPACE/kubeconfig.yaml \ - --output=test/output/supportbundle.tar.gz - ;; - esac + echo "Comparing support bundle against baseline..." + if [ ! -f test/baselines/supportbundle/baseline.tar.gz ]; then + echo "⚠ No baseline found for support bundle - skipping comparison" + echo "baseline_missing=true" >> $GITHUB_OUTPUT + exit 0 + fi - - name: Compare against baselines + python3 scripts/compare_bundles.py \ + --baseline test/baselines/supportbundle/baseline.tar.gz \ + --current test/output/supportbundle.tar.gz \ + --rules scripts/compare_rules.yaml \ + --report test/output/diff-report-supportbundle.json \ + --spec-type supportbundle + + # 4. REPORT RESULTS + - name: Generate summary report + if: always() run: | - case "${{ matrix.suite }}" in - preflight-v1beta3) - if [ -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then - python3 scripts/compare_bundles.py \ - --baseline test/baselines/preflight-v1beta3/baseline.tar.gz \ - --current test/output/preflight-v1beta3-bundle.tar.gz \ - --rules scripts/compare_rules.yaml \ - --report test/output/diff-report-v1beta3.json \ - --spec-type preflight - else - echo "No baseline for v1beta3; skipping comparison" - fi - ;; - preflight-v1beta2) - if [ -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then - python3 scripts/compare_bundles.py \ - --baseline test/baselines/preflight-v1beta2/baseline.tar.gz \ - --current test/output/preflight-v1beta2-bundle.tar.gz \ - --rules scripts/compare_rules.yaml \ - --report test/output/diff-report-v1beta2.json \ - --spec-type preflight - else - echo "No baseline for v1beta2; skipping comparison" - fi - ;; - support-bundle) - if [ -f test/baselines/supportbundle/baseline.tar.gz ]; then - python3 scripts/compare_bundles.py \ - --baseline test/baselines/supportbundle/baseline.tar.gz \ - --current test/output/supportbundle.tar.gz \ - --rules scripts/compare_rules.yaml \ - --report test/output/diff-report-supportbundle.json \ - --spec-type supportbundle - else - echo "No baseline for support bundle; skipping comparison" - fi - ;; - esac + python3 scripts/generate_summary.py \ + --reports test/output/diff-report-*.json \ + --output-file $GITHUB_STEP_SUMMARY \ + --output-console - - name: Upload artifacts + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 with: - name: regression-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }} + name: regression-test-results-${{ github.run_id }}-${{ github.run_attempt }} path: | test/output/*.tar.gz test/output/*.json retention-days: 30 - cleanup: - if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) - needs: [create-cluster, regression] - runs-on: ubuntu-22.04 - steps: + - name: Check for regressions + if: always() + run: | + echo "Checking comparison results..." + + # Check if any comparisons failed + FAILURES=0 + + if [ "${{ steps.compare-v1beta3.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta3.outputs.baseline_missing }}" != "true" ]; then + echo "❌ v1beta3 comparison failed" + FAILURES=$((FAILURES + 1)) + fi + + if [ "${{ steps.compare-v1beta2.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta2.outputs.baseline_missing }}" != "true" ]; then + echo "❌ v1beta2 comparison failed" + FAILURES=$((FAILURES + 1)) + fi + + if [ "${{ steps.compare-supportbundle.outcome }}" == "failure" ] && [ "${{ steps.compare-supportbundle.outputs.baseline_missing }}" != "true" ]; then + echo "❌ Support bundle comparison failed" + FAILURES=$((FAILURES + 1)) + fi + + if [ $FAILURES -gt 0 ]; then + echo "" + echo "❌ $FAILURES regression(s) detected!" + echo "Review the comparison reports in the artifacts." + exit 1 + else + echo "✅ All comparisons passed or skipped (no baseline)" + fi + + # 5. UPDATE BASELINES (optional, manual trigger only) + - name: Update baselines + if: github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch' + run: | + echo "Updating baselines with current bundles..." + + # Copy new bundles as baselines + if [ -f test/output/preflight-v1beta3-bundle.tar.gz ]; then + mkdir -p test/baselines/preflight-v1beta3 + cp test/output/preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz + echo "✓ Updated v1beta3 baseline" + fi + + if [ -f test/output/preflight-v1beta2-bundle.tar.gz ]; then + mkdir -p test/baselines/preflight-v1beta2 + cp test/output/preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz + echo "✓ Updated v1beta2 baseline" + fi + + if [ -f test/output/supportbundle.tar.gz ]; then + mkdir -p test/baselines/supportbundle + cp test/output/supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz + echo "✓ Updated support bundle baseline" + fi + + # Create metadata file + cat > test/baselines/metadata.json <