mirror of
https://github.com/replicatedhq/troubleshoot.git
synced 2026-02-14 10:19:54 +00:00
9343b43e77
* fix(ci): regression test updates binary to latest release * fix cluster resources collector
464 lines
15 KiB
YAML
464 lines
15 KiB
YAML
name: Regression Test Suite
|
|
|
|
on:
|
|
push:
|
|
branches: [main, v1beta3]
|
|
pull_request:
|
|
types: [opened, synchronize, reopened]
|
|
workflow_dispatch:
|
|
inputs:
|
|
update_baselines:
|
|
description: 'Update baselines after run (use with caution)'
|
|
type: boolean
|
|
default: false
|
|
|
|
jobs:
|
|
# Build binaries once (shared by all test jobs)
|
|
build-binaries:
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 5
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
|
|
- name: Build binaries
|
|
run: |
|
|
echo "Building preflight and support-bundle binaries..."
|
|
make bin/preflight bin/support-bundle
|
|
./bin/preflight version
|
|
./bin/support-bundle version
|
|
|
|
- name: Upload binaries
|
|
uses: actions/upload-artifact@v5
|
|
with:
|
|
name: binaries-${{ github.run_id }}
|
|
path: |
|
|
bin/preflight
|
|
bin/support-bundle
|
|
retention-days: 1
|
|
|
|
# Preflight v1beta3 test (parallel job 1)
|
|
test-preflight-v1beta3:
|
|
needs: [build-binaries]
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Create output directory
|
|
run: mkdir -p test/output
|
|
|
|
- name: Create k3s cluster
|
|
uses: replicatedhq/action-k3s@main
|
|
with:
|
|
version: v1.31.2-k3s1
|
|
|
|
- name: Verify cluster access
|
|
run: kubectl get nodes -o wide
|
|
|
|
- name: Wait for all pods to be ready
|
|
run: |
|
|
echo "Waiting for all pods to be running..."
|
|
kubectl get pods --all-namespaces
|
|
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
|
|
kubectl get pods --all-namespaces
|
|
|
|
- name: Download binaries
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: binaries-${{ github.run_id }}
|
|
path: bin/
|
|
|
|
- name: Make binaries executable
|
|
run: |
|
|
chmod +x bin/preflight bin/support-bundle
|
|
./bin/preflight version
|
|
|
|
- name: Setup Python for comparison
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install Python dependencies
|
|
run: pip install pyyaml deepdiff
|
|
|
|
- name: Run preflight v1beta3
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Running preflight v1beta3..."
|
|
./bin/preflight \
|
|
examples/preflight/complex-v1beta3.yaml \
|
|
--values examples/preflight/values-complex-full.yaml \
|
|
--interactive=false \
|
|
--format=json \
|
|
--auto-update=false \
|
|
--output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true
|
|
|
|
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
|
|
if [ -n "$BUNDLE" ]; then
|
|
mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz
|
|
echo "✓ v1beta3 bundle saved"
|
|
fi
|
|
|
|
- name: Compare preflight v1beta3 bundle
|
|
id: compare
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Comparing v1beta3 preflight bundle against baseline..."
|
|
if [ ! -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then
|
|
echo "⚠ No baseline found for v1beta3 - skipping comparison"
|
|
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
python3 scripts/compare_bundles.py \
|
|
--baseline test/baselines/preflight-v1beta3/baseline.tar.gz \
|
|
--current test/output/preflight-v1beta3-bundle.tar.gz \
|
|
--rules scripts/compare_rules.yaml \
|
|
--report test/output/diff-report-v1beta3.json \
|
|
--spec-type preflight
|
|
|
|
- name: Upload test artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v5
|
|
with:
|
|
name: test-results-v1beta3-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
test/output/preflight-v1beta3-bundle.tar.gz
|
|
test/output/preflight-results-v1beta3.json
|
|
test/output/diff-report-v1beta3.json
|
|
test/output/v1beta3.log
|
|
retention-days: 30
|
|
|
|
- name: Set job outcome
|
|
if: always()
|
|
run: |
|
|
if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
|
|
echo "comparison_failed=true" >> $GITHUB_OUTPUT
|
|
exit 1
|
|
fi
|
|
|
|
# Preflight v1beta2 test (parallel job 2)
|
|
test-preflight-v1beta2:
|
|
needs: [build-binaries]
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Create output directory
|
|
run: mkdir -p test/output
|
|
|
|
- name: Create k3s cluster
|
|
uses: replicatedhq/action-k3s@main
|
|
with:
|
|
version: v1.31.2-k3s1
|
|
|
|
- name: Verify cluster access
|
|
run: kubectl get nodes -o wide
|
|
|
|
- name: Wait for all pods to be ready
|
|
run: |
|
|
echo "Waiting for all pods to be running..."
|
|
kubectl get pods --all-namespaces
|
|
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
|
|
kubectl get pods --all-namespaces
|
|
|
|
- name: Download binaries
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: binaries-${{ github.run_id }}
|
|
path: bin/
|
|
|
|
- name: Make binaries executable
|
|
run: |
|
|
chmod +x bin/preflight bin/support-bundle
|
|
./bin/preflight version
|
|
|
|
- name: Setup Python for comparison
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install Python dependencies
|
|
run: pip install pyyaml deepdiff
|
|
|
|
- name: Run preflight v1beta2
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Running preflight v1beta2..."
|
|
./bin/preflight \
|
|
examples/preflight/all-analyzers-v1beta2.yaml \
|
|
--interactive=false \
|
|
--format=json \
|
|
--auto-update=false \
|
|
--output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true
|
|
|
|
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
|
|
if [ -n "$BUNDLE" ]; then
|
|
mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz
|
|
echo "✓ v1beta2 bundle saved"
|
|
fi
|
|
|
|
- name: Compare preflight v1beta2 bundle
|
|
id: compare
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Comparing v1beta2 preflight bundle against baseline..."
|
|
if [ ! -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then
|
|
echo "⚠ No baseline found for v1beta2 - skipping comparison"
|
|
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
python3 scripts/compare_bundles.py \
|
|
--baseline test/baselines/preflight-v1beta2/baseline.tar.gz \
|
|
--current test/output/preflight-v1beta2-bundle.tar.gz \
|
|
--rules scripts/compare_rules.yaml \
|
|
--report test/output/diff-report-v1beta2.json \
|
|
--spec-type preflight
|
|
|
|
- name: Upload test artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v5
|
|
with:
|
|
name: test-results-v1beta2-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
test/output/preflight-v1beta2-bundle.tar.gz
|
|
test/output/preflight-results-v1beta2.json
|
|
test/output/diff-report-v1beta2.json
|
|
test/output/v1beta2.log
|
|
retention-days: 30
|
|
|
|
- name: Set job outcome
|
|
if: always()
|
|
run: |
|
|
if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
|
|
echo "comparison_failed=true" >> $GITHUB_OUTPUT
|
|
exit 1
|
|
fi
|
|
|
|
# Support bundle test (parallel job 3)
|
|
test-supportbundle:
|
|
needs: [build-binaries]
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Create output directory
|
|
run: mkdir -p test/output
|
|
|
|
- name: Create k3s cluster
|
|
uses: replicatedhq/action-k3s@main
|
|
with:
|
|
version: v1.31.2-k3s1
|
|
|
|
- name: Verify cluster access
|
|
run: kubectl get nodes -o wide
|
|
|
|
- name: Wait for all pods to be ready
|
|
run: |
|
|
echo "Waiting for all pods to be running..."
|
|
kubectl get pods --all-namespaces
|
|
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
|
|
kubectl get pods --all-namespaces
|
|
|
|
- name: Download binaries
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: binaries-${{ github.run_id }}
|
|
path: bin/
|
|
|
|
- name: Make binaries executable
|
|
run: |
|
|
chmod +x bin/preflight bin/support-bundle
|
|
./bin/support-bundle version
|
|
|
|
- name: Setup Python for comparison
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install Python dependencies
|
|
run: pip install pyyaml deepdiff
|
|
|
|
- name: Run support bundle
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Running support bundle..."
|
|
./bin/support-bundle \
|
|
examples/collect/host/all-kubernetes-collectors.yaml \
|
|
--interactive=false \
|
|
--auto-update=false \
|
|
--output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true
|
|
|
|
if [ -f test/output/supportbundle.tar.gz ]; then
|
|
echo "✓ Support bundle saved"
|
|
fi
|
|
|
|
- name: Compare support bundle
|
|
id: compare
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Comparing support bundle against baseline..."
|
|
if [ ! -f test/baselines/supportbundle/baseline.tar.gz ]; then
|
|
echo "⚠ No baseline found for support bundle - skipping comparison"
|
|
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
python3 scripts/compare_bundles.py \
|
|
--baseline test/baselines/supportbundle/baseline.tar.gz \
|
|
--current test/output/supportbundle.tar.gz \
|
|
--rules scripts/compare_rules.yaml \
|
|
--report test/output/diff-report-supportbundle.json \
|
|
--spec-type supportbundle
|
|
|
|
- name: Upload test artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v5
|
|
with:
|
|
name: test-results-supportbundle-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
test/output/supportbundle.tar.gz
|
|
test/output/diff-report-supportbundle.json
|
|
test/output/supportbundle.log
|
|
retention-days: 30
|
|
|
|
- name: Set job outcome
|
|
if: always()
|
|
run: |
|
|
if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
|
|
echo "comparison_failed=true" >> $GITHUB_OUTPUT
|
|
exit 1
|
|
fi
|
|
|
|
# Report results (runs after all tests complete)
|
|
report-results:
|
|
needs: [test-preflight-v1beta3, test-preflight-v1beta2, test-supportbundle]
|
|
if: always() && github.actor != 'dependabot[bot]'
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 5
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install Python dependencies
|
|
run: pip install pyyaml deepdiff
|
|
|
|
- name: Download all test artifacts
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
path: test/output
|
|
pattern: test-results-*
|
|
|
|
- name: Reorganize artifacts
|
|
run: |
|
|
# Move artifacts from nested directories to test/output
|
|
find test/output/test-results-* -type f -exec mv {} test/output/ \;
|
|
# Clean up empty directories
|
|
find test/output/test-results-* -type d -empty -delete || true
|
|
|
|
- name: Generate summary report
|
|
run: |
|
|
python3 scripts/generate_summary.py \
|
|
--reports test/output/diff-report-*.json \
|
|
--output-file $GITHUB_STEP_SUMMARY \
|
|
--output-console
|
|
|
|
- name: Check for regressions
|
|
run: |
|
|
echo "Checking comparison results..."
|
|
|
|
FAILURES=0
|
|
|
|
if [ "${{ needs.test-preflight-v1beta3.result }}" == "failure" ]; then
|
|
echo "❌ v1beta3 comparison failed"
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
|
|
if [ "${{ needs.test-preflight-v1beta2.result }}" == "failure" ]; then
|
|
echo "❌ v1beta2 comparison failed"
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
|
|
if [ "${{ needs.test-supportbundle.result }}" == "failure" ]; then
|
|
echo "❌ Support bundle comparison failed"
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
|
|
if [ $FAILURES -gt 0 ]; then
|
|
echo ""
|
|
echo "❌ $FAILURES regression(s) detected!"
|
|
echo "Review the comparison reports in the artifacts."
|
|
exit 1
|
|
else
|
|
echo "✅ All comparisons passed or skipped (no baseline)"
|
|
fi
|
|
|
|
- name: Update baselines
|
|
if: github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch'
|
|
run: |
|
|
echo "Updating baselines with current bundles..."
|
|
|
|
# Copy new bundles as baselines
|
|
if [ -f test/output/preflight-v1beta3-bundle.tar.gz ]; then
|
|
mkdir -p test/baselines/preflight-v1beta3
|
|
cp test/output/preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz
|
|
echo "✓ Updated v1beta3 baseline"
|
|
fi
|
|
|
|
if [ -f test/output/preflight-v1beta2-bundle.tar.gz ]; then
|
|
mkdir -p test/baselines/preflight-v1beta2
|
|
cp test/output/preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz
|
|
echo "✓ Updated v1beta2 baseline"
|
|
fi
|
|
|
|
if [ -f test/output/supportbundle.tar.gz ]; then
|
|
mkdir -p test/baselines/supportbundle
|
|
cp test/output/supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz
|
|
echo "✓ Updated support bundle baseline"
|
|
fi
|
|
|
|
# Create metadata file
|
|
cat > test/baselines/metadata.json <<EOF
|
|
{
|
|
"updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"git_sha": "${{ github.sha }}",
|
|
"k8s_version": "v1.31.2-k3s1",
|
|
"workflow_run": "${{ github.run_id }}"
|
|
}
|
|
EOF
|
|
|
|
# Commit and push
|
|
git config user.name "github-actions[bot]"
|
|
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
git add test/baselines/
|
|
git commit -m "chore: update regression test baselines from run ${{ github.run_id }}"
|
|
git push
|