mirror of
https://github.com/replicatedhq/troubleshoot.git
synced 2026-02-14 18:29:53 +00:00
da51c28767
* chore(deps): bump github.com/opencontainers/selinux Bumps [github.com/opencontainers/selinux](https://github.com/opencontainers/selinux) from 1.12.0 to 1.13.0. - [Release notes](https://github.com/opencontainers/selinux/releases) - [Commits](https://github.com/opencontainers/selinux/compare/v1.12.0...v1.13.0) --- updated-dependencies: - dependency-name: github.com/opencontainers/selinux dependency-version: 1.13.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> * Fix go vet failure in CI on Linux - Add go mod download before make vet in CI to ensure modules are available - Remove vendor directory (not needed, was causing vendoring inconsistencies) - Remove cache: false from all workflow files (not needed, enables caching) - Add replace directive for filepath-securejoin to fix containers/storage build - Clean up go.mod formatting and workflow improvements * downgrade filepath-securejoin --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Xav Paice <xav@replicated.com>
277 lines
9.7 KiB
YAML
277 lines
9.7 KiB
YAML
name: Regression Test Suite
|
|
|
|
on:
|
|
push:
|
|
branches: [main, v1beta3]
|
|
pull_request:
|
|
types: [opened, synchronize, reopened]
|
|
workflow_dispatch:
|
|
inputs:
|
|
update_baselines:
|
|
description: 'Update baselines after run (use with caution)'
|
|
type: boolean
|
|
default: false
|
|
|
|
jobs:
|
|
regression-test:
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 25
|
|
|
|
steps:
|
|
# 1. SETUP
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0 # Fetch all history for git describe to work
|
|
|
|
- name: Create output directory
|
|
run: mkdir -p test/output
|
|
|
|
- name: Create k3s cluster
|
|
id: k3s
|
|
uses: replicatedhq/action-k3s@main
|
|
with:
|
|
version: v1.31.2-k3s1
|
|
|
|
- name: Verify cluster access
|
|
run: kubectl get nodes -o wide
|
|
|
|
- name: Setup Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
|
|
- name: Build binaries
|
|
run: |
|
|
echo "Building preflight and support-bundle binaries..."
|
|
make bin/preflight bin/support-bundle
|
|
./bin/preflight version
|
|
./bin/support-bundle version
|
|
|
|
- name: Setup Python for comparison
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install Python dependencies
|
|
run: |
|
|
pip install pyyaml deepdiff
|
|
|
|
# 2. EXECUTE SPECS (in parallel)
|
|
- name: Run all specs in parallel
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Running all 3 specs in parallel..."
|
|
|
|
# Run v1beta3 in background
|
|
(
|
|
echo "Starting preflight v1beta3..."
|
|
./bin/preflight \
|
|
examples/preflight/complex-v1beta3.yaml \
|
|
--values examples/preflight/values-complex-full.yaml \
|
|
--interactive=false \
|
|
--format=json \
|
|
--output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true
|
|
|
|
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
|
|
if [ -n "$BUNDLE" ]; then
|
|
mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz
|
|
echo "✓ v1beta3 bundle saved"
|
|
fi
|
|
) &
|
|
PID_V1BETA3=$!
|
|
|
|
# Run v1beta2 in background
|
|
(
|
|
echo "Starting preflight v1beta2..."
|
|
./bin/preflight \
|
|
examples/preflight/all-analyzers-v1beta2.yaml \
|
|
--interactive=false \
|
|
--format=json \
|
|
--output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true
|
|
|
|
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
|
|
if [ -n "$BUNDLE" ]; then
|
|
mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz
|
|
echo "✓ v1beta2 bundle saved"
|
|
fi
|
|
) &
|
|
PID_V1BETA2=$!
|
|
|
|
# Run support bundle in background
|
|
(
|
|
echo "Starting support bundle..."
|
|
./bin/support-bundle \
|
|
examples/collect/host/all-kubernetes-collectors.yaml \
|
|
--interactive=false \
|
|
--output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true
|
|
|
|
if [ -f test/output/supportbundle.tar.gz ]; then
|
|
echo "✓ Support bundle saved"
|
|
fi
|
|
) &
|
|
PID_SUPPORTBUNDLE=$!
|
|
|
|
# Wait for all to complete
|
|
echo "Waiting for all specs to complete..."
|
|
wait $PID_V1BETA3
|
|
wait $PID_V1BETA2
|
|
wait $PID_SUPPORTBUNDLE
|
|
|
|
echo "All specs completed!"
|
|
|
|
# Verify bundles exist
|
|
ls -lh test/output/*.tar.gz || echo "Warning: Some bundles may be missing"
|
|
|
|
# 3. COMPARE BUNDLES
|
|
- name: Compare preflight v1beta3 bundle
|
|
id: compare-v1beta3
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Comparing v1beta3 preflight bundle against baseline..."
|
|
if [ ! -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then
|
|
echo "⚠ No baseline found for v1beta3 - skipping comparison"
|
|
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
python3 scripts/compare_bundles.py \
|
|
--baseline test/baselines/preflight-v1beta3/baseline.tar.gz \
|
|
--current test/output/preflight-v1beta3-bundle.tar.gz \
|
|
--rules scripts/compare_rules.yaml \
|
|
--report test/output/diff-report-v1beta3.json \
|
|
--spec-type preflight
|
|
|
|
- name: Compare preflight v1beta2 bundle
|
|
id: compare-v1beta2
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Comparing v1beta2 preflight bundle against baseline..."
|
|
if [ ! -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then
|
|
echo "⚠ No baseline found for v1beta2 - skipping comparison"
|
|
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
python3 scripts/compare_bundles.py \
|
|
--baseline test/baselines/preflight-v1beta2/baseline.tar.gz \
|
|
--current test/output/preflight-v1beta2-bundle.tar.gz \
|
|
--rules scripts/compare_rules.yaml \
|
|
--report test/output/diff-report-v1beta2.json \
|
|
--spec-type preflight
|
|
|
|
- name: Compare support bundle
|
|
id: compare-supportbundle
|
|
continue-on-error: true
|
|
run: |
|
|
echo "Comparing support bundle against baseline..."
|
|
if [ ! -f test/baselines/supportbundle/baseline.tar.gz ]; then
|
|
echo "⚠ No baseline found for support bundle - skipping comparison"
|
|
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
python3 scripts/compare_bundles.py \
|
|
--baseline test/baselines/supportbundle/baseline.tar.gz \
|
|
--current test/output/supportbundle.tar.gz \
|
|
--rules scripts/compare_rules.yaml \
|
|
--report test/output/diff-report-supportbundle.json \
|
|
--spec-type supportbundle
|
|
|
|
# 4. REPORT RESULTS
|
|
- name: Generate summary report
|
|
if: always()
|
|
run: |
|
|
python3 scripts/generate_summary.py \
|
|
--reports test/output/diff-report-*.json \
|
|
--output-file $GITHUB_STEP_SUMMARY \
|
|
--output-console
|
|
|
|
- name: Upload test artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v5
|
|
with:
|
|
name: regression-test-results-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
test/output/*.tar.gz
|
|
test/output/*.json
|
|
retention-days: 30
|
|
|
|
- name: Check for regressions
|
|
if: always()
|
|
run: |
|
|
echo "Checking comparison results..."
|
|
|
|
# Check if any comparisons failed
|
|
FAILURES=0
|
|
|
|
if [ "${{ steps.compare-v1beta3.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta3.outputs.baseline_missing }}" != "true" ]; then
|
|
echo "❌ v1beta3 comparison failed"
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
|
|
if [ "${{ steps.compare-v1beta2.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta2.outputs.baseline_missing }}" != "true" ]; then
|
|
echo "❌ v1beta2 comparison failed"
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
|
|
if [ "${{ steps.compare-supportbundle.outcome }}" == "failure" ] && [ "${{ steps.compare-supportbundle.outputs.baseline_missing }}" != "true" ]; then
|
|
echo "❌ Support bundle comparison failed"
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
|
|
if [ $FAILURES -gt 0 ]; then
|
|
echo ""
|
|
echo "❌ $FAILURES regression(s) detected!"
|
|
echo "Review the comparison reports in the artifacts."
|
|
exit 1
|
|
else
|
|
echo "✅ All comparisons passed or skipped (no baseline)"
|
|
fi
|
|
|
|
# 5. UPDATE BASELINES (optional, manual trigger only)
|
|
- name: Update baselines
|
|
if: github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch'
|
|
run: |
|
|
echo "Updating baselines with current bundles..."
|
|
|
|
# Copy new bundles as baselines
|
|
if [ -f test/output/preflight-v1beta3-bundle.tar.gz ]; then
|
|
mkdir -p test/baselines/preflight-v1beta3
|
|
cp test/output/preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz
|
|
echo "✓ Updated v1beta3 baseline"
|
|
fi
|
|
|
|
if [ -f test/output/preflight-v1beta2-bundle.tar.gz ]; then
|
|
mkdir -p test/baselines/preflight-v1beta2
|
|
cp test/output/preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz
|
|
echo "✓ Updated v1beta2 baseline"
|
|
fi
|
|
|
|
if [ -f test/output/supportbundle.tar.gz ]; then
|
|
mkdir -p test/baselines/supportbundle
|
|
cp test/output/supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz
|
|
echo "✓ Updated support bundle baseline"
|
|
fi
|
|
|
|
# Create metadata file
|
|
cat > test/baselines/metadata.json <<EOF
|
|
{
|
|
"updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"git_sha": "${{ github.sha }}",
|
|
"k8s_version": "v1.28.3",
|
|
"workflow_run": "${{ github.run_id }}"
|
|
}
|
|
EOF
|
|
|
|
# Commit and push
|
|
git config user.name "github-actions[bot]"
|
|
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
git add test/baselines/
|
|
git commit -m "chore: update regression test baselines from run ${{ github.run_id }}"
|
|
git push
|
|
|
|
# 6. CLEANUP
|
|
# Note: k3s cluster cleanup is handled automatically by the action
|