From b6b2bf84830fa1e54c23b128949274eb7c03fbdd Mon Sep 17 00:00:00 2001 From: Noah Campbell Date: Tue, 7 Oct 2025 08:47:05 -0700 Subject: [PATCH] handle node-specific files and multiple report arguments --- .github/workflows/regression-test.yaml | 111 +++++++++++++------------ scripts/compare_bundles.py | 5 ++ scripts/generate_summary.py | 40 ++++----- 3 files changed, 85 insertions(+), 71 deletions(-) diff --git a/.github/workflows/regression-test.yaml b/.github/workflows/regression-test.yaml index fea714ae..a19e197f 100644 --- a/.github/workflows/regression-test.yaml +++ b/.github/workflows/regression-test.yaml @@ -50,6 +50,7 @@ jobs: with: go-version-file: go.mod cache: true + cache-dependency-path: go.sum - name: Build binaries run: | @@ -67,65 +68,71 @@ jobs: run: | pip install pyyaml deepdiff - # 2. EXECUTE SPECS - - name: Run preflight v1beta3 (complex) + # 2. EXECUTE SPECS (in parallel) + - name: Run all specs in parallel continue-on-error: true run: | - echo "Running preflight v1beta3 spec with values file..." - ./bin/preflight \ - examples/preflight/complex-v1beta3.yaml \ - --values examples/preflight/values-complex-full.yaml \ - --interactive=false \ - --format=json \ - --output=test/output/preflight-results-v1beta3.json || true + echo "Running all 3 specs in parallel..." - # Find and rename the most recent bundle - BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) - if [ -n "$BUNDLE" ]; then - echo "Found bundle: $BUNDLE" - mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz - echo "✓ v1beta3 bundle saved" - else - echo "⚠ No v1beta3 bundle found" - exit 1 - fi + # Run v1beta3 in background + ( + echo "Starting preflight v1beta3..." + ./bin/preflight \ + examples/preflight/complex-v1beta3.yaml \ + --values examples/preflight/values-complex-full.yaml \ + --interactive=false \ + --format=json \ + --output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true - - name: Run preflight v1beta2 (all-analyzers) - continue-on-error: true - run: | - echo "Running preflight v1beta2 spec..." - ./bin/preflight \ - examples/preflight/all-analyzers-v1beta2.yaml \ - --interactive=false \ - --format=json \ - --output=test/output/preflight-results-v1beta2.json || true + BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) + if [ -n "$BUNDLE" ]; then + mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz + echo "✓ v1beta3 bundle saved" + fi + ) & + PID_V1BETA3=$! - # Find and rename the most recent bundle - BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) - if [ -n "$BUNDLE" ]; then - echo "Found bundle: $BUNDLE" - mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz - echo "✓ v1beta2 bundle saved" - else - echo "⚠ No v1beta2 bundle found" - exit 1 - fi + # Run v1beta2 in background + ( + echo "Starting preflight v1beta2..." + ./bin/preflight \ + examples/preflight/all-analyzers-v1beta2.yaml \ + --interactive=false \ + --format=json \ + --output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true - - name: Run support bundle (all-kubernetes-collectors) - continue-on-error: true - run: | - echo "Running support bundle spec..." - ./bin/support-bundle \ - examples/collect/host/all-kubernetes-collectors.yaml \ - --interactive=false \ - --output=test/output/supportbundle.tar.gz || true + BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) + if [ -n "$BUNDLE" ]; then + mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz + echo "✓ v1beta2 bundle saved" + fi + ) & + PID_V1BETA2=$! - if [ -f test/output/supportbundle.tar.gz ]; then - echo "✓ Support bundle saved" - else - echo "⚠ No support bundle found" - exit 1 - fi + # Run support bundle in background + ( + echo "Starting support bundle..." + ./bin/support-bundle \ + examples/collect/host/all-kubernetes-collectors.yaml \ + --interactive=false \ + --output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true + + if [ -f test/output/supportbundle.tar.gz ]; then + echo "✓ Support bundle saved" + fi + ) & + PID_SUPPORTBUNDLE=$! + + # Wait for all to complete + echo "Waiting for all specs to complete..." + wait $PID_V1BETA3 + wait $PID_V1BETA2 + wait $PID_SUPPORTBUNDLE + + echo "All specs completed!" + + # Verify bundles exist + ls -lh test/output/*.tar.gz || echo "Warning: Some bundles may be missing" # 3. COMPARE BUNDLES - name: Compare preflight v1beta3 bundle diff --git a/scripts/compare_bundles.py b/scripts/compare_bundles.py index b4afba97..20caca60 100755 --- a/scripts/compare_bundles.py +++ b/scripts/compare_bundles.py @@ -117,6 +117,11 @@ class BundleComparator: # Filter out optional files that may not exist (previous logs, etc.) optional_patterns = [ "*-previous.log", # Previous container logs (only exist after restart) + "node-metrics/*.json", # Node IDs vary between clusters + "sysctl/*", # Node IDs vary between clusters + "collectd/rrd/*", # Node IDs vary between clusters + "copy-from-host-example/*", # Node IDs vary between clusters + "run-daemonset-example/*.log", # Node IDs vary between clusters ] for file in sorted(missing_in_current): diff --git a/scripts/generate_summary.py b/scripts/generate_summary.py index 8a310d8e..e4e83925 100755 --- a/scripts/generate_summary.py +++ b/scripts/generate_summary.py @@ -14,32 +14,33 @@ from pathlib import Path from typing import List, Dict -def load_reports(report_pattern: str) -> List[Dict]: - """Load all report JSON files matching pattern.""" +def load_reports(report_files: List[str]) -> List[Dict]: + """Load all report JSON files.""" reports = [] - # Handle glob pattern - if '*' in report_pattern: - report_dir = Path(report_pattern).parent - pattern = Path(report_pattern).name + for report_file in report_files: + # Handle glob pattern if not already expanded + if '*' in report_file: + report_dir = Path(report_file).parent + pattern = Path(report_file).name - for report_file in sorted(report_dir.glob(pattern)): + for path in sorted(report_dir.glob(pattern)): + try: + with open(path) as f: + report = json.load(f) + report['_filename'] = path.name + reports.append(report) + except (json.JSONDecodeError, FileNotFoundError) as e: + print(f"Warning: Could not load {path}: {e}", file=sys.stderr) + else: + # Single file try: with open(report_file) as f: report = json.load(f) - report['_filename'] = report_file.name + report['_filename'] = Path(report_file).name reports.append(report) except (json.JSONDecodeError, FileNotFoundError) as e: print(f"Warning: Could not load {report_file}: {e}", file=sys.stderr) - else: - # Single file - try: - with open(report_pattern) as f: - report = json.load(f) - report['_filename'] = Path(report_pattern).name - reports.append(report) - except (json.JSONDecodeError, FileNotFoundError) as e: - print(f"Warning: Could not load {report_pattern}: {e}", file=sys.stderr) return reports @@ -233,8 +234,9 @@ def main(): ) parser.add_argument( "--reports", + nargs='+', required=True, - help="Report file(s) pattern (e.g., 'test/output/diff-report-*.json')" + help="Report file(s) or pattern (e.g., 'test/output/diff-report-*.json' or multiple files)" ) parser.add_argument( "--output-file", @@ -248,7 +250,7 @@ def main(): args = parser.parse_args() - # Load reports + # Load reports (args.reports is now a list) reports = load_reports(args.reports) if not reports: