mirror of
https://github.com/replicatedhq/troubleshoot.git
synced 2026-04-15 07:16:34 +00:00
Add regression test suite to github actions
This commit is contained in:
Noah Campbell
@@ -0,0 +1,280 @@
|
||||
name: Regression Test Suite
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, v1beta3]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
update_baselines:
|
||||
description: 'Update baselines after run (use with caution)'
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
regression-test:
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
# 1. SETUP
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Create k3s cluster
|
||||
id: create-cluster
|
||||
uses: replicatedhq/compatibility-actions/create-cluster@v1
|
||||
with:
|
||||
api-token: ${{ secrets.REPLICATED_API_TOKEN }}
|
||||
kubernetes-distribution: k3s
|
||||
kubernetes-version: v1.28.3
|
||||
cluster-name: regression-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
ttl: 25m
|
||||
timeout-minutes: 5
|
||||
|
||||
- name: Configure kubeconfig
|
||||
run: |
|
||||
echo "${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > $GITHUB_WORKSPACE/kubeconfig.yaml
|
||||
echo "KUBECONFIG=$GITHUB_WORKSPACE/kubeconfig.yaml" >> $GITHUB_ENV
|
||||
kubectl get nodes -o wide
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Build binaries
|
||||
run: |
|
||||
echo "Building preflight and support-bundle binaries..."
|
||||
make bin/preflight bin/support-bundle
|
||||
./bin/preflight version
|
||||
./bin/support-bundle version
|
||||
|
||||
- name: Setup Python for comparison
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install pyyaml deepdiff
|
||||
|
||||
# 2. EXECUTE SPECS
|
||||
- name: Run preflight v1beta3 (complex)
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "Running preflight v1beta3 spec with values file..."
|
||||
./bin/preflight \
|
||||
examples/preflight/complex-v1beta3.yaml \
|
||||
--values examples/preflight/values-complex-full.yaml \
|
||||
--interactive=false \
|
||||
--format=json \
|
||||
--output=test/output/preflight-results-v1beta3.json || true
|
||||
|
||||
# Find and rename the most recent bundle
|
||||
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
|
||||
if [ -n "$BUNDLE" ]; then
|
||||
echo "Found bundle: $BUNDLE"
|
||||
mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz
|
||||
echo "✓ v1beta3 bundle saved"
|
||||
else
|
||||
echo "⚠ No v1beta3 bundle found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run preflight v1beta2 (all-analyzers)
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "Running preflight v1beta2 spec..."
|
||||
./bin/preflight \
|
||||
examples/preflight/all-analyzers-v1beta2.yaml \
|
||||
--interactive=false \
|
||||
--format=json \
|
||||
--output=test/output/preflight-results-v1beta2.json || true
|
||||
|
||||
# Find and rename the most recent bundle
|
||||
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
|
||||
if [ -n "$BUNDLE" ]; then
|
||||
echo "Found bundle: $BUNDLE"
|
||||
mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz
|
||||
echo "✓ v1beta2 bundle saved"
|
||||
else
|
||||
echo "⚠ No v1beta2 bundle found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run support bundle (all-kubernetes-collectors)
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "Running support bundle spec..."
|
||||
./bin/support-bundle \
|
||||
examples/collect/host/all-kubernetes-collectors.yaml \
|
||||
--interactive=false \
|
||||
--output=test/output/supportbundle.tar.gz || true
|
||||
|
||||
if [ -f test/output/supportbundle.tar.gz ]; then
|
||||
echo "✓ Support bundle saved"
|
||||
else
|
||||
echo "⚠ No support bundle found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3. COMPARE BUNDLES
|
||||
- name: Compare preflight v1beta3 bundle
|
||||
id: compare-v1beta3
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "Comparing v1beta3 preflight bundle against baseline..."
|
||||
if [ ! -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then
|
||||
echo "⚠ No baseline found for v1beta3 - skipping comparison"
|
||||
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 scripts/compare_bundles.py \
|
||||
--baseline test/baselines/preflight-v1beta3/baseline.tar.gz \
|
||||
--current test/output/preflight-v1beta3-bundle.tar.gz \
|
||||
--rules scripts/compare_rules.yaml \
|
||||
--report test/output/diff-report-v1beta3.json \
|
||||
--spec-type preflight
|
||||
|
||||
- name: Compare preflight v1beta2 bundle
|
||||
id: compare-v1beta2
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "Comparing v1beta2 preflight bundle against baseline..."
|
||||
if [ ! -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then
|
||||
echo "⚠ No baseline found for v1beta2 - skipping comparison"
|
||||
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 scripts/compare_bundles.py \
|
||||
--baseline test/baselines/preflight-v1beta2/baseline.tar.gz \
|
||||
--current test/output/preflight-v1beta2-bundle.tar.gz \
|
||||
--rules scripts/compare_rules.yaml \
|
||||
--report test/output/diff-report-v1beta2.json \
|
||||
--spec-type preflight
|
||||
|
||||
- name: Compare support bundle
|
||||
id: compare-supportbundle
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "Comparing support bundle against baseline..."
|
||||
if [ ! -f test/baselines/supportbundle/baseline.tar.gz ]; then
|
||||
echo "⚠ No baseline found for support bundle - skipping comparison"
|
||||
echo "baseline_missing=true" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 scripts/compare_bundles.py \
|
||||
--baseline test/baselines/supportbundle/baseline.tar.gz \
|
||||
--current test/output/supportbundle.tar.gz \
|
||||
--rules scripts/compare_rules.yaml \
|
||||
--report test/output/diff-report-supportbundle.json \
|
||||
--spec-type supportbundle
|
||||
|
||||
# 4. REPORT RESULTS
|
||||
- name: Generate summary report
|
||||
if: always()
|
||||
run: |
|
||||
python3 scripts/generate_summary.py \
|
||||
--reports test/output/diff-report-*.json \
|
||||
--output-file $GITHUB_STEP_SUMMARY \
|
||||
--output-console
|
||||
|
||||
- name: Upload test artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: regression-test-results-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
test/output/*.tar.gz
|
||||
test/output/*.json
|
||||
retention-days: 30
|
||||
|
||||
- name: Check for regressions
|
||||
if: always()
|
||||
run: |
|
||||
echo "Checking comparison results..."
|
||||
|
||||
# Check if any comparisons failed
|
||||
FAILURES=0
|
||||
|
||||
if [ "${{ steps.compare-v1beta3.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta3.outputs.baseline_missing }}" != "true" ]; then
|
||||
echo "❌ v1beta3 comparison failed"
|
||||
FAILURES=$((FAILURES + 1))
|
||||
fi
|
||||
|
||||
if [ "${{ steps.compare-v1beta2.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta2.outputs.baseline_missing }}" != "true" ]; then
|
||||
echo "❌ v1beta2 comparison failed"
|
||||
FAILURES=$((FAILURES + 1))
|
||||
fi
|
||||
|
||||
if [ "${{ steps.compare-supportbundle.outcome }}" == "failure" ] && [ "${{ steps.compare-supportbundle.outputs.baseline_missing }}" != "true" ]; then
|
||||
echo "❌ Support bundle comparison failed"
|
||||
FAILURES=$((FAILURES + 1))
|
||||
fi
|
||||
|
||||
if [ $FAILURES -gt 0 ]; then
|
||||
echo ""
|
||||
echo "❌ $FAILURES regression(s) detected!"
|
||||
echo "Review the comparison reports in the artifacts."
|
||||
exit 1
|
||||
else
|
||||
echo "✅ All comparisons passed or skipped (no baseline)"
|
||||
fi
|
||||
|
||||
# 5. UPDATE BASELINES (optional, manual trigger only)
|
||||
- name: Update baselines
|
||||
if: github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch'
|
||||
run: |
|
||||
echo "Updating baselines with current bundles..."
|
||||
|
||||
# Copy new bundles as baselines
|
||||
if [ -f test/output/preflight-v1beta3-bundle.tar.gz ]; then
|
||||
mkdir -p test/baselines/preflight-v1beta3
|
||||
cp test/output/preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz
|
||||
echo "✓ Updated v1beta3 baseline"
|
||||
fi
|
||||
|
||||
if [ -f test/output/preflight-v1beta2-bundle.tar.gz ]; then
|
||||
mkdir -p test/baselines/preflight-v1beta2
|
||||
cp test/output/preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz
|
||||
echo "✓ Updated v1beta2 baseline"
|
||||
fi
|
||||
|
||||
if [ -f test/output/supportbundle.tar.gz ]; then
|
||||
mkdir -p test/baselines/supportbundle
|
||||
cp test/output/supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz
|
||||
echo "✓ Updated support bundle baseline"
|
||||
fi
|
||||
|
||||
# Create metadata file
|
||||
cat > test/baselines/metadata.json <<EOF
|
||||
{
|
||||
"updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"git_sha": "${{ github.sha }}",
|
||||
"k8s_version": "v1.28.3",
|
||||
"workflow_run": "${{ github.run_id }}"
|
||||
}
|
||||
EOF
|
||||
|
||||
# Commit and push
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add test/baselines/
|
||||
git commit -m "chore: update regression test baselines from run ${{ github.run_id }}"
|
||||
git push
|
||||
|
||||
# 6. CLEANUP
|
||||
- name: Remove cluster
|
||||
if: always()
|
||||
uses: replicatedhq/compatibility-actions/remove-cluster@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
api-token: ${{ secrets.REPLICATED_API_TOKEN }}
|
||||
cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
|
||||
@@ -48,6 +48,7 @@ sbom/
|
||||
# Ignore generated support bundles
|
||||
*.tar.gz
|
||||
!testdata/supportbundle/*.tar.gz
|
||||
!test/baselines/**/baseline.tar.gz
|
||||
|
||||
# Ignore built binaries
|
||||
troubleshoot
|
||||
|
||||
Executable
+515
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Bundle comparison engine for regression testing.
|
||||
|
||||
Unpacks baseline and current bundles, applies comparison rules,
|
||||
generates diff report, and exits non-zero on regressions.
|
||||
|
||||
Based on the simplified 3-tier approach:
|
||||
1. EXACT match for deterministic files (static data, version.yaml)
|
||||
2. STRUCTURAL comparison for semi-deterministic files (databases, DNS, etc.)
|
||||
3. NON-EMPTY check for variable files (cluster-resources, metrics, logs)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional
|
||||
import fnmatch
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("Error: pyyaml not installed. Run: pip install pyyaml")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from deepdiff import DeepDiff
|
||||
except ImportError:
|
||||
print("Error: deepdiff not installed. Run: pip install deepdiff")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
class BundleComparator:
|
||||
"""Compare two troubleshoot bundles using rule-based comparison."""
|
||||
|
||||
def __init__(self, rules_path: str, spec_type: str):
|
||||
self.rules = self._load_rules(rules_path, spec_type)
|
||||
self.spec_type = spec_type
|
||||
self.results = {
|
||||
"spec_type": spec_type,
|
||||
"files_compared": 0,
|
||||
"exact_matches": 0,
|
||||
"structural_matches": 0,
|
||||
"non_empty_checks": 0,
|
||||
"files_different": 0,
|
||||
"files_missing_in_current": 0,
|
||||
"files_missing_in_baseline": 0,
|
||||
"differences": [],
|
||||
"missing_in_current": [],
|
||||
"missing_in_baseline": [],
|
||||
}
|
||||
|
||||
def _load_rules(self, rules_path: str, spec_type: str) -> Dict:
|
||||
"""Load comparison rules from YAML file."""
|
||||
if not Path(rules_path).exists():
|
||||
print(f"Warning: Rules file not found at {rules_path}, using defaults")
|
||||
return self._get_default_rules()
|
||||
|
||||
with open(rules_path) as f:
|
||||
rules = yaml.safe_load(f)
|
||||
|
||||
return rules.get(spec_type, rules.get("defaults", {}))
|
||||
|
||||
def _get_default_rules(self) -> Dict:
|
||||
"""Return default comparison rules if no config file."""
|
||||
return {
|
||||
"exact_match": [
|
||||
"static-data.txt/static-data",
|
||||
"version.yaml",
|
||||
],
|
||||
"structural_compare": {
|
||||
"postgres/*.json": "database_connection",
|
||||
"mysql/*.json": "database_connection",
|
||||
"mssql/*.json": "database_connection",
|
||||
"redis/*.json": "database_connection",
|
||||
"dns/debug.json": "dns_structure",
|
||||
"registry/*.json": "registry_exists",
|
||||
"http*.json": "http_status",
|
||||
},
|
||||
"non_empty_default": True,
|
||||
}
|
||||
|
||||
def compare(self, baseline_bundle: str, current_bundle: str) -> bool:
|
||||
"""
|
||||
Compare two bundles. Returns True if no regressions detected.
|
||||
|
||||
Args:
|
||||
baseline_bundle: Path to baseline bundle tar.gz
|
||||
current_bundle: Path to current bundle tar.gz
|
||||
|
||||
Returns:
|
||||
True if bundles match (no regressions), False otherwise
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
baseline_dir = Path(tmpdir) / "baseline"
|
||||
current_dir = Path(tmpdir) / "current"
|
||||
|
||||
print(f"Extracting baseline bundle to {baseline_dir}...")
|
||||
self._extract(baseline_bundle, baseline_dir)
|
||||
|
||||
print(f"Extracting current bundle to {current_dir}...")
|
||||
self._extract(current_bundle, current_dir)
|
||||
|
||||
baseline_files = self._get_file_list(baseline_dir)
|
||||
current_files = self._get_file_list(current_dir)
|
||||
|
||||
print(f"Baseline files: {len(baseline_files)}")
|
||||
print(f"Current files: {len(current_files)}")
|
||||
|
||||
# Check for missing files
|
||||
missing_in_current = baseline_files - current_files
|
||||
missing_in_baseline = current_files - baseline_files
|
||||
|
||||
# Filter out optional files that may not exist (previous logs, etc.)
|
||||
optional_patterns = [
|
||||
"*-previous.log", # Previous container logs (only exist after restart)
|
||||
]
|
||||
|
||||
for file in sorted(missing_in_current):
|
||||
# Skip optional files
|
||||
if any(file.match(pattern) for pattern in optional_patterns):
|
||||
print(f" ℹ Optional file missing (OK): {file}")
|
||||
continue
|
||||
self._record_missing("current", str(file))
|
||||
|
||||
for file in sorted(missing_in_baseline):
|
||||
# Optional files added in current are also OK
|
||||
if any(file.match(pattern) for pattern in optional_patterns):
|
||||
print(f" ℹ Optional file added (OK): {file}")
|
||||
continue
|
||||
self._record_missing("baseline", str(file))
|
||||
|
||||
# Compare common files
|
||||
common_files = baseline_files & current_files
|
||||
print(f"Comparing {len(common_files)} common files...")
|
||||
|
||||
for file in sorted(common_files):
|
||||
self._compare_file(
|
||||
baseline_dir / file,
|
||||
current_dir / file,
|
||||
str(file)
|
||||
)
|
||||
|
||||
# Determine if there are regressions
|
||||
has_regressions = (
|
||||
self.results["files_different"] > 0 or
|
||||
self.results["files_missing_in_current"] > 0
|
||||
)
|
||||
|
||||
return not has_regressions
|
||||
|
||||
def _extract(self, bundle_path: str, dest_dir: Path):
|
||||
"""Extract tar.gz bundle to destination directory."""
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tarfile.open(bundle_path, 'r:gz') as tar:
|
||||
tar.extractall(dest_dir)
|
||||
|
||||
# Handle bundles that extract to a nested directory (e.g., preflightbundle-timestamp/)
|
||||
# If there's only one directory at the root, use that as the actual root
|
||||
items = list(dest_dir.iterdir())
|
||||
if len(items) == 1 and items[0].is_dir():
|
||||
# Move contents up one level
|
||||
nested_dir = items[0]
|
||||
for item in nested_dir.iterdir():
|
||||
item.rename(dest_dir / item.name)
|
||||
nested_dir.rmdir()
|
||||
|
||||
def _get_file_list(self, dir_path: Path) -> set:
|
||||
"""Get set of all files in directory (relative paths)."""
|
||||
files = set()
|
||||
for path in dir_path.rglob('*'):
|
||||
if path.is_file():
|
||||
rel_path = path.relative_to(dir_path)
|
||||
files.add(rel_path)
|
||||
return files
|
||||
|
||||
def _compare_file(self, baseline_path: Path, current_path: Path, rel_path: str):
|
||||
"""Compare a single file pair using appropriate rule."""
|
||||
self.results["files_compared"] += 1
|
||||
|
||||
# Determine comparison mode
|
||||
mode = self._get_comparison_mode(rel_path)
|
||||
|
||||
try:
|
||||
if mode == "exact":
|
||||
if self._compare_exact(baseline_path, current_path):
|
||||
self.results["exact_matches"] += 1
|
||||
else:
|
||||
self._record_diff(rel_path, "exact", "Content mismatch")
|
||||
|
||||
elif mode == "structural":
|
||||
comparator = self._get_structural_comparator(rel_path)
|
||||
if self._compare_structural(baseline_path, current_path, comparator):
|
||||
self.results["structural_matches"] += 1
|
||||
else:
|
||||
self._record_diff(rel_path, "structural", f"Structural comparison failed ({comparator})")
|
||||
|
||||
else: # non_empty
|
||||
if self._check_non_empty(current_path):
|
||||
self.results["non_empty_checks"] += 1
|
||||
else:
|
||||
self._record_diff(rel_path, "non_empty", "File is empty")
|
||||
|
||||
except Exception as e:
|
||||
self._record_diff(rel_path, "error", f"Comparison error: {str(e)}")
|
||||
|
||||
def _get_comparison_mode(self, rel_path: str) -> str:
|
||||
"""Determine comparison mode for a file based on rules."""
|
||||
# Check exact match patterns
|
||||
for pattern in self.rules.get("exact_match", []):
|
||||
if fnmatch.fnmatch(rel_path, pattern) or rel_path == pattern:
|
||||
return "exact"
|
||||
|
||||
# Check structural comparison patterns
|
||||
for pattern in self.rules.get("structural_compare", {}).keys():
|
||||
if fnmatch.fnmatch(rel_path, pattern):
|
||||
return "structural"
|
||||
|
||||
# Default: non-empty check
|
||||
return "non_empty"
|
||||
|
||||
def _get_structural_comparator(self, rel_path: str) -> str:
|
||||
"""Get the structural comparator name for a file."""
|
||||
for pattern, comparator in self.rules.get("structural_compare", {}).items():
|
||||
if fnmatch.fnmatch(rel_path, pattern):
|
||||
return comparator
|
||||
return "unknown"
|
||||
|
||||
def _compare_exact(self, baseline_path: Path, current_path: Path) -> bool:
|
||||
"""Compare files byte-for-byte."""
|
||||
return baseline_path.read_bytes() == current_path.read_bytes()
|
||||
|
||||
def _compare_structural(self, baseline_path: Path, current_path: Path, comparator: str) -> bool:
|
||||
"""Compare files using structural comparator."""
|
||||
# Load JSON data
|
||||
try:
|
||||
baseline_data = json.loads(baseline_path.read_text())
|
||||
current_data = json.loads(current_path.read_text())
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" JSON decode error: {e}")
|
||||
return False
|
||||
|
||||
# Apply comparator
|
||||
if comparator == "database_connection":
|
||||
return self._compare_database_connection(baseline_data, current_data)
|
||||
elif comparator == "dns_structure":
|
||||
return self._compare_dns_structure(baseline_data, current_data)
|
||||
elif comparator == "registry_exists":
|
||||
return self._compare_registry_exists(baseline_data, current_data)
|
||||
elif comparator == "http_status":
|
||||
return self._compare_http_status(baseline_data, current_data)
|
||||
elif comparator == "cluster_version":
|
||||
return self._compare_cluster_version(baseline_data, current_data)
|
||||
elif comparator == "analysis_results":
|
||||
return self._compare_analysis_results(baseline_data, current_data)
|
||||
else:
|
||||
# Unknown comparator - fall back to non-empty
|
||||
return True
|
||||
|
||||
def _compare_database_connection(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare database connection results (isConnected field only)."""
|
||||
b_connected = baseline.get("isConnected", False)
|
||||
c_connected = current.get("isConnected", False)
|
||||
|
||||
if b_connected != c_connected:
|
||||
print(f" Database connection status changed: {b_connected} -> {c_connected}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_dns_structure(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare DNS structure (service exists, query succeeds)."""
|
||||
# Check kubernetes service exists
|
||||
if "query" not in current or "kubernetes" not in current["query"]:
|
||||
print(f" DNS query.kubernetes missing")
|
||||
return False
|
||||
|
||||
# Kubernetes ClusterIP should exist (don't compare value, it can vary)
|
||||
if not current["query"]["kubernetes"].get("address"):
|
||||
print(f" DNS kubernetes.address is empty")
|
||||
return False
|
||||
|
||||
# DNS service should exist
|
||||
if not current.get("kubeDNSService"):
|
||||
print(f" DNS kubeDNSService is empty")
|
||||
return False
|
||||
|
||||
# At least one DNS pod should exist
|
||||
if not current.get("kubeDNSPods") or len(current["kubeDNSPods"]) == 0:
|
||||
print(f" DNS kubeDNSPods is empty")
|
||||
return False
|
||||
|
||||
# Non-resolvable domain should be empty
|
||||
if current.get("query", {}).get("nonResolvableDomain", {}).get("address"):
|
||||
print(f" DNS nonResolvableDomain should be empty")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_registry_exists(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare registry image existence (exists boolean per image)."""
|
||||
baseline_images = baseline.get("images", {})
|
||||
current_images = current.get("images", {})
|
||||
|
||||
# Check same images are present
|
||||
if set(baseline_images.keys()) != set(current_images.keys()):
|
||||
print(f" Registry image list changed")
|
||||
print(f" Baseline: {sorted(baseline_images.keys())}")
|
||||
print(f" Current: {sorted(current_images.keys())}")
|
||||
return False
|
||||
|
||||
# Compare exists status for each image
|
||||
for image_name in baseline_images:
|
||||
b_exists = baseline_images[image_name].get("exists", False)
|
||||
c_exists = current_images[image_name].get("exists", False)
|
||||
|
||||
if b_exists != c_exists:
|
||||
print(f" Registry image '{image_name}' existence changed: {b_exists} -> {c_exists}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_http_status(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare HTTP response (status code only)."""
|
||||
b_status = baseline.get("response", {}).get("status", 0)
|
||||
c_status = current.get("response", {}).get("status", 0)
|
||||
|
||||
if b_status != c_status:
|
||||
print(f" HTTP status changed: {b_status} -> {c_status}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_cluster_version(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare cluster version (major/minor only, ignore build details)."""
|
||||
b_info = baseline.get("info", {})
|
||||
c_info = current.get("info", {})
|
||||
|
||||
# Compare major and minor version
|
||||
if b_info.get("major") != c_info.get("major"):
|
||||
print(f" Cluster major version changed: {b_info.get('major')} -> {c_info.get('major')}")
|
||||
return False
|
||||
|
||||
if b_info.get("minor") != c_info.get("minor"):
|
||||
print(f" Cluster minor version changed: {b_info.get('minor')} -> {c_info.get('minor')}")
|
||||
return False
|
||||
|
||||
# Don't compare: gitVersion, gitCommit, buildDate, goVersion (these vary with k3s updates)
|
||||
return True
|
||||
|
||||
def _compare_analysis_results(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare analysis results (analyzer names and count, not specific messages)."""
|
||||
if not isinstance(baseline, list) or not isinstance(current, list):
|
||||
print(f" Analysis results structure changed (expected list)")
|
||||
return False
|
||||
|
||||
# Create map of analyzer name -> severity for comparison
|
||||
baseline_results = {item.get("name"): item.get("severity") for item in baseline if "name" in item}
|
||||
current_results = {item.get("name"): item.get("severity") for item in current if "name" in item}
|
||||
|
||||
# Check if same analyzers ran
|
||||
baseline_names = set(baseline_results.keys())
|
||||
current_names = set(current_results.keys())
|
||||
|
||||
if baseline_names != current_names:
|
||||
missing = baseline_names - current_names
|
||||
extra = current_names - baseline_names
|
||||
if missing:
|
||||
print(f" Missing analyzers: {missing}")
|
||||
if extra:
|
||||
print(f" New analyzers: {extra}")
|
||||
return False
|
||||
|
||||
# Check if severity levels changed significantly (error/warn differences matter)
|
||||
significant_changes = []
|
||||
for name in baseline_names:
|
||||
b_sev = baseline_results[name]
|
||||
c_sev = current_results[name]
|
||||
|
||||
# Only care if error/warn status changes, not debug
|
||||
if b_sev != c_sev:
|
||||
if b_sev in ["error", "warn"] or c_sev in ["error", "warn"]:
|
||||
significant_changes.append(f"{name}: {b_sev} -> {c_sev}")
|
||||
|
||||
if significant_changes:
|
||||
print(f" Analyzer severity changed:")
|
||||
for change in significant_changes[:5]: # Show first 5
|
||||
print(f" {change}")
|
||||
# Don't fail on severity changes - this is informational
|
||||
# return False
|
||||
|
||||
return True
|
||||
|
||||
def _check_non_empty(self, path: Path) -> bool:
|
||||
"""Check that file exists and is non-empty."""
|
||||
if not path.exists():
|
||||
return False
|
||||
|
||||
size = path.stat().st_size
|
||||
if size == 0:
|
||||
return False
|
||||
|
||||
# Optional: validate JSON structure if .json extension
|
||||
if path.suffix == ".json":
|
||||
try:
|
||||
json.loads(path.read_text())
|
||||
except json.JSONDecodeError:
|
||||
print(f" Invalid JSON: {path.name}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _record_diff(self, file: str, mode: str, reason: str):
|
||||
"""Record a difference/regression."""
|
||||
self.results["files_different"] += 1
|
||||
self.results["differences"].append({
|
||||
"file": file,
|
||||
"mode": mode,
|
||||
"reason": reason
|
||||
})
|
||||
print(f" ❌ {file}: {reason}")
|
||||
|
||||
def _record_missing(self, location: str, file: str):
|
||||
"""Record a missing file."""
|
||||
if location == "current":
|
||||
self.results["files_missing_in_current"] += 1
|
||||
self.results["missing_in_current"].append(file)
|
||||
print(f" ⚠ Missing in current: {file}")
|
||||
else:
|
||||
self.results["files_missing_in_baseline"] += 1
|
||||
self.results["missing_in_baseline"].append(file)
|
||||
print(f" ℹ New file in current: {file}")
|
||||
|
||||
def generate_report(self, output_path: str):
|
||||
"""Write JSON report."""
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump(self.results, f, indent=2)
|
||||
|
||||
print(f"\nReport written to: {output_path}")
|
||||
|
||||
def print_summary(self):
|
||||
"""Print human-readable summary to stdout."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Bundle Comparison Report - {self.spec_type}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Files compared: {self.results['files_compared']}")
|
||||
print(f" Exact matches: {self.results['exact_matches']}")
|
||||
print(f" Structural matches: {self.results['structural_matches']}")
|
||||
print(f" Non-empty checks: {self.results['non_empty_checks']}")
|
||||
print(f"Files different: {self.results['files_different']}")
|
||||
print(f"Missing in current: {self.results['files_missing_in_current']}")
|
||||
print(f"Missing in baseline: {self.results['files_missing_in_baseline']}")
|
||||
|
||||
if self.results["differences"]:
|
||||
print(f"\n❌ REGRESSIONS DETECTED ({len(self.results['differences'])}):")
|
||||
for diff in self.results["differences"][:10]: # Show first 10
|
||||
print(f" • {diff['file']}: {diff['reason']}")
|
||||
if len(self.results["differences"]) > 10:
|
||||
print(f" ... and {len(self.results['differences']) - 10} more")
|
||||
|
||||
if self.results["missing_in_current"]:
|
||||
print(f"\n⚠ MISSING FILES ({len(self.results['missing_in_current'])}):")
|
||||
for file in self.results["missing_in_current"][:5]:
|
||||
print(f" • {file}")
|
||||
if len(self.results["missing_in_current"]) > 5:
|
||||
print(f" ... and {len(self.results['missing_in_current']) - 5} more")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare troubleshoot bundles for regression testing"
|
||||
)
|
||||
parser.add_argument("--baseline", required=True, help="Baseline bundle tar.gz path")
|
||||
parser.add_argument("--current", required=True, help="Current bundle tar.gz path")
|
||||
parser.add_argument("--rules", required=True, help="Comparison rules YAML path")
|
||||
parser.add_argument("--report", required=True, help="Output report JSON path")
|
||||
parser.add_argument(
|
||||
"--spec-type",
|
||||
required=True,
|
||||
choices=["preflight", "supportbundle"],
|
||||
help="Type of spec being compared"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Verify files exist
|
||||
if not Path(args.baseline).exists():
|
||||
print(f"Error: Baseline bundle not found: {args.baseline}")
|
||||
sys.exit(1)
|
||||
|
||||
if not Path(args.current).exists():
|
||||
print(f"Error: Current bundle not found: {args.current}")
|
||||
sys.exit(1)
|
||||
|
||||
# Run comparison
|
||||
comparator = BundleComparator(args.rules, args.spec_type)
|
||||
passed = comparator.compare(args.baseline, args.current)
|
||||
comparator.generate_report(args.report)
|
||||
comparator.print_summary()
|
||||
|
||||
# Exit with appropriate code
|
||||
if passed:
|
||||
print("\n✅ No regressions detected")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("\n❌ Regressions detected!")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,121 @@
|
||||
# Comparison rules for regression testing
|
||||
# Defines how different collector outputs should be compared
|
||||
|
||||
# Global configuration (applies to both preflight and supportbundle)
|
||||
global:
|
||||
# Files that should be compared exactly (byte-for-byte)
|
||||
exact_match:
|
||||
- "static-data.txt/static-data"
|
||||
# Note: version.yaml is NOT here - versionNumber varies between builds
|
||||
|
||||
# Default behavior for unknown files
|
||||
non_empty_default: true
|
||||
|
||||
# Preflight-specific rules
|
||||
preflight:
|
||||
# Files that should be compared exactly
|
||||
exact_match:
|
||||
- "static-data.txt/static-data"
|
||||
- "files/example.yaml" # From data collector in v1beta2
|
||||
- "files/example.json" # From data collector in v1beta2
|
||||
- "config/replicas.txt" # From data collector in v1beta2
|
||||
# Note: version.yaml is NOT here - it uses non-empty check (versionNumber varies)
|
||||
|
||||
# Files that need structural/field-specific comparison
|
||||
# Format: "pattern": "comparator_function_name"
|
||||
structural_compare:
|
||||
# Database collectors - compare isConnected boolean only
|
||||
"postgres/*.json": "database_connection"
|
||||
"mysql/*.json": "database_connection"
|
||||
"mssql/*.json": "database_connection"
|
||||
"redis/*.json": "database_connection"
|
||||
|
||||
# DNS collector - compare structure, not exact values
|
||||
"dns/debug.json": "dns_structure"
|
||||
|
||||
# Registry collector - compare exists boolean per image
|
||||
"registry/*.json": "registry_exists"
|
||||
"registry-images/*.json": "registry_exists"
|
||||
|
||||
# HTTP collector - compare status code only
|
||||
"http/*.json": "http_status"
|
||||
"http-*.json": "http_status"
|
||||
|
||||
# Cluster info - compare major/minor version, ignore build details
|
||||
"cluster-info/cluster_version.json": "cluster_version"
|
||||
|
||||
# Analysis results - compare analyzer names and severity levels only
|
||||
"analysis.json": "analysis_results"
|
||||
|
||||
# Everything else uses non-empty check by default
|
||||
# This includes:
|
||||
# - cluster-resources/**/*.json (UIDs, timestamps vary)
|
||||
# - node-metrics/**/*.json (all values vary)
|
||||
# - goldpinger/**/*.json (latencies vary)
|
||||
# - run*/**/* (pod names, output vary)
|
||||
# - ceph/**/* (status/metrics vary or not installed)
|
||||
# - longhorn/**/* (status/metrics vary or not installed)
|
||||
# - certificates/**/*.json (validity time-based)
|
||||
# - configmaps/**/*.json (can have dynamic values)
|
||||
# - secrets/**/*.json (can have dynamic values)
|
||||
# - sysctl/**/* (some counters vary)
|
||||
# - collectd/**/* (time-series data)
|
||||
# - helm/**/*.json (timestamps, revisions vary)
|
||||
# - logs/**/*.log (timestamps in every line)
|
||||
# - copy*/**/* (content depends on what's copied)
|
||||
|
||||
# Support bundle-specific rules
|
||||
supportbundle:
|
||||
# Files that should be compared exactly
|
||||
exact_match:
|
||||
- "static-data.txt/static-data"
|
||||
# Note: version.yaml is NOT here - it uses non-empty check (versionNumber varies)
|
||||
|
||||
# Files that need structural comparison
|
||||
structural_compare:
|
||||
# Database collectors
|
||||
"postgres/*.json": "database_connection"
|
||||
"mysql/*.json": "database_connection"
|
||||
"mssql/*.json": "database_connection"
|
||||
"redis/*.json": "database_connection"
|
||||
|
||||
# DNS collector
|
||||
"dns/debug.json": "dns_structure"
|
||||
|
||||
# Registry collector
|
||||
"registry/*.json": "registry_exists"
|
||||
"registry-images/*.json": "registry_exists"
|
||||
|
||||
# HTTP collector
|
||||
"http*.json": "http_status"
|
||||
|
||||
# Cluster info
|
||||
"cluster-info/cluster_version.json": "cluster_version"
|
||||
|
||||
# Everything else uses non-empty check (see list above in preflight section)
|
||||
|
||||
# Notes on comparison strategies:
|
||||
#
|
||||
# EXACT MATCH:
|
||||
# - Use for static data that should never change between runs
|
||||
# - Byte-for-byte comparison
|
||||
# - Any difference is a regression
|
||||
#
|
||||
# STRUCTURAL COMPARISON:
|
||||
# - Use for semi-deterministic output with consistent structure but variable values
|
||||
# - Compare specific fields only (e.g., status codes, booleans)
|
||||
# - Ignore timing-dependent or environment-specific values
|
||||
#
|
||||
# NON-EMPTY CHECK (default):
|
||||
# - Use for highly variable output where exact comparison is impractical
|
||||
# - Verifies file exists and is not empty
|
||||
# - For JSON files, also validates JSON is parseable
|
||||
# - Appropriate for:
|
||||
# * Kubernetes resources (UIDs, resourceVersions, timestamps)
|
||||
# * Metrics (all values constantly change)
|
||||
# * Logs (timestamps, dynamic content)
|
||||
# * Generated pod/resource names
|
||||
# * Runtime state (pod status, replica counts)
|
||||
#
|
||||
# This strategy catches major regressions (collectors breaking, files missing)
|
||||
# while avoiding false positives from expected variability.
|
||||
Executable
+280
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate summary report from bundle comparison results.
|
||||
|
||||
Reads JSON diff reports and produces:
|
||||
1. GitHub Actions step summary (Markdown)
|
||||
2. Console output (colored text)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict
|
||||
|
||||
|
||||
def load_reports(report_pattern: str) -> List[Dict]:
|
||||
"""Load all report JSON files matching pattern."""
|
||||
reports = []
|
||||
|
||||
# Handle glob pattern
|
||||
if '*' in report_pattern:
|
||||
report_dir = Path(report_pattern).parent
|
||||
pattern = Path(report_pattern).name
|
||||
|
||||
for report_file in sorted(report_dir.glob(pattern)):
|
||||
try:
|
||||
with open(report_file) as f:
|
||||
report = json.load(f)
|
||||
report['_filename'] = report_file.name
|
||||
reports.append(report)
|
||||
except (json.JSONDecodeError, FileNotFoundError) as e:
|
||||
print(f"Warning: Could not load {report_file}: {e}", file=sys.stderr)
|
||||
else:
|
||||
# Single file
|
||||
try:
|
||||
with open(report_pattern) as f:
|
||||
report = json.load(f)
|
||||
report['_filename'] = Path(report_pattern).name
|
||||
reports.append(report)
|
||||
except (json.JSONDecodeError, FileNotFoundError) as e:
|
||||
print(f"Warning: Could not load {report_pattern}: {e}", file=sys.stderr)
|
||||
|
||||
return reports
|
||||
|
||||
|
||||
def generate_markdown_summary(reports: List[Dict]) -> str:
|
||||
"""Generate GitHub Actions step summary in Markdown format."""
|
||||
lines = []
|
||||
|
||||
lines.append("# 🧪 Regression Test Results")
|
||||
lines.append("")
|
||||
|
||||
if not reports:
|
||||
lines.append("⚠️ No comparison reports found. Baselines may be missing.")
|
||||
return "\n".join(lines)
|
||||
|
||||
# Overall status
|
||||
total_regressions = sum(r.get('files_different', 0) for r in reports)
|
||||
total_missing = sum(r.get('files_missing_in_current', 0) for r in reports)
|
||||
|
||||
if total_regressions > 0 or total_missing > 0:
|
||||
lines.append(f"## ❌ Status: FAILED")
|
||||
lines.append(f"**{total_regressions} file(s) with differences, {total_missing} file(s) missing**")
|
||||
else:
|
||||
lines.append(f"## ✅ Status: PASSED")
|
||||
lines.append("All comparisons passed!")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Per-spec breakdown
|
||||
lines.append("## 📊 Comparison Breakdown")
|
||||
lines.append("")
|
||||
|
||||
for report in reports:
|
||||
spec_type = report.get('spec_type', 'unknown')
|
||||
filename = report.get('_filename', 'unknown')
|
||||
|
||||
# Determine status icon
|
||||
has_regressions = (
|
||||
report.get('files_different', 0) > 0 or
|
||||
report.get('files_missing_in_current', 0) > 0
|
||||
)
|
||||
status_icon = "❌" if has_regressions else "✅"
|
||||
|
||||
lines.append(f"### {status_icon} {spec_type.upper()}")
|
||||
lines.append("")
|
||||
lines.append(f"**Report:** `{filename}`")
|
||||
lines.append("")
|
||||
|
||||
# Stats table
|
||||
lines.append("| Metric | Count |")
|
||||
lines.append("|--------|-------|")
|
||||
lines.append(f"| Files compared | {report.get('files_compared', 0)} |")
|
||||
lines.append(f"| Exact matches | {report.get('exact_matches', 0)} |")
|
||||
lines.append(f"| Structural matches | {report.get('structural_matches', 0)} |")
|
||||
lines.append(f"| Non-empty checks | {report.get('non_empty_checks', 0)} |")
|
||||
lines.append(f"| **Files different** | **{report.get('files_different', 0)}** |")
|
||||
lines.append(f"| **Missing in current** | **{report.get('files_missing_in_current', 0)}** |")
|
||||
lines.append(f"| New in current | {report.get('files_missing_in_baseline', 0)} |")
|
||||
lines.append("")
|
||||
|
||||
# Show differences if any
|
||||
differences = report.get('differences', [])
|
||||
if differences:
|
||||
lines.append("<details>")
|
||||
lines.append(f"<summary>⚠️ Show {len(differences)} difference(s)</summary>")
|
||||
lines.append("")
|
||||
lines.append("| File | Mode | Reason |")
|
||||
lines.append("|------|------|--------|")
|
||||
for diff in differences[:20]: # Limit to 20
|
||||
file = diff.get('file', 'unknown')
|
||||
mode = diff.get('mode', 'unknown')
|
||||
reason = diff.get('reason', 'unknown')
|
||||
lines.append(f"| `{file}` | {mode} | {reason} |")
|
||||
|
||||
if len(differences) > 20:
|
||||
lines.append(f"| ... | ... | *{len(differences) - 20} more differences* |")
|
||||
|
||||
lines.append("")
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
# Show missing files if any
|
||||
missing = report.get('missing_in_current', [])
|
||||
if missing:
|
||||
lines.append("<details>")
|
||||
lines.append(f"<summary>⚠️ Show {len(missing)} missing file(s)</summary>")
|
||||
lines.append("")
|
||||
for file in missing[:20]:
|
||||
lines.append(f"- `{file}`")
|
||||
|
||||
if len(missing) > 20:
|
||||
lines.append(f"- *... and {len(missing) - 20} more*")
|
||||
|
||||
lines.append("")
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
# Footer
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("💡 **Tips:**")
|
||||
lines.append("- Download artifacts to inspect bundle contents")
|
||||
lines.append("- Review diff reports for detailed comparison results")
|
||||
lines.append("- Update baselines if changes are intentional (use workflow_dispatch with update_baselines)")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_console_summary(reports: List[Dict]) -> str:
|
||||
"""Generate console output with ANSI colors."""
|
||||
lines = []
|
||||
|
||||
# ANSI color codes
|
||||
RED = "\033[91m"
|
||||
GREEN = "\033[92m"
|
||||
YELLOW = "\033[93m"
|
||||
BLUE = "\033[94m"
|
||||
BOLD = "\033[1m"
|
||||
RESET = "\033[0m"
|
||||
|
||||
lines.append(f"\n{BOLD}{'='*60}{RESET}")
|
||||
lines.append(f"{BOLD}Regression Test Summary{RESET}")
|
||||
lines.append(f"{BOLD}{'='*60}{RESET}\n")
|
||||
|
||||
if not reports:
|
||||
lines.append(f"{YELLOW}⚠ No comparison reports found{RESET}")
|
||||
return "\n".join(lines)
|
||||
|
||||
# Overall status
|
||||
total_regressions = sum(r.get('files_different', 0) for r in reports)
|
||||
total_missing = sum(r.get('files_missing_in_current', 0) for r in reports)
|
||||
|
||||
if total_regressions > 0 or total_missing > 0:
|
||||
lines.append(f"{RED}{BOLD}❌ FAILED{RESET}")
|
||||
lines.append(f" {total_regressions} file(s) with differences")
|
||||
lines.append(f" {total_missing} file(s) missing\n")
|
||||
else:
|
||||
lines.append(f"{GREEN}{BOLD}✅ PASSED{RESET}")
|
||||
lines.append(f" All comparisons successful\n")
|
||||
|
||||
# Per-spec details
|
||||
for i, report in enumerate(reports):
|
||||
spec_type = report.get('spec_type', 'unknown')
|
||||
|
||||
has_regressions = (
|
||||
report.get('files_different', 0) > 0 or
|
||||
report.get('files_missing_in_current', 0) > 0
|
||||
)
|
||||
|
||||
status_color = RED if has_regressions else GREEN
|
||||
status_icon = "❌" if has_regressions else "✅"
|
||||
|
||||
lines.append(f"{BLUE}{BOLD}{spec_type.upper()}{RESET} {status_color}{status_icon}{RESET}")
|
||||
lines.append(f" Files compared: {report.get('files_compared', 0)}")
|
||||
lines.append(f" Exact matches: {report.get('exact_matches', 0)}")
|
||||
lines.append(f" Structural matches: {report.get('structural_matches', 0)}")
|
||||
lines.append(f" Non-empty checks: {report.get('non_empty_checks', 0)}")
|
||||
|
||||
if report.get('files_different', 0) > 0:
|
||||
lines.append(f" {RED}Files different: {report.get('files_different', 0)}{RESET}")
|
||||
|
||||
if report.get('files_missing_in_current', 0) > 0:
|
||||
lines.append(f" {RED}Missing in current: {report.get('files_missing_in_current', 0)}{RESET}")
|
||||
|
||||
if report.get('files_missing_in_baseline', 0) > 0:
|
||||
lines.append(f" {YELLOW}New in current: {report.get('files_missing_in_baseline', 0)}{RESET}")
|
||||
|
||||
# Show first few differences
|
||||
differences = report.get('differences', [])
|
||||
if differences:
|
||||
lines.append(f"\n {RED}Differences:{RESET}")
|
||||
for diff in differences[:5]:
|
||||
file = diff.get('file', 'unknown')
|
||||
reason = diff.get('reason', 'unknown')
|
||||
lines.append(f" • {file}: {reason}")
|
||||
|
||||
if len(differences) > 5:
|
||||
lines.append(f" ... and {len(differences) - 5} more")
|
||||
|
||||
if i < len(reports) - 1:
|
||||
lines.append("") # Spacing between specs
|
||||
|
||||
lines.append(f"\n{BOLD}{'='*60}{RESET}\n")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate summary report from comparison results"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reports",
|
||||
required=True,
|
||||
help="Report file(s) pattern (e.g., 'test/output/diff-report-*.json')"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-file",
|
||||
help="Write markdown summary to file (e.g., $GITHUB_STEP_SUMMARY)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-console",
|
||||
action="store_true",
|
||||
help="Print colored summary to console"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load reports
|
||||
reports = load_reports(args.reports)
|
||||
|
||||
if not reports:
|
||||
print("Warning: No reports loaded", file=sys.stderr)
|
||||
|
||||
# Generate markdown summary
|
||||
markdown = generate_markdown_summary(reports)
|
||||
|
||||
# Write to file if requested
|
||||
if args.output_file:
|
||||
try:
|
||||
with open(args.output_file, 'w') as f:
|
||||
f.write(markdown)
|
||||
print(f"Summary written to {args.output_file}")
|
||||
except IOError as e:
|
||||
print(f"Error writing summary to {args.output_file}: {e}", file=sys.stderr)
|
||||
|
||||
# Print to console if requested
|
||||
if args.output_console:
|
||||
console = generate_console_summary(reports)
|
||||
print(console)
|
||||
|
||||
# If neither output option specified, print to stdout
|
||||
if not args.output_file and not args.output_console:
|
||||
print(markdown)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+158
@@ -0,0 +1,158 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Helper script to update regression test baselines
|
||||
# Usage: ./scripts/update_baselines.sh [run-id]
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${BLUE}===========================================${NC}"
|
||||
echo -e "${BLUE}Regression Test Baseline Update Script${NC}"
|
||||
echo -e "${BLUE}===========================================${NC}\n"
|
||||
|
||||
# Check if gh CLI is installed
|
||||
if ! command -v gh &> /dev/null; then
|
||||
echo -e "${RED}Error: GitHub CLI (gh) not found${NC}"
|
||||
echo "Install from: https://cli.github.com/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get run ID from argument or prompt
|
||||
if [ -n "$1" ]; then
|
||||
RUN_ID="$1"
|
||||
else
|
||||
echo -e "${YELLOW}Enter GitHub Actions run ID (or leave empty for latest):${NC}"
|
||||
read -r RUN_ID
|
||||
fi
|
||||
|
||||
# If no run ID provided, get the latest regression-test workflow run
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
echo "Fetching latest regression-test workflow run..."
|
||||
RUN_ID=$(gh run list --workflow=regression-test.yaml --limit 1 --json databaseId --jq '.[0].databaseId')
|
||||
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
echo -e "${RED}Error: No workflow runs found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "Using latest run: ${GREEN}${RUN_ID}${NC}"
|
||||
fi
|
||||
|
||||
# Create temp directory
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf $TEMP_DIR" EXIT
|
||||
|
||||
echo -e "\n${BLUE}Step 1: Downloading artifacts...${NC}"
|
||||
|
||||
# Download artifacts from the run
|
||||
cd "$TEMP_DIR"
|
||||
if ! gh run download "$RUN_ID" --name "regression-test-results-${RUN_ID}-1" 2>/dev/null; then
|
||||
# Try without attempt suffix
|
||||
if ! gh run download "$RUN_ID" 2>/dev/null; then
|
||||
echo -e "${RED}Error: Failed to download artifacts from run ${RUN_ID}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✓ Artifacts downloaded${NC}"
|
||||
|
||||
# Check which bundles are present
|
||||
echo -e "\n${BLUE}Step 2: Checking available bundles...${NC}"
|
||||
|
||||
V1BETA3_BUNDLE=""
|
||||
V1BETA2_BUNDLE=""
|
||||
SUPPORTBUNDLE=""
|
||||
|
||||
if [ -f "preflight-v1beta3-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta3-bundle.tar.gz" ]; then
|
||||
V1BETA3_BUNDLE=$(find . -name "preflight-v1beta3-bundle.tar.gz" | head -1)
|
||||
echo -e "${GREEN}✓${NC} Found v1beta3 preflight bundle"
|
||||
fi
|
||||
|
||||
if [ -f "preflight-v1beta2-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta2-bundle.tar.gz" ]; then
|
||||
V1BETA2_BUNDLE=$(find . -name "preflight-v1beta2-bundle.tar.gz" | head -1)
|
||||
echo -e "${GREEN}✓${NC} Found v1beta2 preflight bundle"
|
||||
fi
|
||||
|
||||
if [ -f "supportbundle.tar.gz" ] || [ -f "test/output/supportbundle.tar.gz" ]; then
|
||||
SUPPORTBUNDLE=$(find . -name "supportbundle.tar.gz" | head -1)
|
||||
echo -e "${GREEN}✓${NC} Found support bundle"
|
||||
fi
|
||||
|
||||
if [ -z "$V1BETA3_BUNDLE" ] && [ -z "$V1BETA2_BUNDLE" ] && [ -z "$SUPPORTBUNDLE" ]; then
|
||||
echo -e "${RED}Error: No bundles found in artifacts${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Confirm update
|
||||
echo -e "\n${YELLOW}This will update the following baselines:${NC}"
|
||||
[ -n "$V1BETA3_BUNDLE" ] && echo " - test/baselines/preflight-v1beta3/baseline.tar.gz"
|
||||
[ -n "$V1BETA2_BUNDLE" ] && echo " - test/baselines/preflight-v1beta2/baseline.tar.gz"
|
||||
[ -n "$SUPPORTBUNDLE" ] && echo " - test/baselines/supportbundle/baseline.tar.gz"
|
||||
|
||||
echo -e "\n${YELLOW}Continue? (y/N):${NC} "
|
||||
read -r CONFIRM
|
||||
|
||||
if [ "$CONFIRM" != "y" ] && [ "$CONFIRM" != "Y" ]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Get project root (assuming script is in scripts/)
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
echo -e "\n${BLUE}Step 3: Updating baselines...${NC}"
|
||||
|
||||
# Update v1beta3 baseline
|
||||
if [ -n "$V1BETA3_BUNDLE" ]; then
|
||||
mkdir -p test/baselines/preflight-v1beta3
|
||||
cp "$TEMP_DIR/$V1BETA3_BUNDLE" test/baselines/preflight-v1beta3/baseline.tar.gz
|
||||
echo -e "${GREEN}✓${NC} Updated preflight-v1beta3 baseline"
|
||||
fi
|
||||
|
||||
# Update v1beta2 baseline
|
||||
if [ -n "$V1BETA2_BUNDLE" ]; then
|
||||
mkdir -p test/baselines/preflight-v1beta2
|
||||
cp "$TEMP_DIR/$V1BETA2_BUNDLE" test/baselines/preflight-v1beta2/baseline.tar.gz
|
||||
echo -e "${GREEN}✓${NC} Updated preflight-v1beta2 baseline"
|
||||
fi
|
||||
|
||||
# Update support bundle baseline
|
||||
if [ -n "$SUPPORTBUNDLE" ]; then
|
||||
mkdir -p test/baselines/supportbundle
|
||||
cp "$TEMP_DIR/$SUPPORTBUNDLE" test/baselines/supportbundle/baseline.tar.gz
|
||||
echo -e "${GREEN}✓${NC} Updated supportbundle baseline"
|
||||
fi
|
||||
|
||||
# Create metadata file
|
||||
echo -e "\n${BLUE}Step 4: Creating metadata...${NC}"
|
||||
|
||||
GIT_SHA=$(git rev-parse HEAD)
|
||||
CURRENT_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
|
||||
cat > test/baselines/metadata.json <<EOF
|
||||
{
|
||||
"updated_at": "$CURRENT_DATE",
|
||||
"git_sha": "$GIT_SHA",
|
||||
"workflow_run_id": "$RUN_ID",
|
||||
"k8s_version": "v1.28.3",
|
||||
"updated_by": "$(git config user.name) <$(git config user.email)>"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}✓${NC} Created metadata.json"
|
||||
|
||||
# Show git status
|
||||
echo -e "\n${BLUE}Step 5: Git status${NC}"
|
||||
git status test/baselines/
|
||||
|
||||
echo -e "\n${YELLOW}Review the changes above. To commit:${NC}"
|
||||
echo -e " ${BLUE}git add test/baselines/${NC}"
|
||||
echo -e " ${BLUE}git commit -m 'chore: update regression baselines from run ${RUN_ID}'${NC}"
|
||||
echo -e " ${BLUE}git push${NC}"
|
||||
|
||||
echo -e "\n${GREEN}Done!${NC}"
|
||||
@@ -0,0 +1,10 @@
|
||||
# Ignore test outputs (bundles are large, should be in artifacts)
|
||||
output/
|
||||
|
||||
# Ignore extracted bundle contents during local testing
|
||||
extracted/
|
||||
tmp/
|
||||
*.tmp
|
||||
|
||||
# Ignore local test runs
|
||||
local-test-*/
|
||||
+280
@@ -0,0 +1,280 @@
|
||||
# Regression Test Suite
|
||||
|
||||
This directory contains the regression test infrastructure for validating preflight and support bundle collectors.
|
||||
|
||||
## Overview
|
||||
|
||||
The regression test suite:
|
||||
1. Provisions an ephemeral k3s cluster via Replicated Actions
|
||||
2. Runs multiple preflight and support bundle specs
|
||||
3. Compares output bundles against known-good baselines
|
||||
4. Reports regressions (missing files, changed outputs)
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
test/
|
||||
├── README.md # This file
|
||||
├── baselines/ # Known-good baseline bundles
|
||||
│ ├── preflight-v1beta3/
|
||||
│ │ └── baseline.tar.gz
|
||||
│ ├── preflight-v1beta2/
|
||||
│ │ └── baseline.tar.gz
|
||||
│ ├── supportbundle/
|
||||
│ │ └── baseline.tar.gz
|
||||
│ └── metadata.json # Baseline metadata (git sha, date, k8s version)
|
||||
└── output/ # Test run outputs (gitignored)
|
||||
├── preflight-v1beta3-bundle.tar.gz
|
||||
├── preflight-v1beta2-bundle.tar.gz
|
||||
├── supportbundle.tar.gz
|
||||
└── diff-report-*.json
|
||||
```
|
||||
|
||||
## Specs Under Test
|
||||
|
||||
| Spec | File | Values | Description |
|
||||
|------|------|--------|-------------|
|
||||
| Preflight v1beta3 | `examples/preflight/complex-v1beta3.yaml` | `examples/preflight/values-complex-full.yaml` | Templated v1beta3 with ~30 analyzers |
|
||||
| Preflight v1beta2 | `examples/preflight/all-analyzers-v1beta2.yaml` | N/A | Legacy v1beta2 format with all analyzer types |
|
||||
| Support Bundle | `examples/collect/host/all-kubernetes-collectors.yaml` | N/A | Comprehensive collector suite |
|
||||
|
||||
## Running Tests
|
||||
|
||||
### Via GitHub Actions (Recommended)
|
||||
|
||||
The regression test workflow runs automatically on:
|
||||
- Push to `main` or `v1beta3` branches
|
||||
- Pull requests
|
||||
- Manual trigger via workflow_dispatch
|
||||
|
||||
**Manual trigger:**
|
||||
```bash
|
||||
gh workflow run regression-test.yaml
|
||||
```
|
||||
|
||||
### Locally (Manual)
|
||||
|
||||
```bash
|
||||
# 1. Build binaries
|
||||
make bin/preflight bin/support-bundle
|
||||
|
||||
# 2. Create k3s cluster (use your preferred method)
|
||||
k3d cluster create test-cluster --wait
|
||||
|
||||
# 3. Run specs
|
||||
./bin/preflight examples/preflight/complex-v1beta3.yaml \
|
||||
--values examples/preflight/values-complex-full.yaml \
|
||||
--interactive=false
|
||||
|
||||
./bin/preflight examples/preflight/all-analyzers-v1beta2.yaml \
|
||||
--interactive=false
|
||||
|
||||
./bin/support-bundle examples/collect/host/all-kubernetes-collectors.yaml \
|
||||
--interactive=false
|
||||
|
||||
# 4. Compare bundles (if baselines exist)
|
||||
python3 scripts/compare_bundles.py \
|
||||
--baseline test/baselines/preflight-v1beta3/baseline.tar.gz \
|
||||
--current preflightbundle-*.tar.gz \
|
||||
--rules scripts/compare_rules.yaml \
|
||||
--report test/output/diff-report.json \
|
||||
--spec-type preflight
|
||||
|
||||
# 5. Clean up
|
||||
k3d cluster delete test-cluster
|
||||
```
|
||||
|
||||
## Creating Initial Baselines
|
||||
|
||||
If baselines don't exist yet (first time setup):
|
||||
|
||||
1. **Run workflow to generate bundles:**
|
||||
```bash
|
||||
gh workflow run regression-test.yaml
|
||||
```
|
||||
|
||||
2. **Download artifacts:**
|
||||
```bash
|
||||
gh run download <run-id> --name regression-test-results-<run-id>-1
|
||||
```
|
||||
|
||||
3. **Inspect bundles manually:**
|
||||
```bash
|
||||
tar -tzf preflight-v1beta3-bundle.tar.gz | head -20
|
||||
tar -xzf preflight-v1beta3-bundle.tar.gz
|
||||
# Verify contents look correct
|
||||
```
|
||||
|
||||
4. **Copy as baselines and commit:**
|
||||
```bash
|
||||
mkdir -p test/baselines/{preflight-v1beta3,preflight-v1beta2,supportbundle}
|
||||
|
||||
cp preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz
|
||||
cp preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz
|
||||
cp supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz
|
||||
|
||||
git add test/baselines/
|
||||
git commit -m "chore: add initial regression test baselines"
|
||||
git push
|
||||
```
|
||||
|
||||
## Updating Baselines
|
||||
|
||||
When legitimate changes occur (new collectors, changed output format):
|
||||
|
||||
### Option 1: Automatic Update (Workflow Input)
|
||||
|
||||
```bash
|
||||
gh workflow run regression-test.yaml -f update_baselines=true
|
||||
```
|
||||
|
||||
This will:
|
||||
1. Run tests
|
||||
2. Copy new bundles as baselines
|
||||
3. Commit and push updated baselines
|
||||
|
||||
** Use with caution!** Only use this after verifying changes are intentional.
|
||||
|
||||
### Option 2: Manual Update
|
||||
|
||||
```bash
|
||||
# Download artifacts from a successful run
|
||||
gh run download <run-id> --name regression-test-results-<run-id>-1
|
||||
|
||||
# Replace baselines
|
||||
cp preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz
|
||||
cp preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz
|
||||
cp supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz
|
||||
|
||||
# Commit
|
||||
git add test/baselines/
|
||||
git commit -m "chore: update regression baselines - reason for change"
|
||||
git push
|
||||
```
|
||||
|
||||
## Comparison Strategy
|
||||
|
||||
The comparison uses a 3-tier approach:
|
||||
|
||||
### 1. Exact Match (2 files)
|
||||
Files compared byte-for-byte:
|
||||
- `static-data.txt/static-data` - static data collector
|
||||
- `version.yaml` - spec version
|
||||
- Data collector files (`files/example.yaml`, `config/replicas.txt`)
|
||||
|
||||
### 2. Structural Comparison (8 files)
|
||||
Compare specific fields only, ignore variable values:
|
||||
- **Database collectors** (`postgres/*.json`, `mysql/*.json`, etc.) - Compare `isConnected` boolean
|
||||
- **DNS** (`dns/debug.json`) - Verify service exists, queries succeed
|
||||
- **Registry** (`registry/*.json`) - Compare `exists` per image
|
||||
- **HTTP** (`http*.json`) - Compare status code only
|
||||
|
||||
### 3. Non-Empty Check (Everything Else)
|
||||
For highly variable outputs:
|
||||
- **cluster-resources** - UIDs, timestamps, resourceVersions vary
|
||||
- **node-metrics** - All metric values constantly change
|
||||
- **logs** - Timestamps in every line
|
||||
- **run/exec collectors** - Random pod names, variable output
|
||||
- And more...
|
||||
|
||||
Strategy: Verify file exists, is non-empty, and (for JSON) is valid JSON.
|
||||
|
||||
## Understanding Test Results
|
||||
|
||||
### Passing Test
|
||||
- All expected files present
|
||||
- Exact match files identical
|
||||
- Structural comparison fields match
|
||||
- All files non-empty and valid
|
||||
|
||||
### Failing Test - Regressions Detected
|
||||
|
||||
**Files missing:**
|
||||
```
|
||||
⚠ Missing in current: postgres/postgres-example.json
|
||||
```
|
||||
→ Collector stopped producing output (regression)
|
||||
|
||||
**Structural mismatch:**
|
||||
```
|
||||
❌ postgres/postgres-example.json: database connection status changed: true -> false
|
||||
```
|
||||
→ Collector behavior changed (potential regression)
|
||||
|
||||
**Empty file:**
|
||||
```
|
||||
❌ dns/debug.json: File is empty
|
||||
```
|
||||
→ Collector failed to collect data (regression)
|
||||
|
||||
### ℹ️ New Files (Not a Failure)
|
||||
```
|
||||
ℹ New file in current: newcollector/output.json
|
||||
```
|
||||
→ New collector added (expected when adding features)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Workflow fails: "No baseline found"
|
||||
First time setup - baselines need to be created (see above).
|
||||
|
||||
### Many "structural mismatch" failures
|
||||
Check if cluster state changed:
|
||||
- Different k8s version?
|
||||
- Different installed components?
|
||||
- Resources created/deleted?
|
||||
|
||||
### Comparison fails with Python error
|
||||
Ensure dependencies installed:
|
||||
```bash
|
||||
pip install pyyaml deepdiff
|
||||
```
|
||||
|
||||
### Cluster creation times out
|
||||
Check Replicated Actions limits:
|
||||
```bash
|
||||
# View cluster status
|
||||
gh api /repos/replicatedhq/compatibility-actions/...
|
||||
```
|
||||
|
||||
## Configuration Files
|
||||
|
||||
### `scripts/compare_rules.yaml`
|
||||
Defines comparison strategy per file pattern.
|
||||
|
||||
**Add new rule:**
|
||||
```yaml
|
||||
preflight:
|
||||
structural_compare:
|
||||
"mycollector/*.json": "my_comparator_function"
|
||||
```
|
||||
|
||||
Then implement `_compare_my_comparator_function()` in `scripts/compare_bundles.py`.
|
||||
|
||||
### `scripts/compare_bundles.py`
|
||||
Comparison engine - implements comparison logic.
|
||||
|
||||
**Add new comparator:**
|
||||
```python
|
||||
def _compare_my_comparator_function(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare mycollector output."""
|
||||
# Your comparison logic
|
||||
return baseline["field"] == current["field"]
|
||||
```
|
||||
|
||||
### `.github/workflows/regression-test.yaml`
|
||||
GitHub Actions workflow definition.
|
||||
|
||||
## Tips
|
||||
|
||||
- **Start simple**: Begin with baselines for v1beta2 only, add v1beta3 later
|
||||
- **Iterate on rules**: Add structural comparisons as you discover false positives
|
||||
- **Review diffs**: Always inspect diff reports before updating baselines
|
||||
- **Document changes**: In baseline update commits, explain why output changed
|
||||
- **Monitor runtime**: Workflow should complete in < 20 minutes
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [CI Regression Test Proposal](../ci-regression-test-proposal.md)
|
||||
- [Collector Comparison Strategy](../collector-comparison-strategy.md)
|
||||
- [Replicated Actions Docs](https://github.com/replicatedhq/replicated-actions)
|
||||
@@ -0,0 +1,33 @@
|
||||
# Regression Test Baselines
|
||||
|
||||
This directory contains known-good baseline bundles used for regression testing.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
- `preflight-v1beta3/` - Baseline for complex-v1beta3.yaml spec
|
||||
- `preflight-v1beta2/` - Baseline for all-analyzers-v1beta2.yaml spec
|
||||
- `supportbundle/` - Baseline for all-kubernetes-collectors.yaml spec
|
||||
- `metadata.json` - Metadata about when baselines were last updated
|
||||
|
||||
## Creating Initial Baselines
|
||||
|
||||
If this directory is empty, baselines need to be created:
|
||||
|
||||
1. Run the regression test workflow manually
|
||||
2. Download the artifacts
|
||||
3. Inspect bundles to verify correctness
|
||||
4. Use `scripts/update_baselines.sh` to copy them here
|
||||
|
||||
See `test/README.md` for detailed instructions.
|
||||
|
||||
## Updating Baselines
|
||||
|
||||
Baselines should only be updated when:
|
||||
- New collectors are added
|
||||
- Collector output format changes intentionally
|
||||
- Kubernetes version is upgraded
|
||||
- Bug fixes that change collector behavior
|
||||
|
||||
**Never update baselines to make failing tests pass without investigation!**
|
||||
|
||||
Use `scripts/update_baselines.sh` to update from a workflow run.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user