mirror of
https://github.com/replicatedhq/troubleshoot.git
synced 2026-04-15 07:16:34 +00:00
Add regression test suite to github actions
This commit is contained in:
Noah Campbell
Executable
+515
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Bundle comparison engine for regression testing.
|
||||
|
||||
Unpacks baseline and current bundles, applies comparison rules,
|
||||
generates diff report, and exits non-zero on regressions.
|
||||
|
||||
Based on the simplified 3-tier approach:
|
||||
1. EXACT match for deterministic files (static data, version.yaml)
|
||||
2. STRUCTURAL comparison for semi-deterministic files (databases, DNS, etc.)
|
||||
3. NON-EMPTY check for variable files (cluster-resources, metrics, logs)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional
|
||||
import fnmatch
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("Error: pyyaml not installed. Run: pip install pyyaml")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from deepdiff import DeepDiff
|
||||
except ImportError:
|
||||
print("Error: deepdiff not installed. Run: pip install deepdiff")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
class BundleComparator:
|
||||
"""Compare two troubleshoot bundles using rule-based comparison."""
|
||||
|
||||
def __init__(self, rules_path: str, spec_type: str):
|
||||
self.rules = self._load_rules(rules_path, spec_type)
|
||||
self.spec_type = spec_type
|
||||
self.results = {
|
||||
"spec_type": spec_type,
|
||||
"files_compared": 0,
|
||||
"exact_matches": 0,
|
||||
"structural_matches": 0,
|
||||
"non_empty_checks": 0,
|
||||
"files_different": 0,
|
||||
"files_missing_in_current": 0,
|
||||
"files_missing_in_baseline": 0,
|
||||
"differences": [],
|
||||
"missing_in_current": [],
|
||||
"missing_in_baseline": [],
|
||||
}
|
||||
|
||||
def _load_rules(self, rules_path: str, spec_type: str) -> Dict:
|
||||
"""Load comparison rules from YAML file."""
|
||||
if not Path(rules_path).exists():
|
||||
print(f"Warning: Rules file not found at {rules_path}, using defaults")
|
||||
return self._get_default_rules()
|
||||
|
||||
with open(rules_path) as f:
|
||||
rules = yaml.safe_load(f)
|
||||
|
||||
return rules.get(spec_type, rules.get("defaults", {}))
|
||||
|
||||
def _get_default_rules(self) -> Dict:
|
||||
"""Return default comparison rules if no config file."""
|
||||
return {
|
||||
"exact_match": [
|
||||
"static-data.txt/static-data",
|
||||
"version.yaml",
|
||||
],
|
||||
"structural_compare": {
|
||||
"postgres/*.json": "database_connection",
|
||||
"mysql/*.json": "database_connection",
|
||||
"mssql/*.json": "database_connection",
|
||||
"redis/*.json": "database_connection",
|
||||
"dns/debug.json": "dns_structure",
|
||||
"registry/*.json": "registry_exists",
|
||||
"http*.json": "http_status",
|
||||
},
|
||||
"non_empty_default": True,
|
||||
}
|
||||
|
||||
def compare(self, baseline_bundle: str, current_bundle: str) -> bool:
|
||||
"""
|
||||
Compare two bundles. Returns True if no regressions detected.
|
||||
|
||||
Args:
|
||||
baseline_bundle: Path to baseline bundle tar.gz
|
||||
current_bundle: Path to current bundle tar.gz
|
||||
|
||||
Returns:
|
||||
True if bundles match (no regressions), False otherwise
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
baseline_dir = Path(tmpdir) / "baseline"
|
||||
current_dir = Path(tmpdir) / "current"
|
||||
|
||||
print(f"Extracting baseline bundle to {baseline_dir}...")
|
||||
self._extract(baseline_bundle, baseline_dir)
|
||||
|
||||
print(f"Extracting current bundle to {current_dir}...")
|
||||
self._extract(current_bundle, current_dir)
|
||||
|
||||
baseline_files = self._get_file_list(baseline_dir)
|
||||
current_files = self._get_file_list(current_dir)
|
||||
|
||||
print(f"Baseline files: {len(baseline_files)}")
|
||||
print(f"Current files: {len(current_files)}")
|
||||
|
||||
# Check for missing files
|
||||
missing_in_current = baseline_files - current_files
|
||||
missing_in_baseline = current_files - baseline_files
|
||||
|
||||
# Filter out optional files that may not exist (previous logs, etc.)
|
||||
optional_patterns = [
|
||||
"*-previous.log", # Previous container logs (only exist after restart)
|
||||
]
|
||||
|
||||
for file in sorted(missing_in_current):
|
||||
# Skip optional files
|
||||
if any(file.match(pattern) for pattern in optional_patterns):
|
||||
print(f" ℹ Optional file missing (OK): {file}")
|
||||
continue
|
||||
self._record_missing("current", str(file))
|
||||
|
||||
for file in sorted(missing_in_baseline):
|
||||
# Optional files added in current are also OK
|
||||
if any(file.match(pattern) for pattern in optional_patterns):
|
||||
print(f" ℹ Optional file added (OK): {file}")
|
||||
continue
|
||||
self._record_missing("baseline", str(file))
|
||||
|
||||
# Compare common files
|
||||
common_files = baseline_files & current_files
|
||||
print(f"Comparing {len(common_files)} common files...")
|
||||
|
||||
for file in sorted(common_files):
|
||||
self._compare_file(
|
||||
baseline_dir / file,
|
||||
current_dir / file,
|
||||
str(file)
|
||||
)
|
||||
|
||||
# Determine if there are regressions
|
||||
has_regressions = (
|
||||
self.results["files_different"] > 0 or
|
||||
self.results["files_missing_in_current"] > 0
|
||||
)
|
||||
|
||||
return not has_regressions
|
||||
|
||||
def _extract(self, bundle_path: str, dest_dir: Path):
|
||||
"""Extract tar.gz bundle to destination directory."""
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tarfile.open(bundle_path, 'r:gz') as tar:
|
||||
tar.extractall(dest_dir)
|
||||
|
||||
# Handle bundles that extract to a nested directory (e.g., preflightbundle-timestamp/)
|
||||
# If there's only one directory at the root, use that as the actual root
|
||||
items = list(dest_dir.iterdir())
|
||||
if len(items) == 1 and items[0].is_dir():
|
||||
# Move contents up one level
|
||||
nested_dir = items[0]
|
||||
for item in nested_dir.iterdir():
|
||||
item.rename(dest_dir / item.name)
|
||||
nested_dir.rmdir()
|
||||
|
||||
def _get_file_list(self, dir_path: Path) -> set:
|
||||
"""Get set of all files in directory (relative paths)."""
|
||||
files = set()
|
||||
for path in dir_path.rglob('*'):
|
||||
if path.is_file():
|
||||
rel_path = path.relative_to(dir_path)
|
||||
files.add(rel_path)
|
||||
return files
|
||||
|
||||
def _compare_file(self, baseline_path: Path, current_path: Path, rel_path: str):
|
||||
"""Compare a single file pair using appropriate rule."""
|
||||
self.results["files_compared"] += 1
|
||||
|
||||
# Determine comparison mode
|
||||
mode = self._get_comparison_mode(rel_path)
|
||||
|
||||
try:
|
||||
if mode == "exact":
|
||||
if self._compare_exact(baseline_path, current_path):
|
||||
self.results["exact_matches"] += 1
|
||||
else:
|
||||
self._record_diff(rel_path, "exact", "Content mismatch")
|
||||
|
||||
elif mode == "structural":
|
||||
comparator = self._get_structural_comparator(rel_path)
|
||||
if self._compare_structural(baseline_path, current_path, comparator):
|
||||
self.results["structural_matches"] += 1
|
||||
else:
|
||||
self._record_diff(rel_path, "structural", f"Structural comparison failed ({comparator})")
|
||||
|
||||
else: # non_empty
|
||||
if self._check_non_empty(current_path):
|
||||
self.results["non_empty_checks"] += 1
|
||||
else:
|
||||
self._record_diff(rel_path, "non_empty", "File is empty")
|
||||
|
||||
except Exception as e:
|
||||
self._record_diff(rel_path, "error", f"Comparison error: {str(e)}")
|
||||
|
||||
def _get_comparison_mode(self, rel_path: str) -> str:
|
||||
"""Determine comparison mode for a file based on rules."""
|
||||
# Check exact match patterns
|
||||
for pattern in self.rules.get("exact_match", []):
|
||||
if fnmatch.fnmatch(rel_path, pattern) or rel_path == pattern:
|
||||
return "exact"
|
||||
|
||||
# Check structural comparison patterns
|
||||
for pattern in self.rules.get("structural_compare", {}).keys():
|
||||
if fnmatch.fnmatch(rel_path, pattern):
|
||||
return "structural"
|
||||
|
||||
# Default: non-empty check
|
||||
return "non_empty"
|
||||
|
||||
def _get_structural_comparator(self, rel_path: str) -> str:
|
||||
"""Get the structural comparator name for a file."""
|
||||
for pattern, comparator in self.rules.get("structural_compare", {}).items():
|
||||
if fnmatch.fnmatch(rel_path, pattern):
|
||||
return comparator
|
||||
return "unknown"
|
||||
|
||||
def _compare_exact(self, baseline_path: Path, current_path: Path) -> bool:
|
||||
"""Compare files byte-for-byte."""
|
||||
return baseline_path.read_bytes() == current_path.read_bytes()
|
||||
|
||||
def _compare_structural(self, baseline_path: Path, current_path: Path, comparator: str) -> bool:
|
||||
"""Compare files using structural comparator."""
|
||||
# Load JSON data
|
||||
try:
|
||||
baseline_data = json.loads(baseline_path.read_text())
|
||||
current_data = json.loads(current_path.read_text())
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" JSON decode error: {e}")
|
||||
return False
|
||||
|
||||
# Apply comparator
|
||||
if comparator == "database_connection":
|
||||
return self._compare_database_connection(baseline_data, current_data)
|
||||
elif comparator == "dns_structure":
|
||||
return self._compare_dns_structure(baseline_data, current_data)
|
||||
elif comparator == "registry_exists":
|
||||
return self._compare_registry_exists(baseline_data, current_data)
|
||||
elif comparator == "http_status":
|
||||
return self._compare_http_status(baseline_data, current_data)
|
||||
elif comparator == "cluster_version":
|
||||
return self._compare_cluster_version(baseline_data, current_data)
|
||||
elif comparator == "analysis_results":
|
||||
return self._compare_analysis_results(baseline_data, current_data)
|
||||
else:
|
||||
# Unknown comparator - fall back to non-empty
|
||||
return True
|
||||
|
||||
def _compare_database_connection(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare database connection results (isConnected field only)."""
|
||||
b_connected = baseline.get("isConnected", False)
|
||||
c_connected = current.get("isConnected", False)
|
||||
|
||||
if b_connected != c_connected:
|
||||
print(f" Database connection status changed: {b_connected} -> {c_connected}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_dns_structure(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare DNS structure (service exists, query succeeds)."""
|
||||
# Check kubernetes service exists
|
||||
if "query" not in current or "kubernetes" not in current["query"]:
|
||||
print(f" DNS query.kubernetes missing")
|
||||
return False
|
||||
|
||||
# Kubernetes ClusterIP should exist (don't compare value, it can vary)
|
||||
if not current["query"]["kubernetes"].get("address"):
|
||||
print(f" DNS kubernetes.address is empty")
|
||||
return False
|
||||
|
||||
# DNS service should exist
|
||||
if not current.get("kubeDNSService"):
|
||||
print(f" DNS kubeDNSService is empty")
|
||||
return False
|
||||
|
||||
# At least one DNS pod should exist
|
||||
if not current.get("kubeDNSPods") or len(current["kubeDNSPods"]) == 0:
|
||||
print(f" DNS kubeDNSPods is empty")
|
||||
return False
|
||||
|
||||
# Non-resolvable domain should be empty
|
||||
if current.get("query", {}).get("nonResolvableDomain", {}).get("address"):
|
||||
print(f" DNS nonResolvableDomain should be empty")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_registry_exists(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare registry image existence (exists boolean per image)."""
|
||||
baseline_images = baseline.get("images", {})
|
||||
current_images = current.get("images", {})
|
||||
|
||||
# Check same images are present
|
||||
if set(baseline_images.keys()) != set(current_images.keys()):
|
||||
print(f" Registry image list changed")
|
||||
print(f" Baseline: {sorted(baseline_images.keys())}")
|
||||
print(f" Current: {sorted(current_images.keys())}")
|
||||
return False
|
||||
|
||||
# Compare exists status for each image
|
||||
for image_name in baseline_images:
|
||||
b_exists = baseline_images[image_name].get("exists", False)
|
||||
c_exists = current_images[image_name].get("exists", False)
|
||||
|
||||
if b_exists != c_exists:
|
||||
print(f" Registry image '{image_name}' existence changed: {b_exists} -> {c_exists}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_http_status(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare HTTP response (status code only)."""
|
||||
b_status = baseline.get("response", {}).get("status", 0)
|
||||
c_status = current.get("response", {}).get("status", 0)
|
||||
|
||||
if b_status != c_status:
|
||||
print(f" HTTP status changed: {b_status} -> {c_status}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _compare_cluster_version(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare cluster version (major/minor only, ignore build details)."""
|
||||
b_info = baseline.get("info", {})
|
||||
c_info = current.get("info", {})
|
||||
|
||||
# Compare major and minor version
|
||||
if b_info.get("major") != c_info.get("major"):
|
||||
print(f" Cluster major version changed: {b_info.get('major')} -> {c_info.get('major')}")
|
||||
return False
|
||||
|
||||
if b_info.get("minor") != c_info.get("minor"):
|
||||
print(f" Cluster minor version changed: {b_info.get('minor')} -> {c_info.get('minor')}")
|
||||
return False
|
||||
|
||||
# Don't compare: gitVersion, gitCommit, buildDate, goVersion (these vary with k3s updates)
|
||||
return True
|
||||
|
||||
def _compare_analysis_results(self, baseline: Dict, current: Dict) -> bool:
|
||||
"""Compare analysis results (analyzer names and count, not specific messages)."""
|
||||
if not isinstance(baseline, list) or not isinstance(current, list):
|
||||
print(f" Analysis results structure changed (expected list)")
|
||||
return False
|
||||
|
||||
# Create map of analyzer name -> severity for comparison
|
||||
baseline_results = {item.get("name"): item.get("severity") for item in baseline if "name" in item}
|
||||
current_results = {item.get("name"): item.get("severity") for item in current if "name" in item}
|
||||
|
||||
# Check if same analyzers ran
|
||||
baseline_names = set(baseline_results.keys())
|
||||
current_names = set(current_results.keys())
|
||||
|
||||
if baseline_names != current_names:
|
||||
missing = baseline_names - current_names
|
||||
extra = current_names - baseline_names
|
||||
if missing:
|
||||
print(f" Missing analyzers: {missing}")
|
||||
if extra:
|
||||
print(f" New analyzers: {extra}")
|
||||
return False
|
||||
|
||||
# Check if severity levels changed significantly (error/warn differences matter)
|
||||
significant_changes = []
|
||||
for name in baseline_names:
|
||||
b_sev = baseline_results[name]
|
||||
c_sev = current_results[name]
|
||||
|
||||
# Only care if error/warn status changes, not debug
|
||||
if b_sev != c_sev:
|
||||
if b_sev in ["error", "warn"] or c_sev in ["error", "warn"]:
|
||||
significant_changes.append(f"{name}: {b_sev} -> {c_sev}")
|
||||
|
||||
if significant_changes:
|
||||
print(f" Analyzer severity changed:")
|
||||
for change in significant_changes[:5]: # Show first 5
|
||||
print(f" {change}")
|
||||
# Don't fail on severity changes - this is informational
|
||||
# return False
|
||||
|
||||
return True
|
||||
|
||||
def _check_non_empty(self, path: Path) -> bool:
|
||||
"""Check that file exists and is non-empty."""
|
||||
if not path.exists():
|
||||
return False
|
||||
|
||||
size = path.stat().st_size
|
||||
if size == 0:
|
||||
return False
|
||||
|
||||
# Optional: validate JSON structure if .json extension
|
||||
if path.suffix == ".json":
|
||||
try:
|
||||
json.loads(path.read_text())
|
||||
except json.JSONDecodeError:
|
||||
print(f" Invalid JSON: {path.name}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _record_diff(self, file: str, mode: str, reason: str):
|
||||
"""Record a difference/regression."""
|
||||
self.results["files_different"] += 1
|
||||
self.results["differences"].append({
|
||||
"file": file,
|
||||
"mode": mode,
|
||||
"reason": reason
|
||||
})
|
||||
print(f" ❌ {file}: {reason}")
|
||||
|
||||
def _record_missing(self, location: str, file: str):
|
||||
"""Record a missing file."""
|
||||
if location == "current":
|
||||
self.results["files_missing_in_current"] += 1
|
||||
self.results["missing_in_current"].append(file)
|
||||
print(f" ⚠ Missing in current: {file}")
|
||||
else:
|
||||
self.results["files_missing_in_baseline"] += 1
|
||||
self.results["missing_in_baseline"].append(file)
|
||||
print(f" ℹ New file in current: {file}")
|
||||
|
||||
def generate_report(self, output_path: str):
|
||||
"""Write JSON report."""
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump(self.results, f, indent=2)
|
||||
|
||||
print(f"\nReport written to: {output_path}")
|
||||
|
||||
def print_summary(self):
|
||||
"""Print human-readable summary to stdout."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Bundle Comparison Report - {self.spec_type}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Files compared: {self.results['files_compared']}")
|
||||
print(f" Exact matches: {self.results['exact_matches']}")
|
||||
print(f" Structural matches: {self.results['structural_matches']}")
|
||||
print(f" Non-empty checks: {self.results['non_empty_checks']}")
|
||||
print(f"Files different: {self.results['files_different']}")
|
||||
print(f"Missing in current: {self.results['files_missing_in_current']}")
|
||||
print(f"Missing in baseline: {self.results['files_missing_in_baseline']}")
|
||||
|
||||
if self.results["differences"]:
|
||||
print(f"\n❌ REGRESSIONS DETECTED ({len(self.results['differences'])}):")
|
||||
for diff in self.results["differences"][:10]: # Show first 10
|
||||
print(f" • {diff['file']}: {diff['reason']}")
|
||||
if len(self.results["differences"]) > 10:
|
||||
print(f" ... and {len(self.results['differences']) - 10} more")
|
||||
|
||||
if self.results["missing_in_current"]:
|
||||
print(f"\n⚠ MISSING FILES ({len(self.results['missing_in_current'])}):")
|
||||
for file in self.results["missing_in_current"][:5]:
|
||||
print(f" • {file}")
|
||||
if len(self.results["missing_in_current"]) > 5:
|
||||
print(f" ... and {len(self.results['missing_in_current']) - 5} more")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare troubleshoot bundles for regression testing"
|
||||
)
|
||||
parser.add_argument("--baseline", required=True, help="Baseline bundle tar.gz path")
|
||||
parser.add_argument("--current", required=True, help="Current bundle tar.gz path")
|
||||
parser.add_argument("--rules", required=True, help="Comparison rules YAML path")
|
||||
parser.add_argument("--report", required=True, help="Output report JSON path")
|
||||
parser.add_argument(
|
||||
"--spec-type",
|
||||
required=True,
|
||||
choices=["preflight", "supportbundle"],
|
||||
help="Type of spec being compared"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Verify files exist
|
||||
if not Path(args.baseline).exists():
|
||||
print(f"Error: Baseline bundle not found: {args.baseline}")
|
||||
sys.exit(1)
|
||||
|
||||
if not Path(args.current).exists():
|
||||
print(f"Error: Current bundle not found: {args.current}")
|
||||
sys.exit(1)
|
||||
|
||||
# Run comparison
|
||||
comparator = BundleComparator(args.rules, args.spec_type)
|
||||
passed = comparator.compare(args.baseline, args.current)
|
||||
comparator.generate_report(args.report)
|
||||
comparator.print_summary()
|
||||
|
||||
# Exit with appropriate code
|
||||
if passed:
|
||||
print("\n✅ No regressions detected")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("\n❌ Regressions detected!")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,121 @@
|
||||
# Comparison rules for regression testing
|
||||
# Defines how different collector outputs should be compared
|
||||
|
||||
# Global configuration (applies to both preflight and supportbundle)
|
||||
global:
|
||||
# Files that should be compared exactly (byte-for-byte)
|
||||
exact_match:
|
||||
- "static-data.txt/static-data"
|
||||
# Note: version.yaml is NOT here - versionNumber varies between builds
|
||||
|
||||
# Default behavior for unknown files
|
||||
non_empty_default: true
|
||||
|
||||
# Preflight-specific rules
|
||||
preflight:
|
||||
# Files that should be compared exactly
|
||||
exact_match:
|
||||
- "static-data.txt/static-data"
|
||||
- "files/example.yaml" # From data collector in v1beta2
|
||||
- "files/example.json" # From data collector in v1beta2
|
||||
- "config/replicas.txt" # From data collector in v1beta2
|
||||
# Note: version.yaml is NOT here - it uses non-empty check (versionNumber varies)
|
||||
|
||||
# Files that need structural/field-specific comparison
|
||||
# Format: "pattern": "comparator_function_name"
|
||||
structural_compare:
|
||||
# Database collectors - compare isConnected boolean only
|
||||
"postgres/*.json": "database_connection"
|
||||
"mysql/*.json": "database_connection"
|
||||
"mssql/*.json": "database_connection"
|
||||
"redis/*.json": "database_connection"
|
||||
|
||||
# DNS collector - compare structure, not exact values
|
||||
"dns/debug.json": "dns_structure"
|
||||
|
||||
# Registry collector - compare exists boolean per image
|
||||
"registry/*.json": "registry_exists"
|
||||
"registry-images/*.json": "registry_exists"
|
||||
|
||||
# HTTP collector - compare status code only
|
||||
"http/*.json": "http_status"
|
||||
"http-*.json": "http_status"
|
||||
|
||||
# Cluster info - compare major/minor version, ignore build details
|
||||
"cluster-info/cluster_version.json": "cluster_version"
|
||||
|
||||
# Analysis results - compare analyzer names and severity levels only
|
||||
"analysis.json": "analysis_results"
|
||||
|
||||
# Everything else uses non-empty check by default
|
||||
# This includes:
|
||||
# - cluster-resources/**/*.json (UIDs, timestamps vary)
|
||||
# - node-metrics/**/*.json (all values vary)
|
||||
# - goldpinger/**/*.json (latencies vary)
|
||||
# - run*/**/* (pod names, output vary)
|
||||
# - ceph/**/* (status/metrics vary or not installed)
|
||||
# - longhorn/**/* (status/metrics vary or not installed)
|
||||
# - certificates/**/*.json (validity time-based)
|
||||
# - configmaps/**/*.json (can have dynamic values)
|
||||
# - secrets/**/*.json (can have dynamic values)
|
||||
# - sysctl/**/* (some counters vary)
|
||||
# - collectd/**/* (time-series data)
|
||||
# - helm/**/*.json (timestamps, revisions vary)
|
||||
# - logs/**/*.log (timestamps in every line)
|
||||
# - copy*/**/* (content depends on what's copied)
|
||||
|
||||
# Support bundle-specific rules
|
||||
supportbundle:
|
||||
# Files that should be compared exactly
|
||||
exact_match:
|
||||
- "static-data.txt/static-data"
|
||||
# Note: version.yaml is NOT here - it uses non-empty check (versionNumber varies)
|
||||
|
||||
# Files that need structural comparison
|
||||
structural_compare:
|
||||
# Database collectors
|
||||
"postgres/*.json": "database_connection"
|
||||
"mysql/*.json": "database_connection"
|
||||
"mssql/*.json": "database_connection"
|
||||
"redis/*.json": "database_connection"
|
||||
|
||||
# DNS collector
|
||||
"dns/debug.json": "dns_structure"
|
||||
|
||||
# Registry collector
|
||||
"registry/*.json": "registry_exists"
|
||||
"registry-images/*.json": "registry_exists"
|
||||
|
||||
# HTTP collector
|
||||
"http*.json": "http_status"
|
||||
|
||||
# Cluster info
|
||||
"cluster-info/cluster_version.json": "cluster_version"
|
||||
|
||||
# Everything else uses non-empty check (see list above in preflight section)
|
||||
|
||||
# Notes on comparison strategies:
|
||||
#
|
||||
# EXACT MATCH:
|
||||
# - Use for static data that should never change between runs
|
||||
# - Byte-for-byte comparison
|
||||
# - Any difference is a regression
|
||||
#
|
||||
# STRUCTURAL COMPARISON:
|
||||
# - Use for semi-deterministic output with consistent structure but variable values
|
||||
# - Compare specific fields only (e.g., status codes, booleans)
|
||||
# - Ignore timing-dependent or environment-specific values
|
||||
#
|
||||
# NON-EMPTY CHECK (default):
|
||||
# - Use for highly variable output where exact comparison is impractical
|
||||
# - Verifies file exists and is not empty
|
||||
# - For JSON files, also validates JSON is parseable
|
||||
# - Appropriate for:
|
||||
# * Kubernetes resources (UIDs, resourceVersions, timestamps)
|
||||
# * Metrics (all values constantly change)
|
||||
# * Logs (timestamps, dynamic content)
|
||||
# * Generated pod/resource names
|
||||
# * Runtime state (pod status, replica counts)
|
||||
#
|
||||
# This strategy catches major regressions (collectors breaking, files missing)
|
||||
# while avoiding false positives from expected variability.
|
||||
Executable
+280
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate summary report from bundle comparison results.
|
||||
|
||||
Reads JSON diff reports and produces:
|
||||
1. GitHub Actions step summary (Markdown)
|
||||
2. Console output (colored text)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict
|
||||
|
||||
|
||||
def load_reports(report_pattern: str) -> List[Dict]:
|
||||
"""Load all report JSON files matching pattern."""
|
||||
reports = []
|
||||
|
||||
# Handle glob pattern
|
||||
if '*' in report_pattern:
|
||||
report_dir = Path(report_pattern).parent
|
||||
pattern = Path(report_pattern).name
|
||||
|
||||
for report_file in sorted(report_dir.glob(pattern)):
|
||||
try:
|
||||
with open(report_file) as f:
|
||||
report = json.load(f)
|
||||
report['_filename'] = report_file.name
|
||||
reports.append(report)
|
||||
except (json.JSONDecodeError, FileNotFoundError) as e:
|
||||
print(f"Warning: Could not load {report_file}: {e}", file=sys.stderr)
|
||||
else:
|
||||
# Single file
|
||||
try:
|
||||
with open(report_pattern) as f:
|
||||
report = json.load(f)
|
||||
report['_filename'] = Path(report_pattern).name
|
||||
reports.append(report)
|
||||
except (json.JSONDecodeError, FileNotFoundError) as e:
|
||||
print(f"Warning: Could not load {report_pattern}: {e}", file=sys.stderr)
|
||||
|
||||
return reports
|
||||
|
||||
|
||||
def generate_markdown_summary(reports: List[Dict]) -> str:
|
||||
"""Generate GitHub Actions step summary in Markdown format."""
|
||||
lines = []
|
||||
|
||||
lines.append("# 🧪 Regression Test Results")
|
||||
lines.append("")
|
||||
|
||||
if not reports:
|
||||
lines.append("⚠️ No comparison reports found. Baselines may be missing.")
|
||||
return "\n".join(lines)
|
||||
|
||||
# Overall status
|
||||
total_regressions = sum(r.get('files_different', 0) for r in reports)
|
||||
total_missing = sum(r.get('files_missing_in_current', 0) for r in reports)
|
||||
|
||||
if total_regressions > 0 or total_missing > 0:
|
||||
lines.append(f"## ❌ Status: FAILED")
|
||||
lines.append(f"**{total_regressions} file(s) with differences, {total_missing} file(s) missing**")
|
||||
else:
|
||||
lines.append(f"## ✅ Status: PASSED")
|
||||
lines.append("All comparisons passed!")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Per-spec breakdown
|
||||
lines.append("## 📊 Comparison Breakdown")
|
||||
lines.append("")
|
||||
|
||||
for report in reports:
|
||||
spec_type = report.get('spec_type', 'unknown')
|
||||
filename = report.get('_filename', 'unknown')
|
||||
|
||||
# Determine status icon
|
||||
has_regressions = (
|
||||
report.get('files_different', 0) > 0 or
|
||||
report.get('files_missing_in_current', 0) > 0
|
||||
)
|
||||
status_icon = "❌" if has_regressions else "✅"
|
||||
|
||||
lines.append(f"### {status_icon} {spec_type.upper()}")
|
||||
lines.append("")
|
||||
lines.append(f"**Report:** `{filename}`")
|
||||
lines.append("")
|
||||
|
||||
# Stats table
|
||||
lines.append("| Metric | Count |")
|
||||
lines.append("|--------|-------|")
|
||||
lines.append(f"| Files compared | {report.get('files_compared', 0)} |")
|
||||
lines.append(f"| Exact matches | {report.get('exact_matches', 0)} |")
|
||||
lines.append(f"| Structural matches | {report.get('structural_matches', 0)} |")
|
||||
lines.append(f"| Non-empty checks | {report.get('non_empty_checks', 0)} |")
|
||||
lines.append(f"| **Files different** | **{report.get('files_different', 0)}** |")
|
||||
lines.append(f"| **Missing in current** | **{report.get('files_missing_in_current', 0)}** |")
|
||||
lines.append(f"| New in current | {report.get('files_missing_in_baseline', 0)} |")
|
||||
lines.append("")
|
||||
|
||||
# Show differences if any
|
||||
differences = report.get('differences', [])
|
||||
if differences:
|
||||
lines.append("<details>")
|
||||
lines.append(f"<summary>⚠️ Show {len(differences)} difference(s)</summary>")
|
||||
lines.append("")
|
||||
lines.append("| File | Mode | Reason |")
|
||||
lines.append("|------|------|--------|")
|
||||
for diff in differences[:20]: # Limit to 20
|
||||
file = diff.get('file', 'unknown')
|
||||
mode = diff.get('mode', 'unknown')
|
||||
reason = diff.get('reason', 'unknown')
|
||||
lines.append(f"| `{file}` | {mode} | {reason} |")
|
||||
|
||||
if len(differences) > 20:
|
||||
lines.append(f"| ... | ... | *{len(differences) - 20} more differences* |")
|
||||
|
||||
lines.append("")
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
# Show missing files if any
|
||||
missing = report.get('missing_in_current', [])
|
||||
if missing:
|
||||
lines.append("<details>")
|
||||
lines.append(f"<summary>⚠️ Show {len(missing)} missing file(s)</summary>")
|
||||
lines.append("")
|
||||
for file in missing[:20]:
|
||||
lines.append(f"- `{file}`")
|
||||
|
||||
if len(missing) > 20:
|
||||
lines.append(f"- *... and {len(missing) - 20} more*")
|
||||
|
||||
lines.append("")
|
||||
lines.append("</details>")
|
||||
lines.append("")
|
||||
|
||||
# Footer
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("💡 **Tips:**")
|
||||
lines.append("- Download artifacts to inspect bundle contents")
|
||||
lines.append("- Review diff reports for detailed comparison results")
|
||||
lines.append("- Update baselines if changes are intentional (use workflow_dispatch with update_baselines)")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_console_summary(reports: List[Dict]) -> str:
|
||||
"""Generate console output with ANSI colors."""
|
||||
lines = []
|
||||
|
||||
# ANSI color codes
|
||||
RED = "\033[91m"
|
||||
GREEN = "\033[92m"
|
||||
YELLOW = "\033[93m"
|
||||
BLUE = "\033[94m"
|
||||
BOLD = "\033[1m"
|
||||
RESET = "\033[0m"
|
||||
|
||||
lines.append(f"\n{BOLD}{'='*60}{RESET}")
|
||||
lines.append(f"{BOLD}Regression Test Summary{RESET}")
|
||||
lines.append(f"{BOLD}{'='*60}{RESET}\n")
|
||||
|
||||
if not reports:
|
||||
lines.append(f"{YELLOW}⚠ No comparison reports found{RESET}")
|
||||
return "\n".join(lines)
|
||||
|
||||
# Overall status
|
||||
total_regressions = sum(r.get('files_different', 0) for r in reports)
|
||||
total_missing = sum(r.get('files_missing_in_current', 0) for r in reports)
|
||||
|
||||
if total_regressions > 0 or total_missing > 0:
|
||||
lines.append(f"{RED}{BOLD}❌ FAILED{RESET}")
|
||||
lines.append(f" {total_regressions} file(s) with differences")
|
||||
lines.append(f" {total_missing} file(s) missing\n")
|
||||
else:
|
||||
lines.append(f"{GREEN}{BOLD}✅ PASSED{RESET}")
|
||||
lines.append(f" All comparisons successful\n")
|
||||
|
||||
# Per-spec details
|
||||
for i, report in enumerate(reports):
|
||||
spec_type = report.get('spec_type', 'unknown')
|
||||
|
||||
has_regressions = (
|
||||
report.get('files_different', 0) > 0 or
|
||||
report.get('files_missing_in_current', 0) > 0
|
||||
)
|
||||
|
||||
status_color = RED if has_regressions else GREEN
|
||||
status_icon = "❌" if has_regressions else "✅"
|
||||
|
||||
lines.append(f"{BLUE}{BOLD}{spec_type.upper()}{RESET} {status_color}{status_icon}{RESET}")
|
||||
lines.append(f" Files compared: {report.get('files_compared', 0)}")
|
||||
lines.append(f" Exact matches: {report.get('exact_matches', 0)}")
|
||||
lines.append(f" Structural matches: {report.get('structural_matches', 0)}")
|
||||
lines.append(f" Non-empty checks: {report.get('non_empty_checks', 0)}")
|
||||
|
||||
if report.get('files_different', 0) > 0:
|
||||
lines.append(f" {RED}Files different: {report.get('files_different', 0)}{RESET}")
|
||||
|
||||
if report.get('files_missing_in_current', 0) > 0:
|
||||
lines.append(f" {RED}Missing in current: {report.get('files_missing_in_current', 0)}{RESET}")
|
||||
|
||||
if report.get('files_missing_in_baseline', 0) > 0:
|
||||
lines.append(f" {YELLOW}New in current: {report.get('files_missing_in_baseline', 0)}{RESET}")
|
||||
|
||||
# Show first few differences
|
||||
differences = report.get('differences', [])
|
||||
if differences:
|
||||
lines.append(f"\n {RED}Differences:{RESET}")
|
||||
for diff in differences[:5]:
|
||||
file = diff.get('file', 'unknown')
|
||||
reason = diff.get('reason', 'unknown')
|
||||
lines.append(f" • {file}: {reason}")
|
||||
|
||||
if len(differences) > 5:
|
||||
lines.append(f" ... and {len(differences) - 5} more")
|
||||
|
||||
if i < len(reports) - 1:
|
||||
lines.append("") # Spacing between specs
|
||||
|
||||
lines.append(f"\n{BOLD}{'='*60}{RESET}\n")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate summary report from comparison results"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reports",
|
||||
required=True,
|
||||
help="Report file(s) pattern (e.g., 'test/output/diff-report-*.json')"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-file",
|
||||
help="Write markdown summary to file (e.g., $GITHUB_STEP_SUMMARY)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-console",
|
||||
action="store_true",
|
||||
help="Print colored summary to console"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load reports
|
||||
reports = load_reports(args.reports)
|
||||
|
||||
if not reports:
|
||||
print("Warning: No reports loaded", file=sys.stderr)
|
||||
|
||||
# Generate markdown summary
|
||||
markdown = generate_markdown_summary(reports)
|
||||
|
||||
# Write to file if requested
|
||||
if args.output_file:
|
||||
try:
|
||||
with open(args.output_file, 'w') as f:
|
||||
f.write(markdown)
|
||||
print(f"Summary written to {args.output_file}")
|
||||
except IOError as e:
|
||||
print(f"Error writing summary to {args.output_file}: {e}", file=sys.stderr)
|
||||
|
||||
# Print to console if requested
|
||||
if args.output_console:
|
||||
console = generate_console_summary(reports)
|
||||
print(console)
|
||||
|
||||
# If neither output option specified, print to stdout
|
||||
if not args.output_file and not args.output_console:
|
||||
print(markdown)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+158
@@ -0,0 +1,158 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Helper script to update regression test baselines
|
||||
# Usage: ./scripts/update_baselines.sh [run-id]
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${BLUE}===========================================${NC}"
|
||||
echo -e "${BLUE}Regression Test Baseline Update Script${NC}"
|
||||
echo -e "${BLUE}===========================================${NC}\n"
|
||||
|
||||
# Check if gh CLI is installed
|
||||
if ! command -v gh &> /dev/null; then
|
||||
echo -e "${RED}Error: GitHub CLI (gh) not found${NC}"
|
||||
echo "Install from: https://cli.github.com/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get run ID from argument or prompt
|
||||
if [ -n "$1" ]; then
|
||||
RUN_ID="$1"
|
||||
else
|
||||
echo -e "${YELLOW}Enter GitHub Actions run ID (or leave empty for latest):${NC}"
|
||||
read -r RUN_ID
|
||||
fi
|
||||
|
||||
# If no run ID provided, get the latest regression-test workflow run
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
echo "Fetching latest regression-test workflow run..."
|
||||
RUN_ID=$(gh run list --workflow=regression-test.yaml --limit 1 --json databaseId --jq '.[0].databaseId')
|
||||
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
echo -e "${RED}Error: No workflow runs found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "Using latest run: ${GREEN}${RUN_ID}${NC}"
|
||||
fi
|
||||
|
||||
# Create temp directory
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf $TEMP_DIR" EXIT
|
||||
|
||||
echo -e "\n${BLUE}Step 1: Downloading artifacts...${NC}"
|
||||
|
||||
# Download artifacts from the run
|
||||
cd "$TEMP_DIR"
|
||||
if ! gh run download "$RUN_ID" --name "regression-test-results-${RUN_ID}-1" 2>/dev/null; then
|
||||
# Try without attempt suffix
|
||||
if ! gh run download "$RUN_ID" 2>/dev/null; then
|
||||
echo -e "${RED}Error: Failed to download artifacts from run ${RUN_ID}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✓ Artifacts downloaded${NC}"
|
||||
|
||||
# Check which bundles are present
|
||||
echo -e "\n${BLUE}Step 2: Checking available bundles...${NC}"
|
||||
|
||||
V1BETA3_BUNDLE=""
|
||||
V1BETA2_BUNDLE=""
|
||||
SUPPORTBUNDLE=""
|
||||
|
||||
if [ -f "preflight-v1beta3-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta3-bundle.tar.gz" ]; then
|
||||
V1BETA3_BUNDLE=$(find . -name "preflight-v1beta3-bundle.tar.gz" | head -1)
|
||||
echo -e "${GREEN}✓${NC} Found v1beta3 preflight bundle"
|
||||
fi
|
||||
|
||||
if [ -f "preflight-v1beta2-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta2-bundle.tar.gz" ]; then
|
||||
V1BETA2_BUNDLE=$(find . -name "preflight-v1beta2-bundle.tar.gz" | head -1)
|
||||
echo -e "${GREEN}✓${NC} Found v1beta2 preflight bundle"
|
||||
fi
|
||||
|
||||
if [ -f "supportbundle.tar.gz" ] || [ -f "test/output/supportbundle.tar.gz" ]; then
|
||||
SUPPORTBUNDLE=$(find . -name "supportbundle.tar.gz" | head -1)
|
||||
echo -e "${GREEN}✓${NC} Found support bundle"
|
||||
fi
|
||||
|
||||
if [ -z "$V1BETA3_BUNDLE" ] && [ -z "$V1BETA2_BUNDLE" ] && [ -z "$SUPPORTBUNDLE" ]; then
|
||||
echo -e "${RED}Error: No bundles found in artifacts${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Confirm update
|
||||
echo -e "\n${YELLOW}This will update the following baselines:${NC}"
|
||||
[ -n "$V1BETA3_BUNDLE" ] && echo " - test/baselines/preflight-v1beta3/baseline.tar.gz"
|
||||
[ -n "$V1BETA2_BUNDLE" ] && echo " - test/baselines/preflight-v1beta2/baseline.tar.gz"
|
||||
[ -n "$SUPPORTBUNDLE" ] && echo " - test/baselines/supportbundle/baseline.tar.gz"
|
||||
|
||||
echo -e "\n${YELLOW}Continue? (y/N):${NC} "
|
||||
read -r CONFIRM
|
||||
|
||||
if [ "$CONFIRM" != "y" ] && [ "$CONFIRM" != "Y" ]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Get project root (assuming script is in scripts/)
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
echo -e "\n${BLUE}Step 3: Updating baselines...${NC}"
|
||||
|
||||
# Update v1beta3 baseline
|
||||
if [ -n "$V1BETA3_BUNDLE" ]; then
|
||||
mkdir -p test/baselines/preflight-v1beta3
|
||||
cp "$TEMP_DIR/$V1BETA3_BUNDLE" test/baselines/preflight-v1beta3/baseline.tar.gz
|
||||
echo -e "${GREEN}✓${NC} Updated preflight-v1beta3 baseline"
|
||||
fi
|
||||
|
||||
# Update v1beta2 baseline
|
||||
if [ -n "$V1BETA2_BUNDLE" ]; then
|
||||
mkdir -p test/baselines/preflight-v1beta2
|
||||
cp "$TEMP_DIR/$V1BETA2_BUNDLE" test/baselines/preflight-v1beta2/baseline.tar.gz
|
||||
echo -e "${GREEN}✓${NC} Updated preflight-v1beta2 baseline"
|
||||
fi
|
||||
|
||||
# Update support bundle baseline
|
||||
if [ -n "$SUPPORTBUNDLE" ]; then
|
||||
mkdir -p test/baselines/supportbundle
|
||||
cp "$TEMP_DIR/$SUPPORTBUNDLE" test/baselines/supportbundle/baseline.tar.gz
|
||||
echo -e "${GREEN}✓${NC} Updated supportbundle baseline"
|
||||
fi
|
||||
|
||||
# Create metadata file
|
||||
echo -e "\n${BLUE}Step 4: Creating metadata...${NC}"
|
||||
|
||||
GIT_SHA=$(git rev-parse HEAD)
|
||||
CURRENT_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
|
||||
cat > test/baselines/metadata.json <<EOF
|
||||
{
|
||||
"updated_at": "$CURRENT_DATE",
|
||||
"git_sha": "$GIT_SHA",
|
||||
"workflow_run_id": "$RUN_ID",
|
||||
"k8s_version": "v1.28.3",
|
||||
"updated_by": "$(git config user.name) <$(git config user.email)>"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}✓${NC} Created metadata.json"
|
||||
|
||||
# Show git status
|
||||
echo -e "\n${BLUE}Step 5: Git status${NC}"
|
||||
git status test/baselines/
|
||||
|
||||
echo -e "\n${YELLOW}Review the changes above. To commit:${NC}"
|
||||
echo -e " ${BLUE}git add test/baselines/${NC}"
|
||||
echo -e " ${BLUE}git commit -m 'chore: update regression baselines from run ${RUN_ID}'${NC}"
|
||||
echo -e " ${BLUE}git push${NC}"
|
||||
|
||||
echo -e "\n${GREEN}Done!${NC}"
|
||||
Reference in New Issue
Block a user