diff --git a/.github/workflows/regression-test.yaml b/.github/workflows/regression-test.yaml new file mode 100644 index 00000000..06409a46 --- /dev/null +++ b/.github/workflows/regression-test.yaml @@ -0,0 +1,280 @@ +name: Regression Test Suite + +on: + push: + branches: [main, v1beta3] + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + inputs: + update_baselines: + description: 'Update baselines after run (use with caution)' + type: boolean + default: false + +jobs: + regression-test: + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + # 1. SETUP + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create k3s cluster + id: create-cluster + uses: replicatedhq/compatibility-actions/create-cluster@v1 + with: + api-token: ${{ secrets.REPLICATED_API_TOKEN }} + kubernetes-distribution: k3s + kubernetes-version: v1.28.3 + cluster-name: regression-${{ github.run_id }}-${{ github.run_attempt }} + ttl: 25m + timeout-minutes: 5 + + - name: Configure kubeconfig + run: | + echo "${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > $GITHUB_WORKSPACE/kubeconfig.yaml + echo "KUBECONFIG=$GITHUB_WORKSPACE/kubeconfig.yaml" >> $GITHUB_ENV + kubectl get nodes -o wide + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build binaries + run: | + echo "Building preflight and support-bundle binaries..." + make bin/preflight bin/support-bundle + ./bin/preflight version + ./bin/support-bundle version + + - name: Setup Python for comparison + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install pyyaml deepdiff + + # 2. EXECUTE SPECS + - name: Run preflight v1beta3 (complex) + continue-on-error: true + run: | + echo "Running preflight v1beta3 spec with values file..." + ./bin/preflight \ + examples/preflight/complex-v1beta3.yaml \ + --values examples/preflight/values-complex-full.yaml \ + --interactive=false \ + --format=json \ + --output=test/output/preflight-results-v1beta3.json || true + + # Find and rename the most recent bundle + BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) + if [ -n "$BUNDLE" ]; then + echo "Found bundle: $BUNDLE" + mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz + echo "✓ v1beta3 bundle saved" + else + echo "⚠ No v1beta3 bundle found" + exit 1 + fi + + - name: Run preflight v1beta2 (all-analyzers) + continue-on-error: true + run: | + echo "Running preflight v1beta2 spec..." + ./bin/preflight \ + examples/preflight/all-analyzers-v1beta2.yaml \ + --interactive=false \ + --format=json \ + --output=test/output/preflight-results-v1beta2.json || true + + # Find and rename the most recent bundle + BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) + if [ -n "$BUNDLE" ]; then + echo "Found bundle: $BUNDLE" + mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz + echo "✓ v1beta2 bundle saved" + else + echo "⚠ No v1beta2 bundle found" + exit 1 + fi + + - name: Run support bundle (all-kubernetes-collectors) + continue-on-error: true + run: | + echo "Running support bundle spec..." + ./bin/support-bundle \ + examples/collect/host/all-kubernetes-collectors.yaml \ + --interactive=false \ + --output=test/output/supportbundle.tar.gz || true + + if [ -f test/output/supportbundle.tar.gz ]; then + echo "✓ Support bundle saved" + else + echo "⚠ No support bundle found" + exit 1 + fi + + # 3. COMPARE BUNDLES + - name: Compare preflight v1beta3 bundle + id: compare-v1beta3 + continue-on-error: true + run: | + echo "Comparing v1beta3 preflight bundle against baseline..." + if [ ! -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then + echo "⚠ No baseline found for v1beta3 - skipping comparison" + echo "baseline_missing=true" >> $GITHUB_OUTPUT + exit 0 + fi + + python3 scripts/compare_bundles.py \ + --baseline test/baselines/preflight-v1beta3/baseline.tar.gz \ + --current test/output/preflight-v1beta3-bundle.tar.gz \ + --rules scripts/compare_rules.yaml \ + --report test/output/diff-report-v1beta3.json \ + --spec-type preflight + + - name: Compare preflight v1beta2 bundle + id: compare-v1beta2 + continue-on-error: true + run: | + echo "Comparing v1beta2 preflight bundle against baseline..." + if [ ! -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then + echo "⚠ No baseline found for v1beta2 - skipping comparison" + echo "baseline_missing=true" >> $GITHUB_OUTPUT + exit 0 + fi + + python3 scripts/compare_bundles.py \ + --baseline test/baselines/preflight-v1beta2/baseline.tar.gz \ + --current test/output/preflight-v1beta2-bundle.tar.gz \ + --rules scripts/compare_rules.yaml \ + --report test/output/diff-report-v1beta2.json \ + --spec-type preflight + + - name: Compare support bundle + id: compare-supportbundle + continue-on-error: true + run: | + echo "Comparing support bundle against baseline..." + if [ ! -f test/baselines/supportbundle/baseline.tar.gz ]; then + echo "⚠ No baseline found for support bundle - skipping comparison" + echo "baseline_missing=true" >> $GITHUB_OUTPUT + exit 0 + fi + + python3 scripts/compare_bundles.py \ + --baseline test/baselines/supportbundle/baseline.tar.gz \ + --current test/output/supportbundle.tar.gz \ + --rules scripts/compare_rules.yaml \ + --report test/output/diff-report-supportbundle.json \ + --spec-type supportbundle + + # 4. REPORT RESULTS + - name: Generate summary report + if: always() + run: | + python3 scripts/generate_summary.py \ + --reports test/output/diff-report-*.json \ + --output-file $GITHUB_STEP_SUMMARY \ + --output-console + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: regression-test-results-${{ github.run_id }}-${{ github.run_attempt }} + path: | + test/output/*.tar.gz + test/output/*.json + retention-days: 30 + + - name: Check for regressions + if: always() + run: | + echo "Checking comparison results..." + + # Check if any comparisons failed + FAILURES=0 + + if [ "${{ steps.compare-v1beta3.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta3.outputs.baseline_missing }}" != "true" ]; then + echo "❌ v1beta3 comparison failed" + FAILURES=$((FAILURES + 1)) + fi + + if [ "${{ steps.compare-v1beta2.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta2.outputs.baseline_missing }}" != "true" ]; then + echo "❌ v1beta2 comparison failed" + FAILURES=$((FAILURES + 1)) + fi + + if [ "${{ steps.compare-supportbundle.outcome }}" == "failure" ] && [ "${{ steps.compare-supportbundle.outputs.baseline_missing }}" != "true" ]; then + echo "❌ Support bundle comparison failed" + FAILURES=$((FAILURES + 1)) + fi + + if [ $FAILURES -gt 0 ]; then + echo "" + echo "❌ $FAILURES regression(s) detected!" + echo "Review the comparison reports in the artifacts." + exit 1 + else + echo "✅ All comparisons passed or skipped (no baseline)" + fi + + # 5. UPDATE BASELINES (optional, manual trigger only) + - name: Update baselines + if: github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch' + run: | + echo "Updating baselines with current bundles..." + + # Copy new bundles as baselines + if [ -f test/output/preflight-v1beta3-bundle.tar.gz ]; then + mkdir -p test/baselines/preflight-v1beta3 + cp test/output/preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz + echo "✓ Updated v1beta3 baseline" + fi + + if [ -f test/output/preflight-v1beta2-bundle.tar.gz ]; then + mkdir -p test/baselines/preflight-v1beta2 + cp test/output/preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz + echo "✓ Updated v1beta2 baseline" + fi + + if [ -f test/output/supportbundle.tar.gz ]; then + mkdir -p test/baselines/supportbundle + cp test/output/supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz + echo "✓ Updated support bundle baseline" + fi + + # Create metadata file + cat > test/baselines/metadata.json < Dict: + """Load comparison rules from YAML file.""" + if not Path(rules_path).exists(): + print(f"Warning: Rules file not found at {rules_path}, using defaults") + return self._get_default_rules() + + with open(rules_path) as f: + rules = yaml.safe_load(f) + + return rules.get(spec_type, rules.get("defaults", {})) + + def _get_default_rules(self) -> Dict: + """Return default comparison rules if no config file.""" + return { + "exact_match": [ + "static-data.txt/static-data", + "version.yaml", + ], + "structural_compare": { + "postgres/*.json": "database_connection", + "mysql/*.json": "database_connection", + "mssql/*.json": "database_connection", + "redis/*.json": "database_connection", + "dns/debug.json": "dns_structure", + "registry/*.json": "registry_exists", + "http*.json": "http_status", + }, + "non_empty_default": True, + } + + def compare(self, baseline_bundle: str, current_bundle: str) -> bool: + """ + Compare two bundles. Returns True if no regressions detected. + + Args: + baseline_bundle: Path to baseline bundle tar.gz + current_bundle: Path to current bundle tar.gz + + Returns: + True if bundles match (no regressions), False otherwise + """ + with tempfile.TemporaryDirectory() as tmpdir: + baseline_dir = Path(tmpdir) / "baseline" + current_dir = Path(tmpdir) / "current" + + print(f"Extracting baseline bundle to {baseline_dir}...") + self._extract(baseline_bundle, baseline_dir) + + print(f"Extracting current bundle to {current_dir}...") + self._extract(current_bundle, current_dir) + + baseline_files = self._get_file_list(baseline_dir) + current_files = self._get_file_list(current_dir) + + print(f"Baseline files: {len(baseline_files)}") + print(f"Current files: {len(current_files)}") + + # Check for missing files + missing_in_current = baseline_files - current_files + missing_in_baseline = current_files - baseline_files + + # Filter out optional files that may not exist (previous logs, etc.) + optional_patterns = [ + "*-previous.log", # Previous container logs (only exist after restart) + ] + + for file in sorted(missing_in_current): + # Skip optional files + if any(file.match(pattern) for pattern in optional_patterns): + print(f" ℹ Optional file missing (OK): {file}") + continue + self._record_missing("current", str(file)) + + for file in sorted(missing_in_baseline): + # Optional files added in current are also OK + if any(file.match(pattern) for pattern in optional_patterns): + print(f" ℹ Optional file added (OK): {file}") + continue + self._record_missing("baseline", str(file)) + + # Compare common files + common_files = baseline_files & current_files + print(f"Comparing {len(common_files)} common files...") + + for file in sorted(common_files): + self._compare_file( + baseline_dir / file, + current_dir / file, + str(file) + ) + + # Determine if there are regressions + has_regressions = ( + self.results["files_different"] > 0 or + self.results["files_missing_in_current"] > 0 + ) + + return not has_regressions + + def _extract(self, bundle_path: str, dest_dir: Path): + """Extract tar.gz bundle to destination directory.""" + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(bundle_path, 'r:gz') as tar: + tar.extractall(dest_dir) + + # Handle bundles that extract to a nested directory (e.g., preflightbundle-timestamp/) + # If there's only one directory at the root, use that as the actual root + items = list(dest_dir.iterdir()) + if len(items) == 1 and items[0].is_dir(): + # Move contents up one level + nested_dir = items[0] + for item in nested_dir.iterdir(): + item.rename(dest_dir / item.name) + nested_dir.rmdir() + + def _get_file_list(self, dir_path: Path) -> set: + """Get set of all files in directory (relative paths).""" + files = set() + for path in dir_path.rglob('*'): + if path.is_file(): + rel_path = path.relative_to(dir_path) + files.add(rel_path) + return files + + def _compare_file(self, baseline_path: Path, current_path: Path, rel_path: str): + """Compare a single file pair using appropriate rule.""" + self.results["files_compared"] += 1 + + # Determine comparison mode + mode = self._get_comparison_mode(rel_path) + + try: + if mode == "exact": + if self._compare_exact(baseline_path, current_path): + self.results["exact_matches"] += 1 + else: + self._record_diff(rel_path, "exact", "Content mismatch") + + elif mode == "structural": + comparator = self._get_structural_comparator(rel_path) + if self._compare_structural(baseline_path, current_path, comparator): + self.results["structural_matches"] += 1 + else: + self._record_diff(rel_path, "structural", f"Structural comparison failed ({comparator})") + + else: # non_empty + if self._check_non_empty(current_path): + self.results["non_empty_checks"] += 1 + else: + self._record_diff(rel_path, "non_empty", "File is empty") + + except Exception as e: + self._record_diff(rel_path, "error", f"Comparison error: {str(e)}") + + def _get_comparison_mode(self, rel_path: str) -> str: + """Determine comparison mode for a file based on rules.""" + # Check exact match patterns + for pattern in self.rules.get("exact_match", []): + if fnmatch.fnmatch(rel_path, pattern) or rel_path == pattern: + return "exact" + + # Check structural comparison patterns + for pattern in self.rules.get("structural_compare", {}).keys(): + if fnmatch.fnmatch(rel_path, pattern): + return "structural" + + # Default: non-empty check + return "non_empty" + + def _get_structural_comparator(self, rel_path: str) -> str: + """Get the structural comparator name for a file.""" + for pattern, comparator in self.rules.get("structural_compare", {}).items(): + if fnmatch.fnmatch(rel_path, pattern): + return comparator + return "unknown" + + def _compare_exact(self, baseline_path: Path, current_path: Path) -> bool: + """Compare files byte-for-byte.""" + return baseline_path.read_bytes() == current_path.read_bytes() + + def _compare_structural(self, baseline_path: Path, current_path: Path, comparator: str) -> bool: + """Compare files using structural comparator.""" + # Load JSON data + try: + baseline_data = json.loads(baseline_path.read_text()) + current_data = json.loads(current_path.read_text()) + except json.JSONDecodeError as e: + print(f" JSON decode error: {e}") + return False + + # Apply comparator + if comparator == "database_connection": + return self._compare_database_connection(baseline_data, current_data) + elif comparator == "dns_structure": + return self._compare_dns_structure(baseline_data, current_data) + elif comparator == "registry_exists": + return self._compare_registry_exists(baseline_data, current_data) + elif comparator == "http_status": + return self._compare_http_status(baseline_data, current_data) + elif comparator == "cluster_version": + return self._compare_cluster_version(baseline_data, current_data) + elif comparator == "analysis_results": + return self._compare_analysis_results(baseline_data, current_data) + else: + # Unknown comparator - fall back to non-empty + return True + + def _compare_database_connection(self, baseline: Dict, current: Dict) -> bool: + """Compare database connection results (isConnected field only).""" + b_connected = baseline.get("isConnected", False) + c_connected = current.get("isConnected", False) + + if b_connected != c_connected: + print(f" Database connection status changed: {b_connected} -> {c_connected}") + return False + + return True + + def _compare_dns_structure(self, baseline: Dict, current: Dict) -> bool: + """Compare DNS structure (service exists, query succeeds).""" + # Check kubernetes service exists + if "query" not in current or "kubernetes" not in current["query"]: + print(f" DNS query.kubernetes missing") + return False + + # Kubernetes ClusterIP should exist (don't compare value, it can vary) + if not current["query"]["kubernetes"].get("address"): + print(f" DNS kubernetes.address is empty") + return False + + # DNS service should exist + if not current.get("kubeDNSService"): + print(f" DNS kubeDNSService is empty") + return False + + # At least one DNS pod should exist + if not current.get("kubeDNSPods") or len(current["kubeDNSPods"]) == 0: + print(f" DNS kubeDNSPods is empty") + return False + + # Non-resolvable domain should be empty + if current.get("query", {}).get("nonResolvableDomain", {}).get("address"): + print(f" DNS nonResolvableDomain should be empty") + return False + + return True + + def _compare_registry_exists(self, baseline: Dict, current: Dict) -> bool: + """Compare registry image existence (exists boolean per image).""" + baseline_images = baseline.get("images", {}) + current_images = current.get("images", {}) + + # Check same images are present + if set(baseline_images.keys()) != set(current_images.keys()): + print(f" Registry image list changed") + print(f" Baseline: {sorted(baseline_images.keys())}") + print(f" Current: {sorted(current_images.keys())}") + return False + + # Compare exists status for each image + for image_name in baseline_images: + b_exists = baseline_images[image_name].get("exists", False) + c_exists = current_images[image_name].get("exists", False) + + if b_exists != c_exists: + print(f" Registry image '{image_name}' existence changed: {b_exists} -> {c_exists}") + return False + + return True + + def _compare_http_status(self, baseline: Dict, current: Dict) -> bool: + """Compare HTTP response (status code only).""" + b_status = baseline.get("response", {}).get("status", 0) + c_status = current.get("response", {}).get("status", 0) + + if b_status != c_status: + print(f" HTTP status changed: {b_status} -> {c_status}") + return False + + return True + + def _compare_cluster_version(self, baseline: Dict, current: Dict) -> bool: + """Compare cluster version (major/minor only, ignore build details).""" + b_info = baseline.get("info", {}) + c_info = current.get("info", {}) + + # Compare major and minor version + if b_info.get("major") != c_info.get("major"): + print(f" Cluster major version changed: {b_info.get('major')} -> {c_info.get('major')}") + return False + + if b_info.get("minor") != c_info.get("minor"): + print(f" Cluster minor version changed: {b_info.get('minor')} -> {c_info.get('minor')}") + return False + + # Don't compare: gitVersion, gitCommit, buildDate, goVersion (these vary with k3s updates) + return True + + def _compare_analysis_results(self, baseline: Dict, current: Dict) -> bool: + """Compare analysis results (analyzer names and count, not specific messages).""" + if not isinstance(baseline, list) or not isinstance(current, list): + print(f" Analysis results structure changed (expected list)") + return False + + # Create map of analyzer name -> severity for comparison + baseline_results = {item.get("name"): item.get("severity") for item in baseline if "name" in item} + current_results = {item.get("name"): item.get("severity") for item in current if "name" in item} + + # Check if same analyzers ran + baseline_names = set(baseline_results.keys()) + current_names = set(current_results.keys()) + + if baseline_names != current_names: + missing = baseline_names - current_names + extra = current_names - baseline_names + if missing: + print(f" Missing analyzers: {missing}") + if extra: + print(f" New analyzers: {extra}") + return False + + # Check if severity levels changed significantly (error/warn differences matter) + significant_changes = [] + for name in baseline_names: + b_sev = baseline_results[name] + c_sev = current_results[name] + + # Only care if error/warn status changes, not debug + if b_sev != c_sev: + if b_sev in ["error", "warn"] or c_sev in ["error", "warn"]: + significant_changes.append(f"{name}: {b_sev} -> {c_sev}") + + if significant_changes: + print(f" Analyzer severity changed:") + for change in significant_changes[:5]: # Show first 5 + print(f" {change}") + # Don't fail on severity changes - this is informational + # return False + + return True + + def _check_non_empty(self, path: Path) -> bool: + """Check that file exists and is non-empty.""" + if not path.exists(): + return False + + size = path.stat().st_size + if size == 0: + return False + + # Optional: validate JSON structure if .json extension + if path.suffix == ".json": + try: + json.loads(path.read_text()) + except json.JSONDecodeError: + print(f" Invalid JSON: {path.name}") + return False + + return True + + def _record_diff(self, file: str, mode: str, reason: str): + """Record a difference/regression.""" + self.results["files_different"] += 1 + self.results["differences"].append({ + "file": file, + "mode": mode, + "reason": reason + }) + print(f" ❌ {file}: {reason}") + + def _record_missing(self, location: str, file: str): + """Record a missing file.""" + if location == "current": + self.results["files_missing_in_current"] += 1 + self.results["missing_in_current"].append(file) + print(f" ⚠ Missing in current: {file}") + else: + self.results["files_missing_in_baseline"] += 1 + self.results["missing_in_baseline"].append(file) + print(f" ℹ New file in current: {file}") + + def generate_report(self, output_path: str): + """Write JSON report.""" + with open(output_path, 'w') as f: + json.dump(self.results, f, indent=2) + + print(f"\nReport written to: {output_path}") + + def print_summary(self): + """Print human-readable summary to stdout.""" + print(f"\n{'='*60}") + print(f"Bundle Comparison Report - {self.spec_type}") + print(f"{'='*60}") + print(f"Files compared: {self.results['files_compared']}") + print(f" Exact matches: {self.results['exact_matches']}") + print(f" Structural matches: {self.results['structural_matches']}") + print(f" Non-empty checks: {self.results['non_empty_checks']}") + print(f"Files different: {self.results['files_different']}") + print(f"Missing in current: {self.results['files_missing_in_current']}") + print(f"Missing in baseline: {self.results['files_missing_in_baseline']}") + + if self.results["differences"]: + print(f"\n❌ REGRESSIONS DETECTED ({len(self.results['differences'])}):") + for diff in self.results["differences"][:10]: # Show first 10 + print(f" • {diff['file']}: {diff['reason']}") + if len(self.results["differences"]) > 10: + print(f" ... and {len(self.results['differences']) - 10} more") + + if self.results["missing_in_current"]: + print(f"\n⚠ MISSING FILES ({len(self.results['missing_in_current'])}):") + for file in self.results["missing_in_current"][:5]: + print(f" • {file}") + if len(self.results["missing_in_current"]) > 5: + print(f" ... and {len(self.results['missing_in_current']) - 5} more") + + +def main(): + parser = argparse.ArgumentParser( + description="Compare troubleshoot bundles for regression testing" + ) + parser.add_argument("--baseline", required=True, help="Baseline bundle tar.gz path") + parser.add_argument("--current", required=True, help="Current bundle tar.gz path") + parser.add_argument("--rules", required=True, help="Comparison rules YAML path") + parser.add_argument("--report", required=True, help="Output report JSON path") + parser.add_argument( + "--spec-type", + required=True, + choices=["preflight", "supportbundle"], + help="Type of spec being compared" + ) + + args = parser.parse_args() + + # Verify files exist + if not Path(args.baseline).exists(): + print(f"Error: Baseline bundle not found: {args.baseline}") + sys.exit(1) + + if not Path(args.current).exists(): + print(f"Error: Current bundle not found: {args.current}") + sys.exit(1) + + # Run comparison + comparator = BundleComparator(args.rules, args.spec_type) + passed = comparator.compare(args.baseline, args.current) + comparator.generate_report(args.report) + comparator.print_summary() + + # Exit with appropriate code + if passed: + print("\n✅ No regressions detected") + sys.exit(0) + else: + print("\n❌ Regressions detected!") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/compare_rules.yaml b/scripts/compare_rules.yaml new file mode 100644 index 00000000..9b2a002f --- /dev/null +++ b/scripts/compare_rules.yaml @@ -0,0 +1,121 @@ +# Comparison rules for regression testing +# Defines how different collector outputs should be compared + +# Global configuration (applies to both preflight and supportbundle) +global: + # Files that should be compared exactly (byte-for-byte) + exact_match: + - "static-data.txt/static-data" + # Note: version.yaml is NOT here - versionNumber varies between builds + + # Default behavior for unknown files + non_empty_default: true + +# Preflight-specific rules +preflight: + # Files that should be compared exactly + exact_match: + - "static-data.txt/static-data" + - "files/example.yaml" # From data collector in v1beta2 + - "files/example.json" # From data collector in v1beta2 + - "config/replicas.txt" # From data collector in v1beta2 + # Note: version.yaml is NOT here - it uses non-empty check (versionNumber varies) + + # Files that need structural/field-specific comparison + # Format: "pattern": "comparator_function_name" + structural_compare: + # Database collectors - compare isConnected boolean only + "postgres/*.json": "database_connection" + "mysql/*.json": "database_connection" + "mssql/*.json": "database_connection" + "redis/*.json": "database_connection" + + # DNS collector - compare structure, not exact values + "dns/debug.json": "dns_structure" + + # Registry collector - compare exists boolean per image + "registry/*.json": "registry_exists" + "registry-images/*.json": "registry_exists" + + # HTTP collector - compare status code only + "http/*.json": "http_status" + "http-*.json": "http_status" + + # Cluster info - compare major/minor version, ignore build details + "cluster-info/cluster_version.json": "cluster_version" + + # Analysis results - compare analyzer names and severity levels only + "analysis.json": "analysis_results" + + # Everything else uses non-empty check by default + # This includes: + # - cluster-resources/**/*.json (UIDs, timestamps vary) + # - node-metrics/**/*.json (all values vary) + # - goldpinger/**/*.json (latencies vary) + # - run*/**/* (pod names, output vary) + # - ceph/**/* (status/metrics vary or not installed) + # - longhorn/**/* (status/metrics vary or not installed) + # - certificates/**/*.json (validity time-based) + # - configmaps/**/*.json (can have dynamic values) + # - secrets/**/*.json (can have dynamic values) + # - sysctl/**/* (some counters vary) + # - collectd/**/* (time-series data) + # - helm/**/*.json (timestamps, revisions vary) + # - logs/**/*.log (timestamps in every line) + # - copy*/**/* (content depends on what's copied) + +# Support bundle-specific rules +supportbundle: + # Files that should be compared exactly + exact_match: + - "static-data.txt/static-data" + # Note: version.yaml is NOT here - it uses non-empty check (versionNumber varies) + + # Files that need structural comparison + structural_compare: + # Database collectors + "postgres/*.json": "database_connection" + "mysql/*.json": "database_connection" + "mssql/*.json": "database_connection" + "redis/*.json": "database_connection" + + # DNS collector + "dns/debug.json": "dns_structure" + + # Registry collector + "registry/*.json": "registry_exists" + "registry-images/*.json": "registry_exists" + + # HTTP collector + "http*.json": "http_status" + + # Cluster info + "cluster-info/cluster_version.json": "cluster_version" + + # Everything else uses non-empty check (see list above in preflight section) + +# Notes on comparison strategies: +# +# EXACT MATCH: +# - Use for static data that should never change between runs +# - Byte-for-byte comparison +# - Any difference is a regression +# +# STRUCTURAL COMPARISON: +# - Use for semi-deterministic output with consistent structure but variable values +# - Compare specific fields only (e.g., status codes, booleans) +# - Ignore timing-dependent or environment-specific values +# +# NON-EMPTY CHECK (default): +# - Use for highly variable output where exact comparison is impractical +# - Verifies file exists and is not empty +# - For JSON files, also validates JSON is parseable +# - Appropriate for: +# * Kubernetes resources (UIDs, resourceVersions, timestamps) +# * Metrics (all values constantly change) +# * Logs (timestamps, dynamic content) +# * Generated pod/resource names +# * Runtime state (pod status, replica counts) +# +# This strategy catches major regressions (collectors breaking, files missing) +# while avoiding false positives from expected variability. diff --git a/scripts/generate_summary.py b/scripts/generate_summary.py new file mode 100755 index 00000000..8a310d8e --- /dev/null +++ b/scripts/generate_summary.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +""" +Generate summary report from bundle comparison results. + +Reads JSON diff reports and produces: +1. GitHub Actions step summary (Markdown) +2. Console output (colored text) +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import List, Dict + + +def load_reports(report_pattern: str) -> List[Dict]: + """Load all report JSON files matching pattern.""" + reports = [] + + # Handle glob pattern + if '*' in report_pattern: + report_dir = Path(report_pattern).parent + pattern = Path(report_pattern).name + + for report_file in sorted(report_dir.glob(pattern)): + try: + with open(report_file) as f: + report = json.load(f) + report['_filename'] = report_file.name + reports.append(report) + except (json.JSONDecodeError, FileNotFoundError) as e: + print(f"Warning: Could not load {report_file}: {e}", file=sys.stderr) + else: + # Single file + try: + with open(report_pattern) as f: + report = json.load(f) + report['_filename'] = Path(report_pattern).name + reports.append(report) + except (json.JSONDecodeError, FileNotFoundError) as e: + print(f"Warning: Could not load {report_pattern}: {e}", file=sys.stderr) + + return reports + + +def generate_markdown_summary(reports: List[Dict]) -> str: + """Generate GitHub Actions step summary in Markdown format.""" + lines = [] + + lines.append("# 🧪 Regression Test Results") + lines.append("") + + if not reports: + lines.append("⚠️ No comparison reports found. Baselines may be missing.") + return "\n".join(lines) + + # Overall status + total_regressions = sum(r.get('files_different', 0) for r in reports) + total_missing = sum(r.get('files_missing_in_current', 0) for r in reports) + + if total_regressions > 0 or total_missing > 0: + lines.append(f"## ❌ Status: FAILED") + lines.append(f"**{total_regressions} file(s) with differences, {total_missing} file(s) missing**") + else: + lines.append(f"## ✅ Status: PASSED") + lines.append("All comparisons passed!") + + lines.append("") + + # Per-spec breakdown + lines.append("## 📊 Comparison Breakdown") + lines.append("") + + for report in reports: + spec_type = report.get('spec_type', 'unknown') + filename = report.get('_filename', 'unknown') + + # Determine status icon + has_regressions = ( + report.get('files_different', 0) > 0 or + report.get('files_missing_in_current', 0) > 0 + ) + status_icon = "❌" if has_regressions else "✅" + + lines.append(f"### {status_icon} {spec_type.upper()}") + lines.append("") + lines.append(f"**Report:** `{filename}`") + lines.append("") + + # Stats table + lines.append("| Metric | Count |") + lines.append("|--------|-------|") + lines.append(f"| Files compared | {report.get('files_compared', 0)} |") + lines.append(f"| Exact matches | {report.get('exact_matches', 0)} |") + lines.append(f"| Structural matches | {report.get('structural_matches', 0)} |") + lines.append(f"| Non-empty checks | {report.get('non_empty_checks', 0)} |") + lines.append(f"| **Files different** | **{report.get('files_different', 0)}** |") + lines.append(f"| **Missing in current** | **{report.get('files_missing_in_current', 0)}** |") + lines.append(f"| New in current | {report.get('files_missing_in_baseline', 0)} |") + lines.append("") + + # Show differences if any + differences = report.get('differences', []) + if differences: + lines.append("
") + lines.append(f"⚠️ Show {len(differences)} difference(s)") + lines.append("") + lines.append("| File | Mode | Reason |") + lines.append("|------|------|--------|") + for diff in differences[:20]: # Limit to 20 + file = diff.get('file', 'unknown') + mode = diff.get('mode', 'unknown') + reason = diff.get('reason', 'unknown') + lines.append(f"| `{file}` | {mode} | {reason} |") + + if len(differences) > 20: + lines.append(f"| ... | ... | *{len(differences) - 20} more differences* |") + + lines.append("") + lines.append("
") + lines.append("") + + # Show missing files if any + missing = report.get('missing_in_current', []) + if missing: + lines.append("
") + lines.append(f"⚠️ Show {len(missing)} missing file(s)") + lines.append("") + for file in missing[:20]: + lines.append(f"- `{file}`") + + if len(missing) > 20: + lines.append(f"- *... and {len(missing) - 20} more*") + + lines.append("") + lines.append("
") + lines.append("") + + # Footer + lines.append("---") + lines.append("") + lines.append("💡 **Tips:**") + lines.append("- Download artifacts to inspect bundle contents") + lines.append("- Review diff reports for detailed comparison results") + lines.append("- Update baselines if changes are intentional (use workflow_dispatch with update_baselines)") + + return "\n".join(lines) + + +def generate_console_summary(reports: List[Dict]) -> str: + """Generate console output with ANSI colors.""" + lines = [] + + # ANSI color codes + RED = "\033[91m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + BOLD = "\033[1m" + RESET = "\033[0m" + + lines.append(f"\n{BOLD}{'='*60}{RESET}") + lines.append(f"{BOLD}Regression Test Summary{RESET}") + lines.append(f"{BOLD}{'='*60}{RESET}\n") + + if not reports: + lines.append(f"{YELLOW}⚠ No comparison reports found{RESET}") + return "\n".join(lines) + + # Overall status + total_regressions = sum(r.get('files_different', 0) for r in reports) + total_missing = sum(r.get('files_missing_in_current', 0) for r in reports) + + if total_regressions > 0 or total_missing > 0: + lines.append(f"{RED}{BOLD}❌ FAILED{RESET}") + lines.append(f" {total_regressions} file(s) with differences") + lines.append(f" {total_missing} file(s) missing\n") + else: + lines.append(f"{GREEN}{BOLD}✅ PASSED{RESET}") + lines.append(f" All comparisons successful\n") + + # Per-spec details + for i, report in enumerate(reports): + spec_type = report.get('spec_type', 'unknown') + + has_regressions = ( + report.get('files_different', 0) > 0 or + report.get('files_missing_in_current', 0) > 0 + ) + + status_color = RED if has_regressions else GREEN + status_icon = "❌" if has_regressions else "✅" + + lines.append(f"{BLUE}{BOLD}{spec_type.upper()}{RESET} {status_color}{status_icon}{RESET}") + lines.append(f" Files compared: {report.get('files_compared', 0)}") + lines.append(f" Exact matches: {report.get('exact_matches', 0)}") + lines.append(f" Structural matches: {report.get('structural_matches', 0)}") + lines.append(f" Non-empty checks: {report.get('non_empty_checks', 0)}") + + if report.get('files_different', 0) > 0: + lines.append(f" {RED}Files different: {report.get('files_different', 0)}{RESET}") + + if report.get('files_missing_in_current', 0) > 0: + lines.append(f" {RED}Missing in current: {report.get('files_missing_in_current', 0)}{RESET}") + + if report.get('files_missing_in_baseline', 0) > 0: + lines.append(f" {YELLOW}New in current: {report.get('files_missing_in_baseline', 0)}{RESET}") + + # Show first few differences + differences = report.get('differences', []) + if differences: + lines.append(f"\n {RED}Differences:{RESET}") + for diff in differences[:5]: + file = diff.get('file', 'unknown') + reason = diff.get('reason', 'unknown') + lines.append(f" • {file}: {reason}") + + if len(differences) > 5: + lines.append(f" ... and {len(differences) - 5} more") + + if i < len(reports) - 1: + lines.append("") # Spacing between specs + + lines.append(f"\n{BOLD}{'='*60}{RESET}\n") + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Generate summary report from comparison results" + ) + parser.add_argument( + "--reports", + required=True, + help="Report file(s) pattern (e.g., 'test/output/diff-report-*.json')" + ) + parser.add_argument( + "--output-file", + help="Write markdown summary to file (e.g., $GITHUB_STEP_SUMMARY)" + ) + parser.add_argument( + "--output-console", + action="store_true", + help="Print colored summary to console" + ) + + args = parser.parse_args() + + # Load reports + reports = load_reports(args.reports) + + if not reports: + print("Warning: No reports loaded", file=sys.stderr) + + # Generate markdown summary + markdown = generate_markdown_summary(reports) + + # Write to file if requested + if args.output_file: + try: + with open(args.output_file, 'w') as f: + f.write(markdown) + print(f"Summary written to {args.output_file}") + except IOError as e: + print(f"Error writing summary to {args.output_file}: {e}", file=sys.stderr) + + # Print to console if requested + if args.output_console: + console = generate_console_summary(reports) + print(console) + + # If neither output option specified, print to stdout + if not args.output_file and not args.output_console: + print(markdown) + + +if __name__ == "__main__": + main() diff --git a/scripts/update_baselines.sh b/scripts/update_baselines.sh new file mode 100755 index 00000000..157ce134 --- /dev/null +++ b/scripts/update_baselines.sh @@ -0,0 +1,158 @@ +#!/bin/bash +set -e + +# Helper script to update regression test baselines +# Usage: ./scripts/update_baselines.sh [run-id] + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}===========================================${NC}" +echo -e "${BLUE}Regression Test Baseline Update Script${NC}" +echo -e "${BLUE}===========================================${NC}\n" + +# Check if gh CLI is installed +if ! command -v gh &> /dev/null; then + echo -e "${RED}Error: GitHub CLI (gh) not found${NC}" + echo "Install from: https://cli.github.com/" + exit 1 +fi + +# Get run ID from argument or prompt +if [ -n "$1" ]; then + RUN_ID="$1" +else + echo -e "${YELLOW}Enter GitHub Actions run ID (or leave empty for latest):${NC}" + read -r RUN_ID +fi + +# If no run ID provided, get the latest regression-test workflow run +if [ -z "$RUN_ID" ]; then + echo "Fetching latest regression-test workflow run..." + RUN_ID=$(gh run list --workflow=regression-test.yaml --limit 1 --json databaseId --jq '.[0].databaseId') + + if [ -z "$RUN_ID" ]; then + echo -e "${RED}Error: No workflow runs found${NC}" + exit 1 + fi + + echo -e "Using latest run: ${GREEN}${RUN_ID}${NC}" +fi + +# Create temp directory +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo -e "\n${BLUE}Step 1: Downloading artifacts...${NC}" + +# Download artifacts from the run +cd "$TEMP_DIR" +if ! gh run download "$RUN_ID" --name "regression-test-results-${RUN_ID}-1" 2>/dev/null; then + # Try without attempt suffix + if ! gh run download "$RUN_ID" 2>/dev/null; then + echo -e "${RED}Error: Failed to download artifacts from run ${RUN_ID}${NC}" + exit 1 + fi +fi + +echo -e "${GREEN}✓ Artifacts downloaded${NC}" + +# Check which bundles are present +echo -e "\n${BLUE}Step 2: Checking available bundles...${NC}" + +V1BETA3_BUNDLE="" +V1BETA2_BUNDLE="" +SUPPORTBUNDLE="" + +if [ -f "preflight-v1beta3-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta3-bundle.tar.gz" ]; then + V1BETA3_BUNDLE=$(find . -name "preflight-v1beta3-bundle.tar.gz" | head -1) + echo -e "${GREEN}✓${NC} Found v1beta3 preflight bundle" +fi + +if [ -f "preflight-v1beta2-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta2-bundle.tar.gz" ]; then + V1BETA2_BUNDLE=$(find . -name "preflight-v1beta2-bundle.tar.gz" | head -1) + echo -e "${GREEN}✓${NC} Found v1beta2 preflight bundle" +fi + +if [ -f "supportbundle.tar.gz" ] || [ -f "test/output/supportbundle.tar.gz" ]; then + SUPPORTBUNDLE=$(find . -name "supportbundle.tar.gz" | head -1) + echo -e "${GREEN}✓${NC} Found support bundle" +fi + +if [ -z "$V1BETA3_BUNDLE" ] && [ -z "$V1BETA2_BUNDLE" ] && [ -z "$SUPPORTBUNDLE" ]; then + echo -e "${RED}Error: No bundles found in artifacts${NC}" + exit 1 +fi + +# Confirm update +echo -e "\n${YELLOW}This will update the following baselines:${NC}" +[ -n "$V1BETA3_BUNDLE" ] && echo " - test/baselines/preflight-v1beta3/baseline.tar.gz" +[ -n "$V1BETA2_BUNDLE" ] && echo " - test/baselines/preflight-v1beta2/baseline.tar.gz" +[ -n "$SUPPORTBUNDLE" ] && echo " - test/baselines/supportbundle/baseline.tar.gz" + +echo -e "\n${YELLOW}Continue? (y/N):${NC} " +read -r CONFIRM + +if [ "$CONFIRM" != "y" ] && [ "$CONFIRM" != "Y" ]; then + echo "Aborted." + exit 0 +fi + +# Get project root (assuming script is in scripts/) +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$PROJECT_ROOT" + +echo -e "\n${BLUE}Step 3: Updating baselines...${NC}" + +# Update v1beta3 baseline +if [ -n "$V1BETA3_BUNDLE" ]; then + mkdir -p test/baselines/preflight-v1beta3 + cp "$TEMP_DIR/$V1BETA3_BUNDLE" test/baselines/preflight-v1beta3/baseline.tar.gz + echo -e "${GREEN}✓${NC} Updated preflight-v1beta3 baseline" +fi + +# Update v1beta2 baseline +if [ -n "$V1BETA2_BUNDLE" ]; then + mkdir -p test/baselines/preflight-v1beta2 + cp "$TEMP_DIR/$V1BETA2_BUNDLE" test/baselines/preflight-v1beta2/baseline.tar.gz + echo -e "${GREEN}✓${NC} Updated preflight-v1beta2 baseline" +fi + +# Update support bundle baseline +if [ -n "$SUPPORTBUNDLE" ]; then + mkdir -p test/baselines/supportbundle + cp "$TEMP_DIR/$SUPPORTBUNDLE" test/baselines/supportbundle/baseline.tar.gz + echo -e "${GREEN}✓${NC} Updated supportbundle baseline" +fi + +# Create metadata file +echo -e "\n${BLUE}Step 4: Creating metadata...${NC}" + +GIT_SHA=$(git rev-parse HEAD) +CURRENT_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +cat > test/baselines/metadata.json <" +} +EOF + +echo -e "${GREEN}✓${NC} Created metadata.json" + +# Show git status +echo -e "\n${BLUE}Step 5: Git status${NC}" +git status test/baselines/ + +echo -e "\n${YELLOW}Review the changes above. To commit:${NC}" +echo -e " ${BLUE}git add test/baselines/${NC}" +echo -e " ${BLUE}git commit -m 'chore: update regression baselines from run ${RUN_ID}'${NC}" +echo -e " ${BLUE}git push${NC}" + +echo -e "\n${GREEN}Done!${NC}" diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..e841670c --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,10 @@ +# Ignore test outputs (bundles are large, should be in artifacts) +output/ + +# Ignore extracted bundle contents during local testing +extracted/ +tmp/ +*.tmp + +# Ignore local test runs +local-test-*/ diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..4f5fa4fb --- /dev/null +++ b/test/README.md @@ -0,0 +1,280 @@ +# Regression Test Suite + +This directory contains the regression test infrastructure for validating preflight and support bundle collectors. + +## Overview + +The regression test suite: +1. Provisions an ephemeral k3s cluster via Replicated Actions +2. Runs multiple preflight and support bundle specs +3. Compares output bundles against known-good baselines +4. Reports regressions (missing files, changed outputs) + +## Directory Structure + +``` +test/ +├── README.md # This file +├── baselines/ # Known-good baseline bundles +│ ├── preflight-v1beta3/ +│ │ └── baseline.tar.gz +│ ├── preflight-v1beta2/ +│ │ └── baseline.tar.gz +│ ├── supportbundle/ +│ │ └── baseline.tar.gz +│ └── metadata.json # Baseline metadata (git sha, date, k8s version) +└── output/ # Test run outputs (gitignored) + ├── preflight-v1beta3-bundle.tar.gz + ├── preflight-v1beta2-bundle.tar.gz + ├── supportbundle.tar.gz + └── diff-report-*.json +``` + +## Specs Under Test + +| Spec | File | Values | Description | +|------|------|--------|-------------| +| Preflight v1beta3 | `examples/preflight/complex-v1beta3.yaml` | `examples/preflight/values-complex-full.yaml` | Templated v1beta3 with ~30 analyzers | +| Preflight v1beta2 | `examples/preflight/all-analyzers-v1beta2.yaml` | N/A | Legacy v1beta2 format with all analyzer types | +| Support Bundle | `examples/collect/host/all-kubernetes-collectors.yaml` | N/A | Comprehensive collector suite | + +## Running Tests + +### Via GitHub Actions (Recommended) + +The regression test workflow runs automatically on: +- Push to `main` or `v1beta3` branches +- Pull requests +- Manual trigger via workflow_dispatch + +**Manual trigger:** +```bash +gh workflow run regression-test.yaml +``` + +### Locally (Manual) + +```bash +# 1. Build binaries +make bin/preflight bin/support-bundle + +# 2. Create k3s cluster (use your preferred method) +k3d cluster create test-cluster --wait + +# 3. Run specs +./bin/preflight examples/preflight/complex-v1beta3.yaml \ + --values examples/preflight/values-complex-full.yaml \ + --interactive=false + +./bin/preflight examples/preflight/all-analyzers-v1beta2.yaml \ + --interactive=false + +./bin/support-bundle examples/collect/host/all-kubernetes-collectors.yaml \ + --interactive=false + +# 4. Compare bundles (if baselines exist) +python3 scripts/compare_bundles.py \ + --baseline test/baselines/preflight-v1beta3/baseline.tar.gz \ + --current preflightbundle-*.tar.gz \ + --rules scripts/compare_rules.yaml \ + --report test/output/diff-report.json \ + --spec-type preflight + +# 5. Clean up +k3d cluster delete test-cluster +``` + +## Creating Initial Baselines + +If baselines don't exist yet (first time setup): + +1. **Run workflow to generate bundles:** + ```bash + gh workflow run regression-test.yaml + ``` + +2. **Download artifacts:** + ```bash + gh run download --name regression-test-results--1 + ``` + +3. **Inspect bundles manually:** + ```bash + tar -tzf preflight-v1beta3-bundle.tar.gz | head -20 + tar -xzf preflight-v1beta3-bundle.tar.gz + # Verify contents look correct + ``` + +4. **Copy as baselines and commit:** + ```bash + mkdir -p test/baselines/{preflight-v1beta3,preflight-v1beta2,supportbundle} + + cp preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz + cp preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz + cp supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz + + git add test/baselines/ + git commit -m "chore: add initial regression test baselines" + git push + ``` + +## Updating Baselines + +When legitimate changes occur (new collectors, changed output format): + +### Option 1: Automatic Update (Workflow Input) + +```bash +gh workflow run regression-test.yaml -f update_baselines=true +``` + +This will: +1. Run tests +2. Copy new bundles as baselines +3. Commit and push updated baselines + +** Use with caution!** Only use this after verifying changes are intentional. + +### Option 2: Manual Update + +```bash +# Download artifacts from a successful run +gh run download --name regression-test-results--1 + +# Replace baselines +cp preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz +cp preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz +cp supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz + +# Commit +git add test/baselines/ +git commit -m "chore: update regression baselines - reason for change" +git push +``` + +## Comparison Strategy + +The comparison uses a 3-tier approach: + +### 1. Exact Match (2 files) +Files compared byte-for-byte: +- `static-data.txt/static-data` - static data collector +- `version.yaml` - spec version +- Data collector files (`files/example.yaml`, `config/replicas.txt`) + +### 2. Structural Comparison (8 files) +Compare specific fields only, ignore variable values: +- **Database collectors** (`postgres/*.json`, `mysql/*.json`, etc.) - Compare `isConnected` boolean +- **DNS** (`dns/debug.json`) - Verify service exists, queries succeed +- **Registry** (`registry/*.json`) - Compare `exists` per image +- **HTTP** (`http*.json`) - Compare status code only + +### 3. Non-Empty Check (Everything Else) +For highly variable outputs: +- **cluster-resources** - UIDs, timestamps, resourceVersions vary +- **node-metrics** - All metric values constantly change +- **logs** - Timestamps in every line +- **run/exec collectors** - Random pod names, variable output +- And more... + +Strategy: Verify file exists, is non-empty, and (for JSON) is valid JSON. + +## Understanding Test Results + +### Passing Test +- All expected files present +- Exact match files identical +- Structural comparison fields match +- All files non-empty and valid + +### Failing Test - Regressions Detected + +**Files missing:** +``` +⚠ Missing in current: postgres/postgres-example.json +``` +→ Collector stopped producing output (regression) + +**Structural mismatch:** +``` +❌ postgres/postgres-example.json: database connection status changed: true -> false +``` +→ Collector behavior changed (potential regression) + +**Empty file:** +``` +❌ dns/debug.json: File is empty +``` +→ Collector failed to collect data (regression) + +### ℹ️ New Files (Not a Failure) +``` +ℹ New file in current: newcollector/output.json +``` +→ New collector added (expected when adding features) + +## Troubleshooting + +### Workflow fails: "No baseline found" +First time setup - baselines need to be created (see above). + +### Many "structural mismatch" failures +Check if cluster state changed: +- Different k8s version? +- Different installed components? +- Resources created/deleted? + +### Comparison fails with Python error +Ensure dependencies installed: +```bash +pip install pyyaml deepdiff +``` + +### Cluster creation times out +Check Replicated Actions limits: +```bash +# View cluster status +gh api /repos/replicatedhq/compatibility-actions/... +``` + +## Configuration Files + +### `scripts/compare_rules.yaml` +Defines comparison strategy per file pattern. + +**Add new rule:** +```yaml +preflight: + structural_compare: + "mycollector/*.json": "my_comparator_function" +``` + +Then implement `_compare_my_comparator_function()` in `scripts/compare_bundles.py`. + +### `scripts/compare_bundles.py` +Comparison engine - implements comparison logic. + +**Add new comparator:** +```python +def _compare_my_comparator_function(self, baseline: Dict, current: Dict) -> bool: + """Compare mycollector output.""" + # Your comparison logic + return baseline["field"] == current["field"] +``` + +### `.github/workflows/regression-test.yaml` +GitHub Actions workflow definition. + +## Tips + +- **Start simple**: Begin with baselines for v1beta2 only, add v1beta3 later +- **Iterate on rules**: Add structural comparisons as you discover false positives +- **Review diffs**: Always inspect diff reports before updating baselines +- **Document changes**: In baseline update commits, explain why output changed +- **Monitor runtime**: Workflow should complete in < 20 minutes + +## Related Documentation + +- [CI Regression Test Proposal](../ci-regression-test-proposal.md) +- [Collector Comparison Strategy](../collector-comparison-strategy.md) +- [Replicated Actions Docs](https://github.com/replicatedhq/replicated-actions) diff --git a/test/baselines/README.md b/test/baselines/README.md new file mode 100644 index 00000000..06b89b98 --- /dev/null +++ b/test/baselines/README.md @@ -0,0 +1,33 @@ +# Regression Test Baselines + +This directory contains known-good baseline bundles used for regression testing. + +## Directory Structure + +- `preflight-v1beta3/` - Baseline for complex-v1beta3.yaml spec +- `preflight-v1beta2/` - Baseline for all-analyzers-v1beta2.yaml spec +- `supportbundle/` - Baseline for all-kubernetes-collectors.yaml spec +- `metadata.json` - Metadata about when baselines were last updated + +## Creating Initial Baselines + +If this directory is empty, baselines need to be created: + +1. Run the regression test workflow manually +2. Download the artifacts +3. Inspect bundles to verify correctness +4. Use `scripts/update_baselines.sh` to copy them here + +See `test/README.md` for detailed instructions. + +## Updating Baselines + +Baselines should only be updated when: +- New collectors are added +- Collector output format changes intentionally +- Kubernetes version is upgraded +- Bug fixes that change collector behavior + +**Never update baselines to make failing tests pass without investigation!** + +Use `scripts/update_baselines.sh` to update from a workflow run. diff --git a/test/baselines/preflight-v1beta2/baseline.tar.gz b/test/baselines/preflight-v1beta2/baseline.tar.gz new file mode 100644 index 00000000..7b30829b Binary files /dev/null and b/test/baselines/preflight-v1beta2/baseline.tar.gz differ diff --git a/test/baselines/preflight-v1beta3/baseline.tar.gz b/test/baselines/preflight-v1beta3/baseline.tar.gz new file mode 100644 index 00000000..10499074 Binary files /dev/null and b/test/baselines/preflight-v1beta3/baseline.tar.gz differ diff --git a/test/baselines/supportbundle/baseline.tar.gz b/test/baselines/supportbundle/baseline.tar.gz new file mode 100644 index 00000000..b40e088a Binary files /dev/null and b/test/baselines/supportbundle/baseline.tar.gz differ