simplify comparison to detect code regressions only

scripts/compare_bundles.py

@@ -119,8 +119,10 @@ class BundleComparator:
                 "*-previous.log",  # Previous container logs (only exist after restart)
                 "node-metrics/*.json",  # Node IDs vary between clusters
                 "sysctl/*",  # Node IDs vary between clusters
-                "collectd/rrd/*",  # Node IDs vary between clusters
-                "copy-from-host-example/*",  # Node IDs vary between clusters
+                "collectd/rrd/*/**",  # Node IDs vary between clusters (with subdirs)
+                "collectd/rrd/*/*",  # Node IDs vary between clusters
+                "copy-from-host-example/*/**",  # Node IDs vary between clusters (with subdirs)
+                "copy-from-host-example/*/*",  # Node IDs vary between clusters
                 "run-daemonset-example/*.log",  # Node IDs vary between clusters
             ]
 

scripts/compare_rules.yaml

@@ -24,45 +24,28 @@ preflight:
   # Files that need structural/field-specific comparison
   # Format: "pattern": "comparator_function_name"
   structural_compare:
-    # Database collectors - compare isConnected boolean only
-    "postgres/*.json": "database_connection"
-    "mysql/*.json": "database_connection"
-    "mssql/*.json": "database_connection"
-    "redis/*.json": "database_connection"
+    # NOTE: Most collectors are now non-empty checks only
+    # We're testing that collectors RUN and PRODUCE OUTPUT, not that
+    # specific environmental values (versions, connection status, image availability) match
 
-    # DNS collector - compare structure, not exact values
-    "dns/debug.json": "dns_structure"
-
-    # Registry collector - compare exists boolean per image
-    "registry/*.json": "registry_exists"
-    "registry-images/*.json": "registry_exists"
-
-    # HTTP collector - compare status code only
-    "http/*.json": "http_status"
-    "http-*.json": "http_status"
-
-    # Cluster info - compare major/minor version, ignore build details
-    "cluster-info/cluster_version.json": "cluster_version"
-
-    # Analysis results - compare analyzer names and severity levels only
-    "analysis.json": "analysis_results"
+    # Keep structural comparison only for truly deterministic collectors:
+    # (Currently none - all moved to non-empty or exact match)
 
   # Everything else uses non-empty check by default
+  # This tests that collectors RUN and PRODUCE OUTPUT, not environmental state
   # This includes:
-  # - cluster-resources/**/*.json (UIDs, timestamps vary)
-  # - node-metrics/**/*.json (all values vary)
+  # - postgres/*.json, mysql/*.json, etc. (connection status varies)
+  # - dns/debug.json (IPs, pod names vary)
+  # - registry/*.json (image availability varies)
+  # - http/*.json (endpoint status varies)
+  # - cluster-info/*.json (k8s version varies)
+  # - analysis.json (analyzer results vary with cluster state)
+  # - cluster-resources/**/*.json (UIDs, timestamps, status vary)
+  # - node-metrics/**/*.json (all metric values vary)
   # - goldpinger/**/*.json (latencies vary)
-  # - run*/**/* (pod names, output vary)
-  # - ceph/**/* (status/metrics vary or not installed)
-  # - longhorn/**/* (status/metrics vary or not installed)
   # - certificates/**/*.json (validity time-based)
-  # - configmaps/**/*.json (can have dynamic values)
-  # - secrets/**/*.json (can have dynamic values)
-  # - sysctl/**/* (some counters vary)
-  # - collectd/**/* (time-series data)
-  # - helm/**/*.json (timestamps, revisions vary)
-  # - logs/**/*.log (timestamps in every line)
-  # - copy*/**/* (content depends on what's copied)
+  # - sysctl/* (some values vary)
+  # - And everything else...
 
 # Support bundle-specific rules
 supportbundle:
@@ -73,24 +56,9 @@ supportbundle:
 
   # Files that need structural comparison
   structural_compare:
-    # Database collectors
-    "postgres/*.json": "database_connection"
-    "mysql/*.json": "database_connection"
-    "mssql/*.json": "database_connection"
-    "redis/*.json": "database_connection"
-
-    # DNS collector
-    "dns/debug.json": "dns_structure"
-
-    # Registry collector
-    "registry/*.json": "registry_exists"
-    "registry-images/*.json": "registry_exists"
-
-    # HTTP collector
-    "http*.json": "http_status"
-
-    # Cluster info
-    "cluster-info/cluster_version.json": "cluster_version"
+    # NOTE: Like preflight, we only test that collectors produce output
+    # Environmental state (DB connections, registry access, etc.) will vary
+    # (Currently none - all moved to non-empty or exact match)
 
   # Everything else uses non-empty check (see list above in preflight section)