diff --git a/config/config.yaml b/config/config.yaml index fbac08ee..ae39897e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,6 +7,7 @@ kraken: signal_state: RUN # Will wait for the RUN signal when set to PAUSE before running the scenarios, refer docs/signal.md for more details signal_address: 0.0.0.0 # Signal listening address port: 8081 # Signal port + generate_pdf_report: True # Generate a PDF summary report after the run chaos_scenarios: # List of policies/chaos scenarios to load - hog_scenarios: @@ -62,6 +63,7 @@ kraken: - http_load_scenarios: - scenarios/kube/http_load_scenario.yml + resiliency: resiliency_run_mode: standalone # Options: standalone, detailed, disabled resiliency_file: config/alerts.yaml # Path to SLO definitions, will resolve to performance_monitoring: alert_profile: if not specified @@ -143,4 +145,4 @@ kubevirt_checks: # Utilizing virt che ssh_node: "" # If set, will be a backup way to ssh to a node. Will want to set to a node that isn't targeted in chaos node_names: "" exit_on_failure: # If value is True and VMI's are failing post chaos returns failure, values can be True/False - + \ No newline at end of file diff --git a/containers/krknctl-input.json b/containers/krknctl-input.json index fa794192..d54b2a51 100644 --- a/containers/krknctl-input.json +++ b/containers/krknctl-input.json @@ -104,6 +104,18 @@ "required": "false", "group": "general" }, + { + "name": "generate-pdf-report", + "short_description": "Generate PDF report", + "description": "Generate a PDF summary report after the chaos run", + "variable": "GENERATE_PDF_REPORT", + "type": "enum", + "allowed_values": "True,False", + "separator": ",", + "default": "True", + "required": "false", + "group": "general" + }, { "name": "krkn-debug", "short_description": "Krkn debug mode", diff --git a/krkn/resiliency/resiliency.py b/krkn/resiliency/resiliency.py index 1ddbfbe8..42d388f2 100644 --- a/krkn/resiliency/resiliency.py +++ b/krkn/resiliency/resiliency.py @@ -200,6 +200,21 @@ class Resiliency: raise RuntimeError("finalize_report() must be called first") return self.detailed_report + def get_scenario_slo_details(self) -> List[Dict[str, Any]]: + """Return per-scenario SLO details with name, severity, and pass/fail.""" + severity_map = {slo["name"]: slo["severity"] for slo in self._slos} + result = [] + for report in self.scenario_reports: + slo_details = [ + {"name": name, "severity": severity_map.get(name, "unknown"), "passed": passed} + for name, passed in report["slo_results"].items() + ] + result.append({ + "scenario": report["name"], + "slo_details": slo_details, + }) + return result + @staticmethod def compact_breakdown(report: Dict[str, Any]) -> Dict[str, int]: """Return a compact summary dict for a single scenario report.""" diff --git a/krkn/summarized_reports/__init__.py b/krkn/summarized_reports/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/krkn/summarized_reports/templates/report.html b/krkn/summarized_reports/templates/report.html new file mode 100644 index 00000000..f81a284c --- /dev/null +++ b/krkn/summarized_reports/templates/report.html @@ -0,0 +1,700 @@ + + + + +KRKN Run Summary + + + + +

KRKN Run Summary

+

Generated {{ generated_at }}

+ + +

Run Metadata

+ + + + + + + + {% if network_plugins %} + + {% endif %} + {% if security_flags %} + + {% endif %} +
Run UUID{{ run_uuid }}
Cluster Version{{ cluster_version }}
Infrastructure{{ cloud_infrastructure }}
Cloud Type{{ cloud_type }}
Time Window{{ time_window }}
Total Nodes{{ total_node_count }}
Network Plugins{{ network_plugins | join(', ') }}
Security{{ security_flags | join(', ') }}
+ + +{% if node_summary_infos %} +

Cluster Overview

+ + + + + + + + + + {% for ni in node_summary_infos %} + + + + + + + + + {% endfor %} +
TypeCountInstanceArchitectureKubeletOS
{{ ni.nodes_type or 'N/A' }}{{ ni.count or 'N/A' }}{{ ni.instance_type or 'N/A' }}{{ ni.architecture or 'N/A' }}{{ ni.kubelet_version or 'N/A' }}{{ ni.os_version or 'N/A' }}
+{% endif %} + + +

Targets

+{% for s in scenarios %} + + + {% set doc_url = scenario_type_docs.get(s.scenario_type, '') %} + + + + {% if s.selectors %} + + + + + {% endif %} + {% if s.namespaces %} + + + + + {% endif %} + {% if s.exclude_labels %} + + + + + {% endif %} + {% if s.cloud_types %} + + + + + {% endif %} + {% if s.all_pods %} + + + + + {% elif s.pods_error %} + + + + + {% endif %} + {% if s.all_vmis %} + + + + + {% elif s.vmis_error %} + + + + + {% endif %} + {% if s.affected_nodes %} + + + + + {% endif %} +
{{ s.scenario }} ({% if doc_url %}{{ s.scenario_type }}{% else %}{{ s.scenario_type }}{% endif %}) + {% if s.exit_status == "0" %} + PASS + {% else %} + FAIL + {% endif %} +
Label Selector{{ s.selectors | join(', ') }}
Namespace{{ s.namespaces | join(', ') }}
Exclude Label{{ s.exclude_labels | join(', ') }}
Cloud Type{{ s.cloud_types | join(', ') }}
Disrupted Pods +
    + {% for pod in s.all_pods %} +
  • {{ pod }}
    • + {% endfor %} +
+ {% if s.pods_error %} +

Monitoring error: {{ s.pods_error }}

+ {% endif %} +
Pod MonitoringError: {{ s.pods_error }}
Disrupted VMIs +
    + {% for vmi in s.all_vmis %} +
  • {{ vmi }}
    • + {% endfor %} +
+ {% if s.vmis_error %} +

Monitoring error: {{ s.vmis_error }}

+ {% endif %} +
VMI MonitoringError: {{ s.vmis_error }}
Affected Nodes +
    + {% for node in s.affected_nodes %} +
  • {{ node.node_name }}{% if node.node_id %} ({{ node.node_id }}){% endif %}
    • + {% endfor %} +
+
+{% endfor %} + + +{% set has_pods = scenarios | selectattr('all_pods') | list | length > 0 %} +{% if has_pods %} +

Key Metrics

+ + + + + + + + {% for s in scenarios %} + {% if s.recovered_count or s.unrecovered_count %} + + + + + + + {% endif %} + {% endfor %} +
ScenarioPods RecoveredPods UnrecoveredTotal Recovery Time
{{ s.scenario }}{{ s.recovered_count }}{{ s.unrecovered_count }} + {% if s.total_recovery_time is not none %} + {{ "%.2f"|format(s.total_recovery_time) }}s +
Rescheduling: {{ "%.2f"|format(s.rescheduling_time) }}s | Readiness: {{ "%.2f"|format(s.readiness_time) }}s + {% endif %} +
+{% endif %} + +{% set has_vmis = scenarios | selectattr('all_vmis') | list | length > 0 %} +{% if has_vmis %} +

VMI Recovery

+ + + + + + + + {% for s in scenarios %} + {% if s.all_vmis %} + + + + + + + {% endif %} + {% endfor %} +
ScenarioVMIs RecoveredVMIs UnrecoveredTotal Recovery Time
{{ s.scenario }}{{ s.vmi_recovered_count }}{{ s.vmi_unrecovered_count }} + {% if s.vmi_total_recovery_time is not none %} + {{ "%.2f"|format(s.vmi_total_recovery_time) }}s +
Rescheduling: {{ "%.2f"|format(s.vmi_rescheduling_time) }}s | Readiness: {{ "%.2f"|format(s.vmi_readiness_time) }}s + {% endif %} +
+{% endif %} + +{% set has_nodes = scenarios | selectattr('affected_nodes') | list | length > 0 %} +{% if has_nodes %} +

Node Recovery

+{% for s in scenarios %} +{% if s.affected_nodes %} + + + + + + + {% if s.affected_nodes | selectattr('node_id') | list | length > 0 %} + + {% endif %} + + + + + + + {% for node in s.affected_nodes %} + + + {% if s.affected_nodes | selectattr('node_id') | list | length > 0 %} + + {% endif %} + + + + + + + {% endfor %} +
{{ s.scenario }}
NodeInstance IDStoppedRunningTerminatedNot ReadyReady
{{ node.node_name }}{{ node.node_id }}{% if node.stopped_time %}{{ "%.2f"|format(node.stopped_time) }}s{% endif %}{% if node.running_time %}{{ "%.2f"|format(node.running_time) }}s{% endif %}{% if node.terminating_time %}{{ "%.2f"|format(node.terminating_time) }}s{% endif %}{% if node.not_ready_time %}{{ "%.2f"|format(node.not_ready_time) }}s{% endif %}{% if node.ready_time %}{{ "%.2f"|format(node.ready_time) }}s{% endif %}
+{% endif %} +{% endfor %} +{% endif %} + + +{% set has_additional = scenarios | selectattr('additional_telemetry') | list | length > 0 %} +{% if has_additional %} +

Load Test Metrics

+{% for s in scenarios %} +{% if s.additional_telemetry %} + + + {% for key, val in s.additional_telemetry.items() %} + + + + + {% endfor %} +
{{ s.scenario }}
{{ key }}{{ val }}
+{% endif %} +{% endfor %} +{% endif %} + + +{% set has_events = scenarios | selectattr('cluster_events') | list | length > 0 %} +{% if has_events %} +

Cluster Events

+{% for s in scenarios %} +{% if s.cluster_events %} + + + + + + + + + + + + {% for event in s.cluster_events[:10] %} + + {% if event is mapping %} + + + + + + {% else %} + + {% endif %} + + {% endfor %} + {% if s.cluster_events | length > 10 %} + + {% endif %} +
{{ s.scenario }} ({{ s.cluster_events | length }} events)
TypeReasonObjectMessageNamespace
+ {% if event.type == 'Warning' %} + {{ event.type }} + {% elif event.type %} + {{ event.type }} + {% endif %} + {{ event.reason or '' }}{% if event.involved_object_kind %}{{ event.involved_object_kind }}/{{ event.involved_object_name }}{% endif %}{{ event.message or '' }}{{ event.namespace or '' }}{{ event }}
... and {{ s.cluster_events | length - 10 }} more
+{% endif %} +{% endfor %} +{% endif %} + + +{% if health_checks %} +

Health Checks

+ + + + + + + + {% for check in health_checks %} + + {% if check is mapping %} + + + + + {% else %} + + {% endif %} + + {% endfor %} +
URL / EndpointStatus CodeDurationResult
{{ check.url or check.name or check.check_name or '' }}{{ check.status_code or '' }}{% if check.duration is not none and check.duration != '' %}{{ "%.2f"|format(check.duration|float) }}s{% endif %} + {% if check.status or check.passed %} + PASS + {% else %} + FAIL + {% endif %} + {{ check }}
+{% endif %} + + +{% if virt_checks %} +

KubeVirt Health Checks (Pre-Chaos)

+ + + + + + + + + + {% for check in virt_checks %} + + {% if check is mapping %} + + + + + + + {% else %} + + {% endif %} + + {% endfor %} +
VM NameNamespaceNodeIP AddressDurationResult
{{ check.vm_name or check.vmi_name or check.name or '' }}{{ check.namespace or '' }}{{ check.node_name or '' }}{{ check.ip_address or '' }}{% if check.duration is not none and check.duration != '' %}{{ "%.2f"|format(check.duration|float) }}s{% endif %} + {% if check.status is defined and not check.status %} + FAIL + {% else %} + PASS + {% endif %} + {{ check }}
+{% endif %} + + +{% if post_virt_checks %} +

KubeVirt Health Checks (Post-Chaos)

+ + + + + + + + + + + {% for check in post_virt_checks %} + + {% if check is mapping %} + + + + + + + + {% else %} + + {% endif %} + + {% endfor %} +
VM NameNamespaceNodeIP AddressNew IPDurationResult
{{ check.vm_name or check.vmi_name or check.name or '' }}{{ check.namespace or '' }}{{ check.node_name or '' }}{{ check.ip_address or '' }}{% if check.new_ip_address and check.new_ip_address != check.ip_address %}{{ check.new_ip_address }}{% endif %}{% if check.duration is not none and check.duration != '' %}{{ "%.2f"|format(check.duration|float) }}s{% endif %} + {% if check.status is defined and not check.status %} + FAIL + {% else %} + PASS + {% endif %} + {{ check }}
+{% endif %} + + +

Alerts & SLOs

+ + + + + + + + + + + + + + +
SLOs Evaluated{{ total_slos }}
SLOs Passed{{ passed_slos }} / {{ total_slos }}
SLOs Failed{% if failed_slos > 0 %}{{ failed_slos }}{% else %}{{ failed_slos }}{% endif %}
Critical Alerts{% if critical_alert_count > 0 %}{{ critical_alert_count }}{% else %}None{% endif %}
+ + +{% if chaos_alerts %} +

Critical Alerts (During Chaos)

+ + + + + + + + {% for alert in chaos_alerts %} + + {% if alert is mapping %} + + + + + {% else %} + + {% endif %} + + {% endfor %} +
Alert NameSeverityNamespaceState
{{ alert.alertname or 'N/A' }}{{ alert.severity or 'N/A' }}{{ alert.namespace or 'N/A' }}{{ alert.alertstate or 'N/A' }}{{ alert }}
+{% endif %} + +{% if post_chaos_alerts %} +

Critical Alerts (Post Chaos)

+ + + + + + + + {% for alert in post_chaos_alerts %} + + {% if alert is mapping %} + + + + + {% else %} + + {% endif %} + + {% endfor %} +
Alert NameSeverityNamespaceState
{{ alert.alertname or 'N/A' }}{{ alert.severity or 'N/A' }}{{ alert.namespace or 'N/A' }}{{ alert.alertstate or 'N/A' }}{{ alert }}
+{% endif %} + + +{% if error_logs %} +

Error Logs ({{ error_logs | length }})

+ + + + + + {% for log in error_logs[:20] %} + + {% if log is mapping %} + + + {% else %} + + + {% endif %} + + {% endfor %} + {% if error_logs | length > 20 %} + + {% endif %} +
TimestampMessage
{{ log.timestamp or '-' }}{{ log.message or log }}-{{ log }}
... and {{ error_logs | length - 20 }} more
+{% endif %} + + +{% set failed_entries = [] %} +{% for entry in scenario_slo_details %} + {% set failed_slos_list = entry.slo_details | selectattr('passed', 'false') | list %} + {% if failed_slos_list %} + {% set _ = failed_entries.append({"scenario": entry.scenario, "slo_details": failed_slos_list}) %} + {% endif %} +{% endfor %} +{% if failed_entries %} +

Failed SLOs

+{% for entry in failed_entries %} +

{{ entry.scenario }}

+ + + + + + + {% for slo in entry.slo_details %} + + + + + + {% endfor %} +
SLOSeverityStatus
{{ slo.name }}{{ slo.severity }}FAIL
+{% endfor %} +{% endif %} + + +

Resiliency Score

+{% if per_scenario_scores %} + + + + + + {% for name, score in per_scenario_scores.items() %} + + + + + {% endfor %} +
ScenarioScore
{{ name }} + {% if score >= 90 %} + {{ score }} / 100 + {% elif score >= 70 %} + {{ score }} / 100 + {% else %} + {{ score }} / 100 + {% endif %} +
+{% endif %} + +
+ Overall: {{ overall_score }} / 100 +
+ + + diff --git a/krkn/summarized_reports/transform.py b/krkn/summarized_reports/transform.py new file mode 100644 index 00000000..859ae10d --- /dev/null +++ b/krkn/summarized_reports/transform.py @@ -0,0 +1,689 @@ +import logging +import os +from datetime import datetime +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader + +SCENARIO_TYPE_DOCS = { + "pod_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/pod-disruption/", + "container_scenarios": "https://krkn-chaos.dev/docs/scenarios/container-scenarios/", + "node_scenarios": "https://krkn-chaos.dev/docs/scenarios/node-scenarios/", + "hog_scenarios": "https://krkn-chaos.dev/docs/scenarios/hog-scenarios/", + "zone_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/zone-outages/", + "application_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/application-outages/", + "pod_network_scenarios": "https://krkn-chaos.dev/docs/scenarios/pod-network-scenarios/", + "time_scenarios": "https://krkn-chaos.dev/docs/scenarios/time-scenarios/", + "cluster_shut_down_scenarios": "https://krkn-chaos.dev/docs/scenarios/cluster-shut-down/", + "pvc_scenarios": "https://krkn-chaos.dev/docs/scenarios/pvc-scenarios/", + "network_chaos_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos/", + "network_chaos_ng_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos/", + "service_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-disruption/", + "service_hijacking_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-hijacking/", + "syn_flood_scenarios": "https://krkn-chaos.dev/docs/scenarios/syn-flood/", + "http_load_scenarios": "https://krkn-chaos.dev/docs/scenarios/http-load/", + "kubevirt_vm_outage": "https://krkn-chaos.dev/docs/scenarios/kubevirt-vm-outage/", + "managedcluster_scenarios": "https://krkn-chaos.dev/docs/scenarios/managed-cluster/", + "storage_throttle_scenarios": "https://krkn-chaos.dev/docs/scenarios/storage-throttle/", +} + + +def format_ts(unix_ts): + return datetime.fromtimestamp(unix_ts).strftime("%Y-%m-%d %H:%M:%S") + + +def format_window(start_ts, end_ts): + start_dt = datetime.fromtimestamp(start_ts) + end_dt = datetime.fromtimestamp(end_ts) + if start_dt.date() == end_dt.date(): + return f"{start_dt.strftime('%Y-%m-%d %H:%M:%S')} – {end_dt.strftime('%H:%M:%S')}" + return f"{start_dt.strftime('%Y-%m-%d %H:%M:%S')} – {end_dt.strftime('%Y-%m-%d %H:%M:%S')}" + + +def _extract_scenario_params(raw_params): + """Extract label_selectors, namespaces, and exclude_labels from all parameter shapes. + + Walks the entire parameter tree recursively so it works regardless of + how deeply the keys are nested (pod_disruption, container, node, hog, + kubevirt, network_chaos, time_scenarios, application_outage, pvc, etc.). + """ + selectors = [] + namespaces = [] + exclude_labels = [] + cloud_types = [] + + def _walk(obj): + if isinstance(obj, dict): + ls = obj.get("label_selector") + if ls and isinstance(ls, str): + selectors.append(ls) + ns = obj.get("node-selector") + if ns and isinstance(ns, str): + selectors.append(ns) + nls = obj.get("node_label_selector") + if nls and isinstance(nls, str): + selectors.append(nls) + for ns_key in ("namespace_pattern", "namespace", "service_namespace"): + nsp = obj.get(ns_key) + if nsp and isinstance(nsp, str): + namespaces.append(nsp) + break + el = obj.get("exclude_label") + if el and isinstance(el, str): + exclude_labels.append(el) + ct = obj.get("cloud_type") + if ct and isinstance(ct, str): + cloud_types.append(ct) + for v in obj.values(): + if isinstance(v, (dict, list)): + _walk(v) + elif isinstance(obj, list): + for item in obj: + if isinstance(item, (dict, list)): + _walk(item) + + _walk(raw_params) + + # deduplicate while preserving order + seen = set() + selectors = [s for s in selectors if not (s in seen or seen.add(s))] + seen = set() + namespaces = [n for n in namespaces if not (n in seen or seen.add(n))] + seen = set() + exclude_labels = [e for e in exclude_labels if not (e in seen or seen.add(e))] + seen = set() + cloud_types = [c for c in cloud_types if not (c in seen or seen.add(c))] + + return selectors, namespaces, exclude_labels, cloud_types + + +def _extract_pod_name(pod): + if isinstance(pod, dict): + ns = pod.get("namespace", "") + name = pod.get("pod_name", str(pod)) + return f"{ns}/{name}" if ns else name + return str(pod) + + +def _extract_vmi_name(vmi): + if isinstance(vmi, dict): + ns = vmi.get("namespace", "") + name = vmi.get("vmi_name", str(vmi)) + return f"{ns}/{name}" if ns else name + return str(vmi) + + +def _extract_critical_alerts(critical_alerts_raw): + if isinstance(critical_alerts_raw, dict): + chaos = critical_alerts_raw.get("chaos_alerts", []) + post = critical_alerts_raw.get("post_chaos_alerts", []) + return chaos, post + if isinstance(critical_alerts_raw, list): + return critical_alerts_raw, [] + return [], [] + + +def build_chaos_report(chaos_output: dict) -> str: + telemetry = chaos_output.get("telemetry", {}) + scenarios = telemetry.get("scenarios", []) + job_status = telemetry.get("job_status", True) + + lines = [] + lines.append("=" * 80) + lines.append("KRKN RUN SUMMARY") + lines.append("=" * 80) + + # --- Run Metadata --- + lines.append("Run UUID : " + str(telemetry.get("run_uuid", "N/A"))) + lines.append("Status : " + ("PASS" if job_status else "FAIL")) + lines.append( + "Cluster : " + + str(telemetry.get("cluster_version", "N/A")) + + " (" + + str(telemetry.get("cloud_infrastructure", "N/A")) + + ", " + + str(telemetry.get("cloud_type", "N/A")) + + ")" + ) + + starts = [s["start_timestamp"] for s in scenarios if s.get("start_timestamp")] + ends = [s["end_timestamp"] for s in scenarios if s.get("end_timestamp")] + if starts and ends: + lines.append("Window : " + format_window(min(starts), max(ends))) + else: + lines.append("Window : N/A") + lines.append("Nodes : " + str(telemetry.get("total_node_count", "N/A"))) + network = telemetry.get("network_plugins") or [] + if network: + lines.append("Network : " + ", ".join(network)) + security_flags = [] + if telemetry.get("fips_enabled"): + security_flags.append("FIPS") + if telemetry.get("etcd_encryption_enabled"): + security_flags.append("etcd encryption") + if telemetry.get("ipsec_enabled"): + security_flags.append("IPSec") + if security_flags: + lines.append("Security : " + ", ".join(security_flags)) + + # --- Cluster Overview --- + node_infos = telemetry.get("node_summary_infos") or [] + if node_infos: + lines.append("CLUSTER OVERVIEW") + lines.append(f" {'Type':<8} {'Count':<6} {'Instance':<14} {'Arch':<7} {'Kubelet':<10} OS") + for ni in node_infos: + lines.append( + f" {ni.get('nodes_type', 'N/A'):<8} " + f"{ni.get('count', 'N/A'):<6} " + f"{ni.get('instance_type', 'N/A'):<14} " + f"{ni.get('architecture', 'N/A'):<7} " + f"{ni.get('kubelet_version', 'N/A'):<10} " + f"{ni.get('os_version', 'N/A')}" + ) + + # --- Targets --- + lines.append("TARGETS") + for i, s in enumerate(scenarios, 1): + lines.append(f" [{i}] Scenario : " + str(s.get("scenario", "N/A")) + " (" + str(s.get("scenario_type", "N/A")) + ")") + selectors, ns_list, exclude_labels, cloud_types = _extract_scenario_params(s.get("parameters", {})) + if selectors: + lines.append(" Label Selector : " + ", ".join(selectors)) + if ns_list: + lines.append(" Namespace : " + ", ".join(ns_list)) + if exclude_labels: + lines.append(" Exclude Label : " + ", ".join(exclude_labels)) + if cloud_types: + lines.append(" Cloud Type : " + ", ".join(cloud_types)) + + recovered = s.get("affected_pods", {}).get("recovered", []) + unrecovered = s.get("affected_pods", {}).get("unrecovered", []) + pods_error = s.get("affected_pods", {}).get("error") + if unrecovered or recovered: + lines.append(" Pods Disrupted :") + for pod in unrecovered + recovered: + lines.append(" - " + _extract_pod_name(pod)) + if pods_error: + lines.append(f" Pod Monitoring Error: {pods_error}") + + vmi_recovered = s.get("affected_vmis", {}).get("recovered", []) + vmi_unrecovered = s.get("affected_vmis", {}).get("unrecovered", []) + vmis_error = s.get("affected_vmis", {}).get("error") + if vmi_recovered or vmi_unrecovered: + lines.append(" VMIs Disrupted :") + for vmi in vmi_unrecovered + vmi_recovered: + lines.append(" - " + _extract_vmi_name(vmi)) + if vmis_error: + lines.append(f" VMI Monitoring Error: {vmis_error}") + + affected_nodes = s.get("affected_nodes", []) + if affected_nodes: + lines.append(" Nodes Affected :") + for node in affected_nodes: + if isinstance(node, dict): + node_id = node.get("node_id", "") + label = node.get("node_name", str(node)) + if node_id: + label += f" ({node_id})" + lines.append(" - " + label) + else: + lines.append(" - " + str(node)) + + # --- Key Metrics --- + lines.append("KEY METRICS") + for i, s in enumerate(scenarios, 1): + lines.append(f" [{i}] Scenario: " + str(s.get("scenario", "N/A"))) + + exit_status = str(s.get("exit_status", "1")) + status = "PASS (0)" if exit_status == "0" else "FAIL (1)" + lines.append(" Exit Status : " + status) + + recovered = s.get("affected_pods", {}).get("recovered", []) + unrecovered = s.get("affected_pods", {}).get("unrecovered", []) + if recovered or unrecovered: + lines.append(" Pods Recovered : " + str(len(recovered))) + lines.append(" Pods Unrecovered : " + str(len(unrecovered))) + + pod_recovery_times = [ + p.get("total_recovery_time") + for p in recovered + if isinstance(p, dict) and p.get("total_recovery_time") is not None + ] + if pod_recovery_times: + total = max(pod_recovery_times) + reschedule_times = [p.get("pod_rescheduling_time") or 0 for p in recovered if isinstance(p, dict)] + readiness_times = [p.get("pod_readiness_time") or 0 for p in recovered if isinstance(p, dict)] + lines.append(f" Total Recovery Time : {total:.2f}s") + lines.append(f" ├─ Rescheduling Time: {max(reschedule_times):.2f}s") + lines.append(f" └─ Readiness Time : {max(readiness_times):.2f}s") + + vmi_recovered = s.get("affected_vmis", {}).get("recovered", []) + vmi_unrecovered = s.get("affected_vmis", {}).get("unrecovered", []) + if vmi_recovered or vmi_unrecovered: + lines.append(" VMIs Recovered : " + str(len(vmi_recovered))) + lines.append(" VMIs Unrecovered : " + str(len(vmi_unrecovered))) + vmi_recovery_times = [ + v.get("total_recovery_time") + for v in vmi_recovered + if isinstance(v, dict) and v.get("total_recovery_time") is not None + ] + if vmi_recovery_times: + total = max(vmi_recovery_times) + reschedule_times = [v.get("vmi_rescheduling_time") or 0 for v in vmi_recovered if isinstance(v, dict)] + readiness_times = [v.get("vmi_readiness_time") or 0 for v in vmi_recovered if isinstance(v, dict)] + lines.append(f" VMI Recovery Time : {total:.2f}s") + lines.append(f" ├─ Rescheduling Time: {max(reschedule_times):.2f}s") + lines.append(f" └─ Readiness Time : {max(readiness_times):.2f}s") + + affected_nodes = s.get("affected_nodes", []) + if affected_nodes: + lines.append(f" Nodes Affected : {len(affected_nodes)}") + for node in affected_nodes: + if not isinstance(node, dict): + continue + node_label = node.get("node_name", "N/A") + node_id = node.get("node_id", "") + if node_id: + node_label += f" ({node_id})" + lines.append(f" Node: {node_label}") + timings = [] + for key, label in [ + ("stopped_time", "Stopped Time"), + ("running_time", "Running Time"), + ("terminating_time", "Terminating Time"), + ("not_ready_time", "Not Ready Time"), + ("ready_time", "Ready Time"), + ]: + val = node.get(key) + if val is not None and val > 0: + timings.append((label, val)) + for i, (label, val) in enumerate(timings): + connector = "└─" if i == len(timings) - 1 else "├─" + lines.append(f" {connector} {label:<17}: {val:.2f}s") + + # Additional telemetry (HTTP load test / Vegeta metrics) + additional = s.get("additional_telemetry") + if additional and isinstance(additional, dict): + lines.append(" Load Test Metrics:") + for metric_key, metric_val in additional.items(): + lines.append(f" {metric_key}: {metric_val}") + + cluster_events = s.get("cluster_events") or [] + if cluster_events: + lines.append(f" Cluster Events : {len(cluster_events)}") + for event in cluster_events[:10]: + if isinstance(event, dict): + etype = event.get("type", "") + reason = event.get("reason", "N/A") + msg = event.get("message", "N/A") + obj_kind = event.get("involved_object_kind", "") + obj_name = event.get("involved_object_name", "") + ns = event.get("namespace", "") + prefix = f"[{etype}] " if etype else "" + obj_ref = f" ({obj_kind}/{obj_name})" if obj_kind else "" + ns_ref = f" in {ns}" if ns else "" + lines.append(f" - {prefix}{reason}: {msg}{obj_ref}{ns_ref}") + else: + lines.append(f" - {event}") + if len(cluster_events) > 10: + lines.append(f" ... and {len(cluster_events) - 10} more") + + # --- Health Checks --- + health_checks = telemetry.get("health_checks") + if health_checks: + lines.append("HEALTH CHECKS") + for check in health_checks: + if isinstance(check, dict): + url = check.get("url", "") + status_code = check.get("status_code", "") + duration = check.get("duration") + passed = check.get("status") or check.get("passed") + status_str = "PASS" if passed else "FAIL" + detail = url or check.get("name") or check.get("check_name", "N/A") + extra = [] + if status_code: + extra.append(f"HTTP {status_code}") + if duration is not None and duration != "": + extra.append(f"{float(duration):.2f}s") + suffix = f" ({', '.join(extra)})" if extra else "" + lines.append(f" {status_str:<6} {detail}{suffix}") + else: + lines.append(f" {check}") + + # --- KubeVirt Health Checks --- + virt_checks = telemetry.get("virt_checks") + if virt_checks: + lines.append("KUBEVIRT HEALTH CHECKS (pre-chaos)") + for check in virt_checks: + if isinstance(check, dict): + vm = check.get("vm_name") or check.get("vmi_name") or check.get("name", "N/A") + ns = check.get("namespace", "") + node = check.get("node_name", "") + ip = check.get("ip_address", "") + passed = check.get("status", True) + duration = check.get("duration") + status_str = "PASS" if passed else "FAIL" + label = f"{ns}/{vm}" if ns else vm + extra = [] + if ip: + extra.append(ip) + if node: + extra.append(f"on {node}") + if duration is not None and duration != "": + extra.append(f"{float(duration):.2f}s") + suffix = f" ({', '.join(extra)})" if extra else "" + lines.append(f" {status_str:<6} {label}{suffix}") + else: + lines.append(f" {check}") + + post_virt_checks = telemetry.get("post_virt_checks") + if post_virt_checks: + lines.append("KUBEVIRT HEALTH CHECKS (post-chaos)") + for check in post_virt_checks: + if isinstance(check, dict): + vm = check.get("vm_name") or check.get("vmi_name") or check.get("name", "N/A") + ns = check.get("namespace", "") + node = check.get("node_name", "") + ip = check.get("ip_address", "") + new_ip = check.get("new_ip_address", "") + passed = check.get("status", True) + duration = check.get("duration") + status_str = "PASS" if passed else "FAIL" + label = f"{ns}/{vm}" if ns else vm + extra = [] + if ip: + ip_str = ip + if new_ip and new_ip != ip: + ip_str += f" → {new_ip}" + extra.append(ip_str) + if node: + extra.append(f"on {node}") + if duration is not None and duration != "": + extra.append(f"{float(duration):.2f}s") + suffix = f" ({', '.join(extra)})" if extra else "" + lines.append(f" {status_str:<6} {label}{suffix}") + else: + lines.append(f" {check}") + + # --- Alerts & SLOs --- + + scenario_slo_details = chaos_output.get("scenario_slo_details", []) + resiliency = telemetry.get("overall_resiliency_report", {}) + total_slos = resiliency.get("total_slos", 0) + passed_slos = resiliency.get("passed_slos", 0) + failed_slos = total_slos - passed_slos + critical_alerts_raw = chaos_output.get("critical_alerts") or {} + chaos_alerts, post_chaos_alerts = _extract_critical_alerts(critical_alerts_raw) + error_logs = telemetry.get("error_logs") or [] + + lines.append("ALERTS & SLOs") + lines.append(" SLOs Evaluated : " + str(total_slos)) + lines.append(f" SLOs Passed : {passed_slos} / {total_slos}") + lines.append(" SLOs Failed : " + str(failed_slos)) + + total_alert_count = len(chaos_alerts) + len(post_chaos_alerts) + lines.append(" Critical Alerts : " + (str(total_alert_count) if total_alert_count else "None")) + if chaos_alerts: + lines.append(" During Chaos:") + for alert in chaos_alerts: + if isinstance(alert, dict): + lines.append( + f" - {alert.get('alertname', 'N/A')} " + f"[{alert.get('severity', 'N/A')}] " + f"ns={alert.get('namespace', 'N/A')} " + f"state={alert.get('alertstate', 'N/A')}" + ) + else: + lines.append(f" - {alert}") + if post_chaos_alerts: + lines.append(" Post Chaos:") + for alert in post_chaos_alerts: + if isinstance(alert, dict): + lines.append( + f" - {alert.get('alertname', 'N/A')} " + f"[{alert.get('severity', 'N/A')}] " + f"ns={alert.get('namespace', 'N/A')} " + f"state={alert.get('alertstate', 'N/A')}" + ) + else: + lines.append(f" - {alert}") + + if error_logs: + lines.append(f" Error Logs : {len(error_logs)}") + for log_entry in error_logs[:20]: + if isinstance(log_entry, dict): + ts = log_entry.get("timestamp", "") + msg = log_entry.get("message", str(log_entry)) + lines.append(f" [{ts}] {msg}" if ts else f" {msg}") + else: + lines.append(f" {log_entry}") + if len(error_logs) > 20: + lines.append(f" ... and {len(error_logs) - 20} more") + + # --- Failed SLOs --- + if scenario_slo_details: + has_failures = any( + not s["passed"] + for entry in scenario_slo_details + for s in entry.get("slo_details", []) + ) + if has_failures: + lines.append("FAILED SLOs (per scenario)") + for entry in scenario_slo_details: + failed = [s for s in entry.get("slo_details", []) if not s["passed"]] + if not failed: + continue + lines.append(f" Scenario: {entry['scenario']}") + for slo in failed: + lines.append(f" FAIL [{slo.get('severity', 'unknown'):<8}] {slo['name']}") + + # --- Resiliency Score --- + lines.append("RESILIENCY SCORE") + per_scenario_scores = resiliency.get("scenarios", {}) + for scenario_name, score in per_scenario_scores.items(): + lines.append(f" {scenario_name:<28} : {score} / 100") + + overall_score = resiliency.get("resiliency_score", "N/A") + emoji = "" + if isinstance(overall_score, (int, float)): + emoji = " ✅" if overall_score >= 90 else (" ⚠️" if overall_score >= 70 else " ❌") + lines.append(f" Overall Score : {overall_score} / 100{emoji}") + lines.append("=" * 80) + + return "\n".join(lines) + + +def build_chaos_report_pdf(chaos_output: dict, output_path: str) -> str: + from weasyprint import HTML + logging.getLogger("weasyprint").setLevel(logging.WARNING) + + telemetry = chaos_output.get("telemetry", {}) + scenarios_raw = telemetry.get("scenarios", []) + + starts = [s["start_timestamp"] for s in scenarios_raw if s.get("start_timestamp")] + ends = [s["end_timestamp"] for s in scenarios_raw if s.get("end_timestamp")] + time_window = format_window(min(starts), max(ends)) if starts and ends else "N/A" + + scenarios = [] + for s in scenarios_raw: + selectors, ns_list, exclude_labels, cloud_types = _extract_scenario_params(s.get("parameters", {})) + + recovered = s.get("affected_pods", {}).get("recovered", []) + unrecovered = s.get("affected_pods", {}).get("unrecovered", []) + all_pods = [_extract_pod_name(p) for p in unrecovered + recovered] + pods_error = s.get("affected_pods", {}).get("error") + + pod_recovery_times = [ + p.get("total_recovery_time") + for p in recovered + if isinstance(p, dict) and p.get("total_recovery_time") is not None + ] + if pod_recovery_times: + total_recovery = max(pod_recovery_times) + reschedule = max(p.get("pod_rescheduling_time") or 0 for p in recovered if isinstance(p, dict)) + readiness = max(p.get("pod_readiness_time") or 0 for p in recovered if isinstance(p, dict)) + else: + total_recovery = None + reschedule = None + readiness = None + + vmi_recovered = s.get("affected_vmis", {}).get("recovered", []) + vmi_unrecovered = s.get("affected_vmis", {}).get("unrecovered", []) + all_vmis = [_extract_vmi_name(v) for v in vmi_unrecovered + vmi_recovered] + vmis_error = s.get("affected_vmis", {}).get("error") + + vmi_recovery_times = [ + v.get("total_recovery_time") + for v in vmi_recovered + if isinstance(v, dict) and v.get("total_recovery_time") is not None + ] + if vmi_recovery_times: + vmi_total_recovery = max(vmi_recovery_times) + vmi_reschedule = max(v.get("vmi_rescheduling_time") or 0 for v in vmi_recovered if isinstance(v, dict)) + vmi_readiness = max(v.get("vmi_readiness_time") or 0 for v in vmi_recovered if isinstance(v, dict)) + else: + vmi_total_recovery = None + vmi_reschedule = None + vmi_readiness = None + + affected_nodes = s.get("affected_nodes", []) + node_list = [] + for node in affected_nodes: + if isinstance(node, dict): + node_list.append({ + "node_name": node.get("node_name", "N/A"), + "node_id": node.get("node_id", ""), + "stopped_time": node.get("stopped_time"), + "running_time": node.get("running_time"), + "terminating_time": node.get("terminating_time"), + "not_ready_time": node.get("not_ready_time"), + "ready_time": node.get("ready_time"), + }) + + # Cluster events with full detail + raw_events = s.get("cluster_events") or [] + cluster_events = [] + for event in raw_events: + if isinstance(event, dict): + cluster_events.append({ + "reason": event.get("reason", "N/A"), + "message": event.get("message", "N/A"), + "type": event.get("type", ""), + "namespace": event.get("namespace", ""), + "source_component": event.get("source_component", ""), + "involved_object_kind": event.get("involved_object_kind", ""), + "involved_object_name": event.get("involved_object_name", ""), + "creation": event.get("creation", ""), + }) + else: + cluster_events.append({"reason": "Event", "message": str(event)}) + + # Additional telemetry (HTTP load test metrics) + additional_telemetry = s.get("additional_telemetry") + + scenarios.append({ + "scenario": s.get("scenario", "N/A"), + "scenario_type": s.get("scenario_type", "N/A"), + "selectors": selectors, + "namespaces": ns_list, + "exclude_labels": exclude_labels, + "cloud_types": cloud_types, + "all_pods": all_pods, + "pods_error": pods_error, + "exit_status": str(s.get("exit_status", "1")), + "recovered_count": len(recovered), + "unrecovered_count": len(unrecovered), + "total_recovery_time": total_recovery, + "rescheduling_time": reschedule, + "readiness_time": readiness, + "all_vmis": all_vmis, + "vmis_error": vmis_error, + "vmi_recovered_count": len(vmi_recovered), + "vmi_unrecovered_count": len(vmi_unrecovered), + "vmi_total_recovery_time": vmi_total_recovery, + "vmi_rescheduling_time": vmi_reschedule, + "vmi_readiness_time": vmi_readiness, + "affected_nodes": node_list, + "cluster_events": cluster_events, + "additional_telemetry": additional_telemetry, + }) + + resiliency = telemetry.get("overall_resiliency_report", {}) + total_slos = resiliency.get("total_slos", 0) + passed_slos = resiliency.get("passed_slos", 0) + + critical_alerts_raw = chaos_output.get("critical_alerts") or {} + chaos_alerts, post_chaos_alerts = _extract_critical_alerts(critical_alerts_raw) + total_alert_count = len(chaos_alerts) + len(post_chaos_alerts) + + error_logs = telemetry.get("error_logs") or [] + + template_dir = Path(__file__).parent / "templates" + env = Environment(loader=FileSystemLoader(str(template_dir)), autoescape=True) + template = env.get_template("report.html") + + scenario_slo_details = chaos_output.get("scenario_slo_details", []) + + security_flags = [] + if telemetry.get("fips_enabled"): + security_flags.append("FIPS") + if telemetry.get("etcd_encryption_enabled"): + security_flags.append("etcd encryption") + if telemetry.get("ipsec_enabled"): + security_flags.append("IPSec") + + node_infos = telemetry.get("node_summary_infos") or [] + health_checks = telemetry.get("health_checks") + virt_checks = telemetry.get("virt_checks") + post_virt_checks = telemetry.get("post_virt_checks") + + SCENARIO_TYPE_DOCS = { + "hog_scenarios": "https://krkn-chaos.dev/docs/scenarios/hog-scenarios/", + "application_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/application-outages/", + "container_scenarios": "https://krkn-chaos.dev/docs/scenarios/container-scenarios/", + "pod_network_scenarios": "https://krkn-chaos.dev/docs/scenarios/pod-network-scenario/", + "pod_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-disruption-scenarios/", + "node_scenarios": "https://krkn-chaos.dev/docs/scenarios/node-scenarios/", + "time_scenarios": "https://krkn-chaos.dev/docs/scenarios/time-scenarios/", + "cluster_shut_down_scenarios": "https://krkn-chaos.dev/docs/scenarios/power-outage-scenarios/", + "service_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-disruption-scenarios/", + "zone_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/zone-outage-scenarios/", + "pvc_scenarios": "https://krkn-chaos.dev/docs/scenarios/pvc-scenario/", + "storage_throttle_scenarios": "https://krkn-chaos.dev/docs/scenarios/storage-throttle-scenario/", + "network_chaos_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos-scenario/", + "service_hijacking_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-hijacking-scenario/", + "syn_flood_scenarios": "https://krkn-chaos.dev/docs/scenarios/syn-flood-scenario/", + "network_chaos_ng_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos-ng-scenarios/", + "kubevirt_vm_outage": "https://krkn-chaos.dev/docs/scenarios/kubevirt-vm-outage-scenario/", + "http_load_scenarios": "https://krkn-chaos.dev/docs/scenarios/http-load-scenario/" + } + + html_content = template.render( + generated_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + run_uuid=telemetry.get("run_uuid", "N/A"), + cluster_version=telemetry.get("cluster_version", "N/A"), + cloud_infrastructure=telemetry.get("cloud_infrastructure", "N/A"), + cloud_type=telemetry.get("cloud_type", "N/A"), + time_window=time_window, + total_node_count=telemetry.get("total_node_count", "N/A"), + network_plugins=telemetry.get("network_plugins") or [], + security_flags=security_flags, + node_summary_infos=node_infos, + health_checks=health_checks, + virt_checks=virt_checks, + post_virt_checks=post_virt_checks, + scenarios=scenarios, + total_slos=total_slos, + passed_slos=passed_slos, + failed_slos=total_slos - passed_slos, + scenario_slo_details=scenario_slo_details, + critical_alert_count=total_alert_count, + chaos_alerts=chaos_alerts, + post_chaos_alerts=post_chaos_alerts, + error_logs=error_logs, + per_scenario_scores=resiliency.get("scenarios", {}), + overall_score=resiliency.get("resiliency_score", "N/A"), + scenario_type_docs=SCENARIO_TYPE_DOCS + ) + + HTML(string=html_content).write_pdf(output_path) + return output_path diff --git a/requirements.txt b/requirements.txt index 2d6cfc96..c9ce3246 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ ibm-cloud-sdk-core>=3.24.4 # Requires requests>=2.32.4 ibm_vpc==0.26.3 # Requires ibm_cloud_sdk_core jinja2==3.1.6 jaraco-context>=6.1.0 # Fixes GHSA-58pv-8j8x-9vj2 +weasyprint>=63.0 cbor2<5.7.0 # Pinned by arcaflow-plugin-sdk lxml==6.1.0 kubernetes>=35.0.0,<36.0.0 diff --git a/run_kraken.py b/run_kraken.py index 139ddd2e..8445ac0f 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -72,6 +72,7 @@ from krkn.rollback.command import ( list_rollback as list_rollback_command, execute_rollback as execute_rollback_command, ) +from krkn.summarized_reports.transform import build_chaos_report, build_chaos_report_pdf from krkn.scenario_plugins.triggers.trigger_manager import TriggerManager # removes TripleDES warning @@ -104,6 +105,7 @@ def main(options, command: Optional[str]) -> int: config["kraken"], "publish_kraken_status", False ) port = get_yaml_item_value(config["kraken"], "port", 8081) + generate_pdf_report = get_yaml_item_value(config["kraken"], "generate_pdf_report", True) rollback_versions_dir = get_yaml_item_value( config["kraken"], "rollback_versions_directory", @@ -670,6 +672,26 @@ def main(options, command: Optional[str]) -> int: ) chaos_output.telemetry = decoded_chaos_run_telemetry logging.info(f"Chaos data:\n{chaos_output.to_json()}") + + chaos_output_dict = json.loads(chaos_output.to_json()) + if resiliency_obj and hasattr(resiliency_obj, 'scenario_reports') and resiliency_obj.scenario_reports: + chaos_output_dict["scenario_slo_details"] = resiliency_obj.get_scenario_slo_details() + try: + text_summary = build_chaos_report(chaos_output_dict) + logging.info(f"\n{text_summary}") + except Exception as e: + logging.error(f"Failed to build text summary: {e}") + + if generate_pdf_report: + pdf_path = report_file + ".pdf" + try: + abs_pdf_path = os.path.abspath(pdf_path) + build_chaos_report_pdf(chaos_output_dict, abs_pdf_path) + logging.info("PDF report generated: %s", abs_pdf_path) + print(f"\nfile://{abs_pdf_path}\n") + except Exception as e: + logging.error(f"Failed to generate PDF report: {e}") + if enable_elastic: result = elastic_search.push_telemetry( decoded_chaos_run_telemetry, elastic_telemetry_index diff --git a/tests/test_summarized_reports.py b/tests/test_summarized_reports.py new file mode 100644 index 00000000..95221056 --- /dev/null +++ b/tests/test_summarized_reports.py @@ -0,0 +1,962 @@ +#!/usr/bin/env python3 + +""" +Test suite for krkn.summarized_reports.transform module + +Usage: + python -m coverage run -a -m unittest tests/test_summarized_reports.py -v + +Assisted By: Claude Code +""" + +import os +import tempfile +import unittest +from unittest.mock import patch, MagicMock + +from krkn.summarized_reports.transform import ( + build_chaos_report, + build_chaos_report_pdf, + format_ts, + format_window, + _extract_scenario_params, + _extract_pod_name, + _extract_vmi_name, + _extract_critical_alerts, +) + + +def _minimal_chaos_output(**overrides): + base = { + "telemetry": { + "run_uuid": "test-uuid-1234", + "cluster_version": "4.22.0", + "cloud_infrastructure": "AWS", + "cloud_type": "self-managed", + "total_node_count": 6, + "network_plugins": ["OVNKubernetes"], + "scenarios": [], + "overall_resiliency_report": { + "total_slos": 0, + "passed_slos": 0, + "resiliency_score": 100, + "scenarios": {}, + }, + }, + } + base.update(overrides) + return base + + +def _make_scenario(scenario_name="test.yml", scenario_type="pod_disruption_scenarios", + exit_status=0, parameters=None, affected_pods=None, + affected_vmis=None, affected_nodes=None, + additional_telemetry=None, cluster_events=None, + start_timestamp=1000000, end_timestamp=1000060): + s = { + "scenario": scenario_name, + "scenario_type": scenario_type, + "exit_status": exit_status, + "parameters": parameters or {}, + "start_timestamp": start_timestamp, + "end_timestamp": end_timestamp, + } + if affected_pods is not None: + s["affected_pods"] = affected_pods + if affected_vmis is not None: + s["affected_vmis"] = affected_vmis + if affected_nodes is not None: + s["affected_nodes"] = affected_nodes + if additional_telemetry is not None: + s["additional_telemetry"] = additional_telemetry + if cluster_events is not None: + s["cluster_events"] = cluster_events + return s + + +class TestFormatTs(unittest.TestCase): + + def test_formats_unix_timestamp(self): + result = format_ts(1700000000) + self.assertIn("2023", result) + self.assertIn(":", result) + + +class TestFormatWindow(unittest.TestCase): + + def test_same_day_omits_date_on_end(self): + result = format_window(1000000, 1000060) + parts = result.split(" – ") + self.assertEqual(len(parts), 2) + self.assertIn("-", parts[0]) + self.assertNotIn("-", parts[1]) + + def test_different_days_shows_full_dates(self): + result = format_window(1000000, 1000000 + 86400 * 2) + parts = result.split(" – ") + self.assertIn("-", parts[0]) + self.assertIn("-", parts[1]) + + +class TestExtractScenarioParams(unittest.TestCase): + + def test_empty_params(self): + selectors, ns, exclude, cloud = _extract_scenario_params({}) + self.assertEqual(selectors, []) + self.assertEqual(ns, []) + self.assertEqual(exclude, []) + self.assertEqual(cloud, []) + + def test_flat_label_selector(self): + params = {"label_selector": "app=etcd"} + selectors, _, _, _ = _extract_scenario_params(params) + self.assertEqual(selectors, ["app=etcd"]) + + def test_node_selector_hyphenated_key(self): + params = {"node-selector": "node-role.kubernetes.io/worker"} + selectors, _, _, _ = _extract_scenario_params(params) + self.assertEqual(selectors, ["node-role.kubernetes.io/worker"]) + + def test_node_label_selector(self): + params = {"node_label_selector": "node-role.kubernetes.io/worker"} + selectors, _, _, _ = _extract_scenario_params(params) + self.assertEqual(selectors, ["node-role.kubernetes.io/worker"]) + + def test_namespace_pattern_priority(self): + params = {"namespace_pattern": "^openshift-etcd$", "namespace": "openshift-etcd"} + _, ns, _, _ = _extract_scenario_params(params) + self.assertEqual(ns, ["^openshift-etcd$"]) + + def test_namespace_fallback(self): + params = {"namespace": "default"} + _, ns, _, _ = _extract_scenario_params(params) + self.assertEqual(ns, ["default"]) + + def test_service_namespace(self): + params = {"service_namespace": "openshift-console"} + _, ns, _, _ = _extract_scenario_params(params) + self.assertEqual(ns, ["openshift-console"]) + + def test_exclude_label(self): + params = {"exclude_label": "component=downloads"} + _, _, exclude, _ = _extract_scenario_params(params) + self.assertEqual(exclude, ["component=downloads"]) + + def test_empty_exclude_label_ignored(self): + params = {"exclude_label": ""} + _, _, exclude, _ = _extract_scenario_params(params) + self.assertEqual(exclude, []) + + def test_cloud_type(self): + params = {"cloud_type": "aws"} + _, _, _, cloud = _extract_scenario_params(params) + self.assertEqual(cloud, ["aws"]) + + def test_nested_pod_disruption_format(self): + params = [{"id": "kill-pods", "config": { + "namespace_pattern": "^openshift-etcd$", + "label_selector": "k8s-app=etcd", + "exclude_label": "app=etcd-backup", + "node_label_selector": "node-role.kubernetes.io/worker", + }}] + selectors, ns, exclude, _ = _extract_scenario_params(params) + self.assertIn("k8s-app=etcd", selectors) + self.assertIn("node-role.kubernetes.io/worker", selectors) + self.assertEqual(ns, ["^openshift-etcd$"]) + self.assertEqual(exclude, ["app=etcd-backup"]) + + def test_node_scenario_format(self): + params = {"node_scenarios": [{"actions": [ + {"node-selector": "node-role.kubernetes.io/worker", "cloud_type": "aws"} + ]}]} + selectors, _, _, cloud = _extract_scenario_params(params) + self.assertIn("node-role.kubernetes.io/worker", selectors) + self.assertIn("aws", cloud) + + def test_deduplicates_selectors(self): + params = [ + {"config": {"label_selector": "app=etcd"}}, + {"config": {"label_selector": "app=etcd"}}, + ] + selectors, _, _, _ = _extract_scenario_params(params) + self.assertEqual(selectors, ["app=etcd"]) + + def test_non_string_label_selector_ignored(self): + params = {"label_selector": 123} + selectors, _, _, _ = _extract_scenario_params(params) + self.assertEqual(selectors, []) + + def test_none_params(self): + selectors, ns, exclude, cloud = _extract_scenario_params(None) + self.assertEqual(selectors, []) + + +class TestExtractPodName(unittest.TestCase): + + def test_dict_with_namespace(self): + pod = {"namespace": "openshift-etcd", "pod_name": "etcd-0"} + self.assertEqual(_extract_pod_name(pod), "openshift-etcd/etcd-0") + + def test_dict_without_namespace(self): + pod = {"pod_name": "etcd-0"} + self.assertEqual(_extract_pod_name(pod), "etcd-0") + + def test_string_pod(self): + self.assertEqual(_extract_pod_name("my-pod"), "my-pod") + + +class TestExtractVmiName(unittest.TestCase): + + def test_dict_with_namespace(self): + vmi = {"namespace": "kubevirt", "vmi_name": "test-vm"} + self.assertEqual(_extract_vmi_name(vmi), "kubevirt/test-vm") + + def test_string_vmi(self): + self.assertEqual(_extract_vmi_name("my-vmi"), "my-vmi") + + +class TestExtractCriticalAlerts(unittest.TestCase): + + def test_dict_format(self): + raw = {"chaos_alerts": [{"alertname": "a1"}], "post_chaos_alerts": [{"alertname": "a2"}]} + chaos, post = _extract_critical_alerts(raw) + self.assertEqual(len(chaos), 1) + self.assertEqual(len(post), 1) + + def test_list_format(self): + raw = [{"alertname": "a1"}] + chaos, post = _extract_critical_alerts(raw) + self.assertEqual(len(chaos), 1) + self.assertEqual(post, []) + + def test_empty_dict(self): + chaos, post = _extract_critical_alerts({}) + self.assertEqual(chaos, []) + self.assertEqual(post, []) + + def test_none_input(self): + chaos, post = _extract_critical_alerts(None) + self.assertEqual(chaos, []) + self.assertEqual(post, []) + + +class TestBuildChaosReportStructure(unittest.TestCase): + + def test_contains_all_sections(self): + output = _minimal_chaos_output() + report = build_chaos_report(output) + self.assertIn("KRKN RUN SUMMARY", report) + self.assertIn("Run UUID", report) + self.assertIn("TARGETS", report) + self.assertIn("KEY METRICS", report) + self.assertIn("ALERTS & SLOs", report) + self.assertIn("RESILIENCY SCORE", report) + + def test_run_metadata(self): + output = _minimal_chaos_output() + report = build_chaos_report(output) + self.assertIn("test-uuid-1234", report) + self.assertIn("4.22.0", report) + self.assertIn("AWS", report) + self.assertIn("OVNKubernetes", report) + + def test_empty_scenarios(self): + output = _minimal_chaos_output() + report = build_chaos_report(output) + self.assertIn("TARGETS", report) + self.assertIn("KEY METRICS", report) + + def test_missing_telemetry_key(self): + report = build_chaos_report({}) + self.assertIn("KRKN RUN SUMMARY", report) + self.assertIn("N/A", report) + + +class TestBuildChaosReportExitStatusOnly(unittest.TestCase): + """Scenarios like hog_scenarios that only have exit status, no pod/node recovery.""" + + def test_pass(self): + scenario = _make_scenario(scenario_name="cpu-hog.yml", scenario_type="hog_scenarios", exit_status=0) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("PASS (0)", report) + self.assertNotIn("Pods Recovered", report) + self.assertNotIn("Nodes Affected", report) + + def test_fail(self): + scenario = _make_scenario(exit_status=1) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("FAIL (1)", report) + + +class TestBuildChaosReportPodRecovery(unittest.TestCase): + + def _pod_scenario(self, rescheduling=0.5, readiness=3.0, total=3.5): + return _make_scenario( + scenario_name="etcd.yml", + exit_status=0, + parameters=[{"id": "kill-pods", "config": { + "namespace_pattern": "^openshift-etcd$", + "label_selector": "k8s-app=etcd", + }}], + affected_pods={ + "recovered": [{ + "namespace": "openshift-etcd", + "pod_name": "etcd-0", + "total_recovery_time": total, + "pod_rescheduling_time": rescheduling, + "pod_readiness_time": readiness, + }], + "unrecovered": [], + }, + ) + + def test_recovery_times_displayed(self): + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [self._pod_scenario()] + report = build_chaos_report(output) + self.assertIn("Pods Recovered : 1", report) + self.assertIn("Pods Unrecovered : 0", report) + self.assertIn("Total Recovery Time", report) + self.assertIn("Rescheduling Time", report) + self.assertIn("Readiness Time", report) + + def test_pod_listed_in_targets(self): + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [self._pod_scenario()] + report = build_chaos_report(output) + self.assertIn("openshift-etcd/etcd-0", report) + self.assertIn("k8s-app=etcd", report) + self.assertIn("^openshift-etcd$", report) + + def test_none_rescheduling_time_coalesced(self): + scenario = _make_scenario( + affected_pods={ + "recovered": [{ + "total_recovery_time": 5.0, + "pod_rescheduling_time": None, + "pod_readiness_time": None, + }], + "unrecovered": [], + }, + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Rescheduling Time: 0.00s", report) + self.assertIn("Readiness Time : 0.00s", report) + + def test_unrecovered_pods(self): + scenario = _make_scenario( + affected_pods={ + "recovered": [], + "unrecovered": [{"namespace": "ns", "pod_name": "dead-pod"}], + }, + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Pods Recovered : 0", report) + self.assertIn("Pods Unrecovered : 1", report) + self.assertNotIn("Total Recovery Time", report) + + +class TestBuildChaosReportNodeRecovery(unittest.TestCase): + + def test_node_timings(self): + scenario = _make_scenario( + scenario_name="cluster_shut_down.yml", + scenario_type="cluster_shut_down_scenarios", + parameters={"cloud_type": "aws"}, + affected_nodes=[{ + "node_name": "ip-10-0-1-1.compute.internal", + "node_id": "i-abc123", + "stopped_time": 138.5, + "running_time": 17.0, + }], + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Nodes Affected : 1", report) + self.assertIn("ip-10-0-1-1.compute.internal (i-abc123)", report) + self.assertIn("Stopped Time", report) + self.assertIn("138.50s", report) + self.assertIn("Running Time", report) + self.assertIn("Cloud Type : aws", report) + + def test_node_without_id(self): + scenario = _make_scenario( + affected_nodes=[{"node_name": "worker-1", "node_id": "", "stopped_time": 10.0}], + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("worker-1", report) + self.assertNotIn("()", report) + + +class TestBuildChaosReportVMIRecovery(unittest.TestCase): + + def test_vmi_recovery_times(self): + scenario = _make_scenario( + affected_vmis={ + "recovered": [{ + "namespace": "kubevirt", + "vmi_name": "test-vm", + "total_recovery_time": 10.0, + "vmi_rescheduling_time": 2.0, + "vmi_readiness_time": 8.0, + }], + "unrecovered": [], + }, + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("VMIs Recovered : 1", report) + self.assertIn("VMI Recovery Time", report) + self.assertIn("kubevirt/test-vm", report) + + def test_none_vmi_times_coalesced(self): + scenario = _make_scenario( + affected_vmis={ + "recovered": [{ + "total_recovery_time": 5.0, + "vmi_rescheduling_time": None, + "vmi_readiness_time": None, + }], + "unrecovered": [], + }, + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Rescheduling Time: 0.00s", report) + + +class TestBuildChaosReportLoadTestMetrics(unittest.TestCase): + + def test_additional_telemetry_displayed(self): + scenario = _make_scenario( + scenario_name="http_load.yml", + scenario_type="http_load_scenarios", + additional_telemetry={ + "requests_per_sec": 150.5, + "p99_latency_ms": 42.3, + }, + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Load Test Metrics", report) + self.assertIn("requests_per_sec", report) + self.assertIn("150.5", report) + + +class TestBuildChaosReportSLOs(unittest.TestCase): + + def test_slo_counts(self): + output = _minimal_chaos_output() + output["telemetry"]["overall_resiliency_report"] = { + "total_slos": 27, + "passed_slos": 13, + "resiliency_score": 93, + "scenarios": {"etcd.yml": 100}, + } + report = build_chaos_report(output) + self.assertIn("SLOs Evaluated : 27", report) + self.assertIn("SLOs Passed : 13 / 27", report) + self.assertIn("SLOs Failed : 14", report) + + def test_failed_slos_listed(self): + output = _minimal_chaos_output() + output["scenario_slo_details"] = [{ + "scenario": "etcd.yml", + "slo_details": [ + {"name": "etcd latency too high", "severity": "warning", "passed": False}, + {"name": "api latency ok", "severity": "warning", "passed": True}, + ], + }] + report = build_chaos_report(output) + self.assertIn("FAILED SLOs (per scenario)", report) + self.assertIn("etcd latency too high", report) + self.assertNotIn("api latency ok", report) + + def test_no_failed_slos_section_when_all_pass(self): + output = _minimal_chaos_output() + output["scenario_slo_details"] = [{ + "scenario": "etcd.yml", + "slo_details": [ + {"name": "check1", "severity": "warning", "passed": True}, + ], + }] + report = build_chaos_report(output) + self.assertNotIn("FAILED SLOs", report) + + +class TestBuildChaosReportResiliencyScore(unittest.TestCase): + + def test_high_score_checkmark(self): + output = _minimal_chaos_output() + output["telemetry"]["overall_resiliency_report"]["resiliency_score"] = 95 + report = build_chaos_report(output) + self.assertIn("95 / 100", report) + + def test_per_scenario_scores(self): + output = _minimal_chaos_output() + output["telemetry"]["overall_resiliency_report"]["scenarios"] = { + "etcd.yml": 100, + "node.yml": 75, + } + report = build_chaos_report(output) + self.assertIn("etcd.yml", report) + self.assertIn("100 / 100", report) + self.assertIn("node.yml", report) + self.assertIn("75 / 100", report) + + +class TestBuildChaosReportAlerts(unittest.TestCase): + + def test_critical_alerts_displayed(self): + output = _minimal_chaos_output() + output["critical_alerts"] = { + "chaos_alerts": [{"alertname": "EtcdDown", "severity": "critical", + "namespace": "openshift-etcd", "alertstate": "firing"}], + "post_chaos_alerts": [], + } + report = build_chaos_report(output) + self.assertIn("Critical Alerts : 1", report) + self.assertIn("EtcdDown", report) + + def test_no_alerts(self): + output = _minimal_chaos_output() + report = build_chaos_report(output) + self.assertIn("Critical Alerts : None", report) + + +class TestBuildChaosReportErrorLogs(unittest.TestCase): + + def test_error_logs_displayed(self): + output = _minimal_chaos_output() + output["telemetry"]["error_logs"] = [ + {"timestamp": "2026-07-27T10:00:00Z", "message": "Pod creation failed"}, + ] + report = build_chaos_report(output) + self.assertIn("Error Logs : 1", report) + self.assertIn("Pod creation failed", report) + + def test_error_logs_truncated_at_20(self): + output = _minimal_chaos_output() + output["telemetry"]["error_logs"] = [ + {"timestamp": f"ts-{i}", "message": f"error {i}"} for i in range(25) + ] + report = build_chaos_report(output) + self.assertIn("... and 5 more", report) + + def test_string_error_logs(self): + output = _minimal_chaos_output() + output["telemetry"]["error_logs"] = ["raw error string"] + report = build_chaos_report(output) + self.assertIn("raw error string", report) + + +class TestBuildChaosReportSecurityFlags(unittest.TestCase): + + def test_fips_shown(self): + output = _minimal_chaos_output() + output["telemetry"]["fips_enabled"] = True + report = build_chaos_report(output) + self.assertIn("Security : FIPS", report) + + def test_no_security_when_all_false(self): + output = _minimal_chaos_output() + output["telemetry"]["fips_enabled"] = False + report = build_chaos_report(output) + self.assertNotIn("Security", report) + + +class TestBuildChaosReportClusterOverview(unittest.TestCase): + + def test_node_summary_table(self): + output = _minimal_chaos_output() + output["telemetry"]["node_summary_infos"] = [{ + "nodes_type": "worker", + "count": 3, + "instance_type": "m6i.xlarge", + "architecture": "amd64", + "kubelet_version": "v1.35.5", + "os_version": "RHCOS 9.8", + }] + report = build_chaos_report(output) + self.assertIn("CLUSTER OVERVIEW", report) + self.assertIn("worker", report) + self.assertIn("m6i.xlarge", report) + + def test_no_overview_when_empty(self): + output = _minimal_chaos_output() + output["telemetry"]["node_summary_infos"] = [] + report = build_chaos_report(output) + self.assertNotIn("CLUSTER OVERVIEW", report) + + +class TestBuildChaosReportClusterEvents(unittest.TestCase): + + def test_events_shown(self): + scenario = _make_scenario( + cluster_events=[{ + "type": "Warning", + "reason": "BackOff", + "message": "Back-off restarting failed container", + "involved_object_kind": "Pod", + "involved_object_name": "etcd-0", + "namespace": "openshift-etcd", + }], + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Cluster Events", report) + self.assertIn("BackOff", report) + + def test_no_events_section_when_empty(self): + scenario = _make_scenario(cluster_events=[]) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertNotIn("Cluster Events", report) + + +class TestBuildChaosReportMultipleScenarios(unittest.TestCase): + + def test_scenarios_numbered(self): + s1 = _make_scenario(scenario_name="etcd.yml") + s2 = _make_scenario(scenario_name="cpu-hog.yml") + s3 = _make_scenario(scenario_name="node.yml") + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [s1, s2, s3] + report = build_chaos_report(output) + self.assertIn("[1] Scenario", report) + self.assertIn("[2] Scenario", report) + self.assertIn("[3] Scenario", report) + + +class TestBuildChaosReportHealthChecks(unittest.TestCase): + + def test_health_checks_shown(self): + output = _minimal_chaos_output() + output["telemetry"]["health_checks"] = [ + {"url": "https://api.cluster.local:6443", "status_code": 200, + "status": True, "duration": 0.12}, + ] + report = build_chaos_report(output) + self.assertIn("HEALTH CHECKS", report) + self.assertIn("PASS", report) + self.assertIn("api.cluster.local", report) + + def test_no_health_checks_when_null(self): + output = _minimal_chaos_output() + output["telemetry"]["health_checks"] = None + report = build_chaos_report(output) + self.assertNotIn("HEALTH CHECKS", report) + + def test_string_health_check(self): + output = _minimal_chaos_output() + output["telemetry"]["health_checks"] = ["raw check string"] + report = build_chaos_report(output) + self.assertIn("raw check string", report) + + +class TestBuildChaosReportKubevirtChecks(unittest.TestCase): + + def test_virt_checks_shown(self): + output = _minimal_chaos_output() + output["telemetry"]["virt_checks"] = [{ + "vm_name": "test-vm", + "namespace": "kubevirt", + "node_name": "worker-1", + "ip_address": "10.0.0.5", + "status": True, + "duration": 1.23, + }] + report = build_chaos_report(output) + self.assertIn("KUBEVIRT HEALTH CHECKS (pre-chaos)", report) + self.assertIn("kubevirt/test-vm", report) + self.assertIn("10.0.0.5", report) + + def test_post_virt_checks_shown(self): + output = _minimal_chaos_output() + output["telemetry"]["post_virt_checks"] = [{ + "vmi_name": "test-vmi", + "namespace": "kubevirt", + "node_name": "worker-2", + "ip_address": "10.0.0.6", + "new_ip_address": "10.0.0.7", + "status": True, + "duration": 2.5, + }] + report = build_chaos_report(output) + self.assertIn("KUBEVIRT HEALTH CHECKS (post-chaos)", report) + self.assertIn("10.0.0.6", report) + self.assertIn("10.0.0.7", report) + + def test_string_virt_check(self): + output = _minimal_chaos_output() + output["telemetry"]["virt_checks"] = ["raw virt check"] + report = build_chaos_report(output) + self.assertIn("raw virt check", report) + + def test_post_virt_string_check(self): + output = _minimal_chaos_output() + output["telemetry"]["post_virt_checks"] = ["raw post check"] + report = build_chaos_report(output) + self.assertIn("raw post check", report) + + +class TestBuildChaosReportEdgeCases(unittest.TestCase): + + def test_security_flags_etcd_and_ipsec(self): + output = _minimal_chaos_output() + output["telemetry"]["etcd_encryption_enabled"] = True + output["telemetry"]["ipsec_enabled"] = True + report = build_chaos_report(output) + self.assertIn("etcd encryption", report) + self.assertIn("IPSec", report) + + def test_exclude_label_in_targets(self): + scenario = _make_scenario( + parameters=[{"id": "kill", "config": { + "label_selector": "app=etcd", + "exclude_label": "component=backup", + }}], + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Exclude Label : component=backup", report) + + def test_pod_monitoring_error(self): + scenario = _make_scenario( + affected_pods={"recovered": [], "unrecovered": [], "error": "timeout reached"}, + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("Pod Monitoring Error: timeout reached", report) + + def test_vmi_monitoring_error(self): + scenario = _make_scenario( + affected_vmis={"recovered": [], "unrecovered": [], "error": "ssh failed"}, + ) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("VMI Monitoring Error: ssh failed", report) + + def test_string_node_in_affected_nodes(self): + scenario = _make_scenario(affected_nodes=["node-as-string"]) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("node-as-string", report) + + def test_cluster_events_truncated_beyond_10(self): + events = [{"type": "Warning", "reason": f"Reason{i}", + "message": f"msg{i}"} for i in range(15)] + scenario = _make_scenario(cluster_events=events) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("... and 5 more", report) + + def test_string_cluster_event(self): + scenario = _make_scenario(cluster_events=["raw event string"]) + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [scenario] + report = build_chaos_report(output) + self.assertIn("raw event string", report) + + def test_post_chaos_alerts(self): + output = _minimal_chaos_output() + output["critical_alerts"] = { + "chaos_alerts": [{"alertname": "A1", "severity": "critical", + "namespace": "ns1", "alertstate": "firing"}], + "post_chaos_alerts": [{"alertname": "A2", "severity": "warning", + "namespace": "ns2", "alertstate": "pending"}], + } + report = build_chaos_report(output) + self.assertIn("During Chaos:", report) + self.assertIn("Post Chaos:", report) + self.assertIn("A1", report) + self.assertIn("A2", report) + self.assertIn("Critical Alerts : 2", report) + + def test_string_alert(self): + output = _minimal_chaos_output() + output["critical_alerts"] = { + "chaos_alerts": ["raw alert string"], + "post_chaos_alerts": ["raw post alert"], + } + report = build_chaos_report(output) + self.assertIn("raw alert string", report) + self.assertIn("raw post alert", report) + + +class TestBuildChaosReportPdf(unittest.TestCase): + + @patch("weasyprint.HTML") + def test_pdf_generated(self, mock_html_cls): + mock_html_instance = MagicMock() + mock_html_cls.return_value = mock_html_instance + + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [_make_scenario( + parameters=[{"id": "kill", "config": {"label_selector": "app=etcd", + "namespace_pattern": "^ns$"}}], + affected_pods={ + "recovered": [{"namespace": "ns", "pod_name": "p1", + "total_recovery_time": 3.0, + "pod_rescheduling_time": 0.5, + "pod_readiness_time": 2.5}], + "unrecovered": [], + }, + )] + output["telemetry"]["overall_resiliency_report"]["scenarios"] = {"test.yml": 100} + output["scenario_slo_details"] = [{ + "scenario": "test.yml", + "slo_details": [{"name": "slo1", "severity": "warning", "passed": False}], + }] + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + pdf_path = f.name + try: + result = build_chaos_report_pdf(output, pdf_path) + self.assertEqual(result, pdf_path) + mock_html_cls.assert_called_once() + mock_html_instance.write_pdf.assert_called_once_with(pdf_path) + finally: + if os.path.exists(pdf_path): + os.unlink(pdf_path) + + @patch("weasyprint.HTML") + def test_pdf_with_node_recovery(self, mock_html_cls): + mock_html_cls.return_value = MagicMock() + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [_make_scenario( + affected_nodes=[{ + "node_name": "worker-1", "node_id": "i-123", + "stopped_time": 100.0, "running_time": 15.0, + }], + )] + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + pdf_path = f.name + try: + build_chaos_report_pdf(output, pdf_path) + mock_html_cls.assert_called_once() + finally: + if os.path.exists(pdf_path): + os.unlink(pdf_path) + + @patch("weasyprint.HTML") + def test_pdf_with_vmi_recovery(self, mock_html_cls): + mock_html_cls.return_value = MagicMock() + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [_make_scenario( + affected_vmis={ + "recovered": [{"namespace": "kv", "vmi_name": "vm1", + "total_recovery_time": 8.0, + "vmi_rescheduling_time": 1.0, + "vmi_readiness_time": 7.0}], + "unrecovered": [], + }, + )] + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + pdf_path = f.name + try: + build_chaos_report_pdf(output, pdf_path) + mock_html_cls.assert_called_once() + finally: + if os.path.exists(pdf_path): + os.unlink(pdf_path) + + @patch("weasyprint.HTML") + def test_pdf_with_none_recovery_times(self, mock_html_cls): + mock_html_cls.return_value = MagicMock() + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [_make_scenario( + affected_pods={ + "recovered": [{"total_recovery_time": 5.0, + "pod_rescheduling_time": None, + "pod_readiness_time": None}], + "unrecovered": [], + }, + affected_vmis={ + "recovered": [{"total_recovery_time": 3.0, + "vmi_rescheduling_time": None, + "vmi_readiness_time": None}], + "unrecovered": [], + }, + )] + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + pdf_path = f.name + try: + build_chaos_report_pdf(output, pdf_path) + mock_html_cls.assert_called_once() + finally: + if os.path.exists(pdf_path): + os.unlink(pdf_path) + + @patch("weasyprint.HTML") + def test_pdf_with_cluster_events(self, mock_html_cls): + mock_html_cls.return_value = MagicMock() + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [_make_scenario( + cluster_events=[ + {"reason": "Pulled", "message": "image pulled", "type": "Normal"}, + "string event", + ], + )] + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + pdf_path = f.name + try: + build_chaos_report_pdf(output, pdf_path) + mock_html_cls.assert_called_once() + finally: + if os.path.exists(pdf_path): + os.unlink(pdf_path) + + @patch("weasyprint.HTML") + def test_pdf_with_additional_telemetry(self, mock_html_cls): + mock_html_cls.return_value = MagicMock() + output = _minimal_chaos_output() + output["telemetry"]["scenarios"] = [_make_scenario( + additional_telemetry={"rps": 100}, + )] + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + pdf_path = f.name + try: + build_chaos_report_pdf(output, pdf_path) + mock_html_cls.assert_called_once() + finally: + if os.path.exists(pdf_path): + os.unlink(pdf_path) + + @patch("weasyprint.HTML") + def test_pdf_empty_scenarios(self, mock_html_cls): + mock_html_cls.return_value = MagicMock() + output = _minimal_chaos_output() + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + pdf_path = f.name + try: + build_chaos_report_pdf(output, pdf_path) + mock_html_cls.assert_called_once() + finally: + if os.path.exists(pdf_path): + os.unlink(pdf_path) + + +if __name__ == "__main__": + unittest.main()