mirror of
https://github.com/krkn-chaos/krkn.git
synced 2026-08-25 09:27:36 +00:00
improve krkn output to summarize key areas (#1520)
Signed-off-by: Teju Gangisetty <tgangise@redhat.com> Co-authored-by: Paige Patton <64206430+paigerube14@users.noreply.github.com>
This commit is contained in:
Teju Gangisetty
co-authored by
Paige Patton
Paige Patton
parent
fb602e91da
commit
cab32f3e98
+3
-1
@@ -7,6 +7,7 @@ kraken:
|
||||
signal_state: RUN # Will wait for the RUN signal when set to PAUSE before running the scenarios, refer docs/signal.md for more details
|
||||
signal_address: 0.0.0.0 # Signal listening address
|
||||
port: 8081 # Signal port
|
||||
generate_pdf_report: True # Generate a PDF summary report after the run
|
||||
chaos_scenarios:
|
||||
# List of policies/chaos scenarios to load
|
||||
- hog_scenarios:
|
||||
@@ -62,6 +63,7 @@ kraken:
|
||||
- http_load_scenarios:
|
||||
- scenarios/kube/http_load_scenario.yml
|
||||
|
||||
|
||||
resiliency:
|
||||
resiliency_run_mode: standalone # Options: standalone, detailed, disabled
|
||||
resiliency_file: config/alerts.yaml # Path to SLO definitions, will resolve to performance_monitoring: alert_profile: if not specified
|
||||
@@ -143,4 +145,4 @@ kubevirt_checks: # Utilizing virt che
|
||||
ssh_node: "" # If set, will be a backup way to ssh to a node. Will want to set to a node that isn't targeted in chaos
|
||||
node_names: ""
|
||||
exit_on_failure: # If value is True and VMI's are failing post chaos returns failure, values can be True/False
|
||||
|
||||
|
||||
@@ -104,6 +104,18 @@
|
||||
"required": "false",
|
||||
"group": "general"
|
||||
},
|
||||
{
|
||||
"name": "generate-pdf-report",
|
||||
"short_description": "Generate PDF report",
|
||||
"description": "Generate a PDF summary report after the chaos run",
|
||||
"variable": "GENERATE_PDF_REPORT",
|
||||
"type": "enum",
|
||||
"allowed_values": "True,False",
|
||||
"separator": ",",
|
||||
"default": "True",
|
||||
"required": "false",
|
||||
"group": "general"
|
||||
},
|
||||
{
|
||||
"name": "krkn-debug",
|
||||
"short_description": "Krkn debug mode",
|
||||
|
||||
@@ -200,6 +200,21 @@ class Resiliency:
|
||||
raise RuntimeError("finalize_report() must be called first")
|
||||
return self.detailed_report
|
||||
|
||||
def get_scenario_slo_details(self) -> List[Dict[str, Any]]:
|
||||
"""Return per-scenario SLO details with name, severity, and pass/fail."""
|
||||
severity_map = {slo["name"]: slo["severity"] for slo in self._slos}
|
||||
result = []
|
||||
for report in self.scenario_reports:
|
||||
slo_details = [
|
||||
{"name": name, "severity": severity_map.get(name, "unknown"), "passed": passed}
|
||||
for name, passed in report["slo_results"].items()
|
||||
]
|
||||
result.append({
|
||||
"scenario": report["name"],
|
||||
"slo_details": slo_details,
|
||||
})
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def compact_breakdown(report: Dict[str, Any]) -> Dict[str, int]:
|
||||
"""Return a compact summary dict for a single scenario report."""
|
||||
|
||||
@@ -0,0 +1,700 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>KRKN Run Summary</title>
|
||||
<style>
|
||||
@page {
|
||||
size: A4;
|
||||
margin: 1.5cm;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
font-size: 10pt;
|
||||
color: #1a1a1a;
|
||||
line-height: 1.4;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
h1 {
|
||||
text-align: center;
|
||||
font-size: 18pt;
|
||||
margin: 0 0 4px 0;
|
||||
color: #111;
|
||||
}
|
||||
|
||||
.subtitle {
|
||||
text-align: center;
|
||||
font-size: 9pt;
|
||||
color: #666;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
h2 {
|
||||
font-size: 12pt;
|
||||
color: #222;
|
||||
border-bottom: 2px solid #cc0000;
|
||||
padding-bottom: 3px;
|
||||
margin: 18px 0 8px 0;
|
||||
}
|
||||
|
||||
h3 {
|
||||
font-size: 10pt;
|
||||
color: #333;
|
||||
margin: 12px 0 4px 0;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-bottom: 12px;
|
||||
page-break-inside: auto;
|
||||
}
|
||||
|
||||
tr {
|
||||
page-break-inside: avoid;
|
||||
page-break-after: auto;
|
||||
}
|
||||
|
||||
th, td {
|
||||
text-align: left;
|
||||
padding: 5px 8px;
|
||||
border: 1px solid #ccc;
|
||||
font-size: 9pt;
|
||||
}
|
||||
|
||||
th {
|
||||
background-color: #e8e8e8;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
h2, h3 {
|
||||
page-break-after: avoid;
|
||||
}
|
||||
|
||||
.pass { color: #1a7f37; font-weight: bold; }
|
||||
.fail { color: #cf222e; font-weight: bold; }
|
||||
|
||||
.score-green { color: #1a7f37; font-weight: bold; }
|
||||
.score-yellow { color: #9a6700; font-weight: bold; }
|
||||
.score-red { color: #cf222e; font-weight: bold; }
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 3px;
|
||||
font-size: 8pt;
|
||||
font-weight: bold;
|
||||
color: #fff;
|
||||
}
|
||||
.badge-pass { background-color: #1a7f37; }
|
||||
.badge-fail { background-color: #cf222e; }
|
||||
.badge-warn { background-color: #9a6700; }
|
||||
|
||||
.meta-table td:first-child {
|
||||
width: 160px;
|
||||
font-weight: bold;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
.recovery-sub {
|
||||
color: #555;
|
||||
font-size: 8.5pt;
|
||||
}
|
||||
|
||||
.overall-score {
|
||||
text-align: center;
|
||||
font-size: 16pt;
|
||||
font-weight: bold;
|
||||
padding: 10px;
|
||||
margin-top: 8px;
|
||||
border: 2px solid #ccc;
|
||||
border-radius: 6px;
|
||||
page-break-inside: avoid;
|
||||
}
|
||||
|
||||
.meta-table {
|
||||
page-break-inside: avoid;
|
||||
}
|
||||
|
||||
.pod-list {
|
||||
margin: 0;
|
||||
padding-left: 16px;
|
||||
}
|
||||
.pod-list li {
|
||||
font-size: 8.5pt;
|
||||
margin-bottom: 1px;
|
||||
}
|
||||
|
||||
.error-note {
|
||||
color: #cf222e;
|
||||
font-size: 8.5pt;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.event-meta {
|
||||
color: #666;
|
||||
font-size: 8pt;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h1>KRKN Run Summary</h1>
|
||||
<p class="subtitle">Generated {{ generated_at }}</p>
|
||||
|
||||
<!-- Run Metadata -->
|
||||
<h2>Run Metadata</h2>
|
||||
<table class="meta-table">
|
||||
<tr><td>Run UUID</td><td>{{ run_uuid }}</td></tr>
|
||||
<tr><td>Cluster Version</td><td>{{ cluster_version }}</td></tr>
|
||||
<tr><td>Infrastructure</td><td>{{ cloud_infrastructure }}</td></tr>
|
||||
<tr><td>Cloud Type</td><td>{{ cloud_type }}</td></tr>
|
||||
<tr><td>Time Window</td><td>{{ time_window }}</td></tr>
|
||||
<tr><td>Total Nodes</td><td>{{ total_node_count }}</td></tr>
|
||||
{% if network_plugins %}
|
||||
<tr><td>Network Plugins</td><td>{{ network_plugins | join(', ') }}</td></tr>
|
||||
{% endif %}
|
||||
{% if security_flags %}
|
||||
<tr><td>Security</td><td>{{ security_flags | join(', ') }}</td></tr>
|
||||
{% endif %}
|
||||
</table>
|
||||
|
||||
<!-- Cluster Overview -->
|
||||
{% if node_summary_infos %}
|
||||
<h2>Cluster Overview</h2>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Count</th>
|
||||
<th>Instance</th>
|
||||
<th>Architecture</th>
|
||||
<th>Kubelet</th>
|
||||
<th>OS</th>
|
||||
</tr>
|
||||
{% for ni in node_summary_infos %}
|
||||
<tr>
|
||||
<td>{{ ni.nodes_type or 'N/A' }}</td>
|
||||
<td>{{ ni.count or 'N/A' }}</td>
|
||||
<td>{{ ni.instance_type or 'N/A' }}</td>
|
||||
<td>{{ ni.architecture or 'N/A' }}</td>
|
||||
<td>{{ ni.kubelet_version or 'N/A' }}</td>
|
||||
<td>{{ ni.os_version or 'N/A' }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<!-- Targets -->
|
||||
<h2>Targets</h2>
|
||||
{% for s in scenarios %}
|
||||
<table>
|
||||
<tr>
|
||||
{% set doc_url = scenario_type_docs.get(s.scenario_type, '') %}
|
||||
<th>{{ s.scenario }} ({% if doc_url %}<a href="{{ doc_url }}">{{ s.scenario_type }}</a>{% else %}{{ s.scenario_type }}{% endif %})</th>
|
||||
<th style="width: 80px; text-align: right;">
|
||||
{% if s.exit_status == "0" %}
|
||||
<span class="badge badge-pass">PASS</span>
|
||||
{% else %}
|
||||
<span class="badge badge-fail">FAIL</span>
|
||||
{% endif %}
|
||||
</th>
|
||||
</tr>
|
||||
{% if s.selectors %}
|
||||
<tr>
|
||||
<td style="width:160px; font-weight:bold;">Label Selector</td>
|
||||
<td>{{ s.selectors | join(', ') }}</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% if s.namespaces %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">Namespace</td>
|
||||
<td>{{ s.namespaces | join(', ') }}</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% if s.exclude_labels %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">Exclude Label</td>
|
||||
<td>{{ s.exclude_labels | join(', ') }}</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% if s.cloud_types %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">Cloud Type</td>
|
||||
<td>{{ s.cloud_types | join(', ') }}</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% if s.all_pods %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">Disrupted Pods</td>
|
||||
<td>
|
||||
<ul class="pod-list">
|
||||
{% for pod in s.all_pods %}
|
||||
<li>{{ pod }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% if s.pods_error %}
|
||||
<p class="error-note">Monitoring error: {{ s.pods_error }}</p>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% elif s.pods_error %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">Pod Monitoring</td>
|
||||
<td><span class="error-note">Error: {{ s.pods_error }}</span></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% if s.all_vmis %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">Disrupted VMIs</td>
|
||||
<td>
|
||||
<ul class="pod-list">
|
||||
{% for vmi in s.all_vmis %}
|
||||
<li>{{ vmi }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% if s.vmis_error %}
|
||||
<p class="error-note">Monitoring error: {{ s.vmis_error }}</p>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% elif s.vmis_error %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">VMI Monitoring</td>
|
||||
<td><span class="error-note">Error: {{ s.vmis_error }}</span></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% if s.affected_nodes %}
|
||||
<tr>
|
||||
<td style="font-weight:bold;">Affected Nodes</td>
|
||||
<td>
|
||||
<ul class="pod-list">
|
||||
{% for node in s.affected_nodes %}
|
||||
<li>{{ node.node_name }}{% if node.node_id %} ({{ node.node_id }}){% endif %}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
</table>
|
||||
{% endfor %}
|
||||
|
||||
<!-- Key Metrics -->
|
||||
{% set has_pods = scenarios | selectattr('all_pods') | list | length > 0 %}
|
||||
{% if has_pods %}
|
||||
<h2>Key Metrics</h2>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Scenario</th>
|
||||
<th>Pods Recovered</th>
|
||||
<th>Pods Unrecovered</th>
|
||||
<th>Total Recovery Time</th>
|
||||
</tr>
|
||||
{% for s in scenarios %}
|
||||
{% if s.recovered_count or s.unrecovered_count %}
|
||||
<tr>
|
||||
<td>{{ s.scenario }}</td>
|
||||
<td>{{ s.recovered_count }}</td>
|
||||
<td>{{ s.unrecovered_count }}</td>
|
||||
<td>
|
||||
{% if s.total_recovery_time is not none %}
|
||||
{{ "%.2f"|format(s.total_recovery_time) }}s
|
||||
<br><span class="recovery-sub">Rescheduling: {{ "%.2f"|format(s.rescheduling_time) }}s | Readiness: {{ "%.2f"|format(s.readiness_time) }}s</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
{% set has_vmis = scenarios | selectattr('all_vmis') | list | length > 0 %}
|
||||
{% if has_vmis %}
|
||||
<h3>VMI Recovery</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Scenario</th>
|
||||
<th>VMIs Recovered</th>
|
||||
<th>VMIs Unrecovered</th>
|
||||
<th>Total Recovery Time</th>
|
||||
</tr>
|
||||
{% for s in scenarios %}
|
||||
{% if s.all_vmis %}
|
||||
<tr>
|
||||
<td>{{ s.scenario }}</td>
|
||||
<td>{{ s.vmi_recovered_count }}</td>
|
||||
<td>{{ s.vmi_unrecovered_count }}</td>
|
||||
<td>
|
||||
{% if s.vmi_total_recovery_time is not none %}
|
||||
{{ "%.2f"|format(s.vmi_total_recovery_time) }}s
|
||||
<br><span class="recovery-sub">Rescheduling: {{ "%.2f"|format(s.vmi_rescheduling_time) }}s | Readiness: {{ "%.2f"|format(s.vmi_readiness_time) }}s</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
{% set has_nodes = scenarios | selectattr('affected_nodes') | list | length > 0 %}
|
||||
{% if has_nodes %}
|
||||
<h3>Node Recovery</h3>
|
||||
{% for s in scenarios %}
|
||||
{% if s.affected_nodes %}
|
||||
<table>
|
||||
<tr>
|
||||
<th colspan="{% if s.affected_nodes | selectattr('node_id') | list | length > 0 %}7{% else %}6{% endif %}">{{ s.scenario }}</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Node</th>
|
||||
{% if s.affected_nodes | selectattr('node_id') | list | length > 0 %}
|
||||
<th>Instance ID</th>
|
||||
{% endif %}
|
||||
<th>Stopped</th>
|
||||
<th>Running</th>
|
||||
<th>Terminated</th>
|
||||
<th>Not Ready</th>
|
||||
<th>Ready</th>
|
||||
</tr>
|
||||
{% for node in s.affected_nodes %}
|
||||
<tr>
|
||||
<td>{{ node.node_name }}</td>
|
||||
{% if s.affected_nodes | selectattr('node_id') | list | length > 0 %}
|
||||
<td>{{ node.node_id }}</td>
|
||||
{% endif %}
|
||||
<td>{% if node.stopped_time %}{{ "%.2f"|format(node.stopped_time) }}s{% endif %}</td>
|
||||
<td>{% if node.running_time %}{{ "%.2f"|format(node.running_time) }}s{% endif %}</td>
|
||||
<td>{% if node.terminating_time %}{{ "%.2f"|format(node.terminating_time) }}s{% endif %}</td>
|
||||
<td>{% if node.not_ready_time %}{{ "%.2f"|format(node.not_ready_time) }}s{% endif %}</td>
|
||||
<td>{% if node.ready_time %}{{ "%.2f"|format(node.ready_time) }}s{% endif %}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
<!-- Additional Telemetry (HTTP Load Test) -->
|
||||
{% set has_additional = scenarios | selectattr('additional_telemetry') | list | length > 0 %}
|
||||
{% if has_additional %}
|
||||
<h3>Load Test Metrics</h3>
|
||||
{% for s in scenarios %}
|
||||
{% if s.additional_telemetry %}
|
||||
<table>
|
||||
<tr><th colspan="2">{{ s.scenario }}</th></tr>
|
||||
{% for key, val in s.additional_telemetry.items() %}
|
||||
<tr>
|
||||
<td style="width: 200px; font-weight: bold;">{{ key }}</td>
|
||||
<td>{{ val }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
<!-- Cluster Events (per scenario) -->
|
||||
{% set has_events = scenarios | selectattr('cluster_events') | list | length > 0 %}
|
||||
{% if has_events %}
|
||||
<h3>Cluster Events</h3>
|
||||
{% for s in scenarios %}
|
||||
{% if s.cluster_events %}
|
||||
<table>
|
||||
<tr>
|
||||
<th colspan="5">{{ s.scenario }} ({{ s.cluster_events | length }} events)</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Reason</th>
|
||||
<th>Object</th>
|
||||
<th>Message</th>
|
||||
<th>Namespace</th>
|
||||
</tr>
|
||||
{% for event in s.cluster_events[:10] %}
|
||||
<tr>
|
||||
{% if event is mapping %}
|
||||
<td>
|
||||
{% if event.type == 'Warning' %}
|
||||
<span class="badge badge-warn">{{ event.type }}</span>
|
||||
{% elif event.type %}
|
||||
{{ event.type }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ event.reason or '' }}</td>
|
||||
<td>{% if event.involved_object_kind %}{{ event.involved_object_kind }}/{{ event.involved_object_name }}{% endif %}</td>
|
||||
<td>{{ event.message or '' }}</td>
|
||||
<td>{{ event.namespace or '' }}</td>
|
||||
{% else %}
|
||||
<td colspan="5">{{ event }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% if s.cluster_events | length > 10 %}
|
||||
<tr><td colspan="5" style="color: #888; font-size: 8pt;">... and {{ s.cluster_events | length - 10 }} more</td></tr>
|
||||
{% endif %}
|
||||
</table>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
<!-- Health Checks -->
|
||||
{% if health_checks %}
|
||||
<h2>Health Checks</h2>
|
||||
<table>
|
||||
<tr>
|
||||
<th>URL / Endpoint</th>
|
||||
<th>Status Code</th>
|
||||
<th>Duration</th>
|
||||
<th>Result</th>
|
||||
</tr>
|
||||
{% for check in health_checks %}
|
||||
<tr>
|
||||
{% if check is mapping %}
|
||||
<td>{{ check.url or check.name or check.check_name or '' }}</td>
|
||||
<td>{{ check.status_code or '' }}</td>
|
||||
<td>{% if check.duration is not none and check.duration != '' %}{{ "%.2f"|format(check.duration|float) }}s{% endif %}</td>
|
||||
<td>
|
||||
{% if check.status or check.passed %}
|
||||
<span class="badge badge-pass">PASS</span>
|
||||
{% else %}
|
||||
<span class="badge badge-fail">FAIL</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
{% else %}
|
||||
<td colspan="4">{{ check }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<!-- KubeVirt Health Checks (pre-chaos) -->
|
||||
{% if virt_checks %}
|
||||
<h2>KubeVirt Health Checks (Pre-Chaos)</h2>
|
||||
<table>
|
||||
<tr>
|
||||
<th>VM Name</th>
|
||||
<th>Namespace</th>
|
||||
<th>Node</th>
|
||||
<th>IP Address</th>
|
||||
<th>Duration</th>
|
||||
<th>Result</th>
|
||||
</tr>
|
||||
{% for check in virt_checks %}
|
||||
<tr>
|
||||
{% if check is mapping %}
|
||||
<td>{{ check.vm_name or check.vmi_name or check.name or '' }}</td>
|
||||
<td>{{ check.namespace or '' }}</td>
|
||||
<td>{{ check.node_name or '' }}</td>
|
||||
<td>{{ check.ip_address or '' }}</td>
|
||||
<td>{% if check.duration is not none and check.duration != '' %}{{ "%.2f"|format(check.duration|float) }}s{% endif %}</td>
|
||||
<td>
|
||||
{% if check.status is defined and not check.status %}
|
||||
<span class="badge badge-fail">FAIL</span>
|
||||
{% else %}
|
||||
<span class="badge badge-pass">PASS</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
{% else %}
|
||||
<td colspan="6">{{ check }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<!-- KubeVirt Health Checks (post-chaos) -->
|
||||
{% if post_virt_checks %}
|
||||
<h2>KubeVirt Health Checks (Post-Chaos)</h2>
|
||||
<table>
|
||||
<tr>
|
||||
<th>VM Name</th>
|
||||
<th>Namespace</th>
|
||||
<th>Node</th>
|
||||
<th>IP Address</th>
|
||||
<th>New IP</th>
|
||||
<th>Duration</th>
|
||||
<th>Result</th>
|
||||
</tr>
|
||||
{% for check in post_virt_checks %}
|
||||
<tr>
|
||||
{% if check is mapping %}
|
||||
<td>{{ check.vm_name or check.vmi_name or check.name or '' }}</td>
|
||||
<td>{{ check.namespace or '' }}</td>
|
||||
<td>{{ check.node_name or '' }}</td>
|
||||
<td>{{ check.ip_address or '' }}</td>
|
||||
<td>{% if check.new_ip_address and check.new_ip_address != check.ip_address %}{{ check.new_ip_address }}{% endif %}</td>
|
||||
<td>{% if check.duration is not none and check.duration != '' %}{{ "%.2f"|format(check.duration|float) }}s{% endif %}</td>
|
||||
<td>
|
||||
{% if check.status is defined and not check.status %}
|
||||
<span class="badge badge-fail">FAIL</span>
|
||||
{% else %}
|
||||
<span class="badge badge-pass">PASS</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
{% else %}
|
||||
<td colspan="7">{{ check }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<!-- Alerts & SLOs -->
|
||||
<h2>Alerts & SLOs</h2>
|
||||
<table class="meta-table">
|
||||
<tr><td>SLOs Evaluated</td><td>{{ total_slos }}</td></tr>
|
||||
<tr>
|
||||
<td>SLOs Passed</td>
|
||||
<td>{{ passed_slos }} / {{ total_slos }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>SLOs Failed</td>
|
||||
<td>{% if failed_slos > 0 %}<span class="fail">{{ failed_slos }}</span>{% else %}{{ failed_slos }}{% endif %}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Critical Alerts</td>
|
||||
<td>{% if critical_alert_count > 0 %}<span class="fail">{{ critical_alert_count }}</span>{% else %}None{% endif %}</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<!-- Critical Alert Details -->
|
||||
{% if chaos_alerts %}
|
||||
<h3>Critical Alerts (During Chaos)</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Alert Name</th>
|
||||
<th>Severity</th>
|
||||
<th>Namespace</th>
|
||||
<th>State</th>
|
||||
</tr>
|
||||
{% for alert in chaos_alerts %}
|
||||
<tr>
|
||||
{% if alert is mapping %}
|
||||
<td>{{ alert.alertname or 'N/A' }}</td>
|
||||
<td>{{ alert.severity or 'N/A' }}</td>
|
||||
<td>{{ alert.namespace or 'N/A' }}</td>
|
||||
<td>{{ alert.alertstate or 'N/A' }}</td>
|
||||
{% else %}
|
||||
<td colspan="4">{{ alert }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
{% if post_chaos_alerts %}
|
||||
<h3>Critical Alerts (Post Chaos)</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Alert Name</th>
|
||||
<th>Severity</th>
|
||||
<th>Namespace</th>
|
||||
<th>State</th>
|
||||
</tr>
|
||||
{% for alert in post_chaos_alerts %}
|
||||
<tr>
|
||||
{% if alert is mapping %}
|
||||
<td>{{ alert.alertname or 'N/A' }}</td>
|
||||
<td>{{ alert.severity or 'N/A' }}</td>
|
||||
<td>{{ alert.namespace or 'N/A' }}</td>
|
||||
<td>{{ alert.alertstate or 'N/A' }}</td>
|
||||
{% else %}
|
||||
<td colspan="4">{{ alert }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<!-- Error Logs -->
|
||||
{% if error_logs %}
|
||||
<h3>Error Logs ({{ error_logs | length }})</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<th style="width: 160px;">Timestamp</th>
|
||||
<th>Message</th>
|
||||
</tr>
|
||||
{% for log in error_logs[:20] %}
|
||||
<tr>
|
||||
{% if log is mapping %}
|
||||
<td>{{ log.timestamp or '-' }}</td>
|
||||
<td>{{ log.message or log }}</td>
|
||||
{% else %}
|
||||
<td>-</td>
|
||||
<td>{{ log }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% if error_logs | length > 20 %}
|
||||
<tr><td colspan="2" style="color: #888; font-size: 8pt;">... and {{ error_logs | length - 20 }} more</td></tr>
|
||||
{% endif %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<!-- Failed SLOs -->
|
||||
{% set failed_entries = [] %}
|
||||
{% for entry in scenario_slo_details %}
|
||||
{% set failed_slos_list = entry.slo_details | selectattr('passed', 'false') | list %}
|
||||
{% if failed_slos_list %}
|
||||
{% set _ = failed_entries.append({"scenario": entry.scenario, "slo_details": failed_slos_list}) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if failed_entries %}
|
||||
<h2>Failed SLOs</h2>
|
||||
{% for entry in failed_entries %}
|
||||
<h3>{{ entry.scenario }}</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<th>SLO</th>
|
||||
<th>Severity</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
{% for slo in entry.slo_details %}
|
||||
<tr>
|
||||
<td>{{ slo.name }}</td>
|
||||
<td>{{ slo.severity }}</td>
|
||||
<td><span class="badge badge-fail">FAIL</span></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
<!-- Resiliency Score -->
|
||||
<h2>Resiliency Score</h2>
|
||||
{% if per_scenario_scores %}
|
||||
<table>
|
||||
<tr>
|
||||
<th>Scenario</th>
|
||||
<th>Score</th>
|
||||
</tr>
|
||||
{% for name, score in per_scenario_scores.items() %}
|
||||
<tr>
|
||||
<td>{{ name }}</td>
|
||||
<td>
|
||||
{% if score >= 90 %}
|
||||
<span class="score-green">{{ score }} / 100</span>
|
||||
{% elif score >= 70 %}
|
||||
<span class="score-yellow">{{ score }} / 100</span>
|
||||
{% else %}
|
||||
<span class="score-red">{{ score }} / 100</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
<div class="overall-score
|
||||
{%- if overall_score is number and overall_score >= 90 %} score-green
|
||||
{%- elif overall_score is number and overall_score >= 70 %} score-yellow
|
||||
{%- elif overall_score is number %} score-red
|
||||
{%- endif %}">
|
||||
Overall: {{ overall_score }} / 100
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,689 @@
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
SCENARIO_TYPE_DOCS = {
|
||||
"pod_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/pod-disruption/",
|
||||
"container_scenarios": "https://krkn-chaos.dev/docs/scenarios/container-scenarios/",
|
||||
"node_scenarios": "https://krkn-chaos.dev/docs/scenarios/node-scenarios/",
|
||||
"hog_scenarios": "https://krkn-chaos.dev/docs/scenarios/hog-scenarios/",
|
||||
"zone_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/zone-outages/",
|
||||
"application_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/application-outages/",
|
||||
"pod_network_scenarios": "https://krkn-chaos.dev/docs/scenarios/pod-network-scenarios/",
|
||||
"time_scenarios": "https://krkn-chaos.dev/docs/scenarios/time-scenarios/",
|
||||
"cluster_shut_down_scenarios": "https://krkn-chaos.dev/docs/scenarios/cluster-shut-down/",
|
||||
"pvc_scenarios": "https://krkn-chaos.dev/docs/scenarios/pvc-scenarios/",
|
||||
"network_chaos_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos/",
|
||||
"network_chaos_ng_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos/",
|
||||
"service_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-disruption/",
|
||||
"service_hijacking_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-hijacking/",
|
||||
"syn_flood_scenarios": "https://krkn-chaos.dev/docs/scenarios/syn-flood/",
|
||||
"http_load_scenarios": "https://krkn-chaos.dev/docs/scenarios/http-load/",
|
||||
"kubevirt_vm_outage": "https://krkn-chaos.dev/docs/scenarios/kubevirt-vm-outage/",
|
||||
"managedcluster_scenarios": "https://krkn-chaos.dev/docs/scenarios/managed-cluster/",
|
||||
"storage_throttle_scenarios": "https://krkn-chaos.dev/docs/scenarios/storage-throttle/",
|
||||
}
|
||||
|
||||
|
||||
def format_ts(unix_ts):
|
||||
return datetime.fromtimestamp(unix_ts).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def format_window(start_ts, end_ts):
|
||||
start_dt = datetime.fromtimestamp(start_ts)
|
||||
end_dt = datetime.fromtimestamp(end_ts)
|
||||
if start_dt.date() == end_dt.date():
|
||||
return f"{start_dt.strftime('%Y-%m-%d %H:%M:%S')} – {end_dt.strftime('%H:%M:%S')}"
|
||||
return f"{start_dt.strftime('%Y-%m-%d %H:%M:%S')} – {end_dt.strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
|
||||
|
||||
def _extract_scenario_params(raw_params):
|
||||
"""Extract label_selectors, namespaces, and exclude_labels from all parameter shapes.
|
||||
|
||||
Walks the entire parameter tree recursively so it works regardless of
|
||||
how deeply the keys are nested (pod_disruption, container, node, hog,
|
||||
kubevirt, network_chaos, time_scenarios, application_outage, pvc, etc.).
|
||||
"""
|
||||
selectors = []
|
||||
namespaces = []
|
||||
exclude_labels = []
|
||||
cloud_types = []
|
||||
|
||||
def _walk(obj):
|
||||
if isinstance(obj, dict):
|
||||
ls = obj.get("label_selector")
|
||||
if ls and isinstance(ls, str):
|
||||
selectors.append(ls)
|
||||
ns = obj.get("node-selector")
|
||||
if ns and isinstance(ns, str):
|
||||
selectors.append(ns)
|
||||
nls = obj.get("node_label_selector")
|
||||
if nls and isinstance(nls, str):
|
||||
selectors.append(nls)
|
||||
for ns_key in ("namespace_pattern", "namespace", "service_namespace"):
|
||||
nsp = obj.get(ns_key)
|
||||
if nsp and isinstance(nsp, str):
|
||||
namespaces.append(nsp)
|
||||
break
|
||||
el = obj.get("exclude_label")
|
||||
if el and isinstance(el, str):
|
||||
exclude_labels.append(el)
|
||||
ct = obj.get("cloud_type")
|
||||
if ct and isinstance(ct, str):
|
||||
cloud_types.append(ct)
|
||||
for v in obj.values():
|
||||
if isinstance(v, (dict, list)):
|
||||
_walk(v)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
if isinstance(item, (dict, list)):
|
||||
_walk(item)
|
||||
|
||||
_walk(raw_params)
|
||||
|
||||
# deduplicate while preserving order
|
||||
seen = set()
|
||||
selectors = [s for s in selectors if not (s in seen or seen.add(s))]
|
||||
seen = set()
|
||||
namespaces = [n for n in namespaces if not (n in seen or seen.add(n))]
|
||||
seen = set()
|
||||
exclude_labels = [e for e in exclude_labels if not (e in seen or seen.add(e))]
|
||||
seen = set()
|
||||
cloud_types = [c for c in cloud_types if not (c in seen or seen.add(c))]
|
||||
|
||||
return selectors, namespaces, exclude_labels, cloud_types
|
||||
|
||||
|
||||
def _extract_pod_name(pod):
|
||||
if isinstance(pod, dict):
|
||||
ns = pod.get("namespace", "")
|
||||
name = pod.get("pod_name", str(pod))
|
||||
return f"{ns}/{name}" if ns else name
|
||||
return str(pod)
|
||||
|
||||
|
||||
def _extract_vmi_name(vmi):
|
||||
if isinstance(vmi, dict):
|
||||
ns = vmi.get("namespace", "")
|
||||
name = vmi.get("vmi_name", str(vmi))
|
||||
return f"{ns}/{name}" if ns else name
|
||||
return str(vmi)
|
||||
|
||||
|
||||
def _extract_critical_alerts(critical_alerts_raw):
|
||||
if isinstance(critical_alerts_raw, dict):
|
||||
chaos = critical_alerts_raw.get("chaos_alerts", [])
|
||||
post = critical_alerts_raw.get("post_chaos_alerts", [])
|
||||
return chaos, post
|
||||
if isinstance(critical_alerts_raw, list):
|
||||
return critical_alerts_raw, []
|
||||
return [], []
|
||||
|
||||
|
||||
def build_chaos_report(chaos_output: dict) -> str:
|
||||
telemetry = chaos_output.get("telemetry", {})
|
||||
scenarios = telemetry.get("scenarios", [])
|
||||
job_status = telemetry.get("job_status", True)
|
||||
|
||||
lines = []
|
||||
lines.append("=" * 80)
|
||||
lines.append("KRKN RUN SUMMARY")
|
||||
lines.append("=" * 80)
|
||||
|
||||
# --- Run Metadata ---
|
||||
lines.append("Run UUID : " + str(telemetry.get("run_uuid", "N/A")))
|
||||
lines.append("Status : " + ("PASS" if job_status else "FAIL"))
|
||||
lines.append(
|
||||
"Cluster : "
|
||||
+ str(telemetry.get("cluster_version", "N/A"))
|
||||
+ " ("
|
||||
+ str(telemetry.get("cloud_infrastructure", "N/A"))
|
||||
+ ", "
|
||||
+ str(telemetry.get("cloud_type", "N/A"))
|
||||
+ ")"
|
||||
)
|
||||
|
||||
starts = [s["start_timestamp"] for s in scenarios if s.get("start_timestamp")]
|
||||
ends = [s["end_timestamp"] for s in scenarios if s.get("end_timestamp")]
|
||||
if starts and ends:
|
||||
lines.append("Window : " + format_window(min(starts), max(ends)))
|
||||
else:
|
||||
lines.append("Window : N/A")
|
||||
lines.append("Nodes : " + str(telemetry.get("total_node_count", "N/A")))
|
||||
network = telemetry.get("network_plugins") or []
|
||||
if network:
|
||||
lines.append("Network : " + ", ".join(network))
|
||||
security_flags = []
|
||||
if telemetry.get("fips_enabled"):
|
||||
security_flags.append("FIPS")
|
||||
if telemetry.get("etcd_encryption_enabled"):
|
||||
security_flags.append("etcd encryption")
|
||||
if telemetry.get("ipsec_enabled"):
|
||||
security_flags.append("IPSec")
|
||||
if security_flags:
|
||||
lines.append("Security : " + ", ".join(security_flags))
|
||||
|
||||
# --- Cluster Overview ---
|
||||
node_infos = telemetry.get("node_summary_infos") or []
|
||||
if node_infos:
|
||||
lines.append("CLUSTER OVERVIEW")
|
||||
lines.append(f" {'Type':<8} {'Count':<6} {'Instance':<14} {'Arch':<7} {'Kubelet':<10} OS")
|
||||
for ni in node_infos:
|
||||
lines.append(
|
||||
f" {ni.get('nodes_type', 'N/A'):<8} "
|
||||
f"{ni.get('count', 'N/A'):<6} "
|
||||
f"{ni.get('instance_type', 'N/A'):<14} "
|
||||
f"{ni.get('architecture', 'N/A'):<7} "
|
||||
f"{ni.get('kubelet_version', 'N/A'):<10} "
|
||||
f"{ni.get('os_version', 'N/A')}"
|
||||
)
|
||||
|
||||
# --- Targets ---
|
||||
lines.append("TARGETS")
|
||||
for i, s in enumerate(scenarios, 1):
|
||||
lines.append(f" [{i}] Scenario : " + str(s.get("scenario", "N/A")) + " (" + str(s.get("scenario_type", "N/A")) + ")")
|
||||
selectors, ns_list, exclude_labels, cloud_types = _extract_scenario_params(s.get("parameters", {}))
|
||||
if selectors:
|
||||
lines.append(" Label Selector : " + ", ".join(selectors))
|
||||
if ns_list:
|
||||
lines.append(" Namespace : " + ", ".join(ns_list))
|
||||
if exclude_labels:
|
||||
lines.append(" Exclude Label : " + ", ".join(exclude_labels))
|
||||
if cloud_types:
|
||||
lines.append(" Cloud Type : " + ", ".join(cloud_types))
|
||||
|
||||
recovered = s.get("affected_pods", {}).get("recovered", [])
|
||||
unrecovered = s.get("affected_pods", {}).get("unrecovered", [])
|
||||
pods_error = s.get("affected_pods", {}).get("error")
|
||||
if unrecovered or recovered:
|
||||
lines.append(" Pods Disrupted :")
|
||||
for pod in unrecovered + recovered:
|
||||
lines.append(" - " + _extract_pod_name(pod))
|
||||
if pods_error:
|
||||
lines.append(f" Pod Monitoring Error: {pods_error}")
|
||||
|
||||
vmi_recovered = s.get("affected_vmis", {}).get("recovered", [])
|
||||
vmi_unrecovered = s.get("affected_vmis", {}).get("unrecovered", [])
|
||||
vmis_error = s.get("affected_vmis", {}).get("error")
|
||||
if vmi_recovered or vmi_unrecovered:
|
||||
lines.append(" VMIs Disrupted :")
|
||||
for vmi in vmi_unrecovered + vmi_recovered:
|
||||
lines.append(" - " + _extract_vmi_name(vmi))
|
||||
if vmis_error:
|
||||
lines.append(f" VMI Monitoring Error: {vmis_error}")
|
||||
|
||||
affected_nodes = s.get("affected_nodes", [])
|
||||
if affected_nodes:
|
||||
lines.append(" Nodes Affected :")
|
||||
for node in affected_nodes:
|
||||
if isinstance(node, dict):
|
||||
node_id = node.get("node_id", "")
|
||||
label = node.get("node_name", str(node))
|
||||
if node_id:
|
||||
label += f" ({node_id})"
|
||||
lines.append(" - " + label)
|
||||
else:
|
||||
lines.append(" - " + str(node))
|
||||
|
||||
# --- Key Metrics ---
|
||||
lines.append("KEY METRICS")
|
||||
for i, s in enumerate(scenarios, 1):
|
||||
lines.append(f" [{i}] Scenario: " + str(s.get("scenario", "N/A")))
|
||||
|
||||
exit_status = str(s.get("exit_status", "1"))
|
||||
status = "PASS (0)" if exit_status == "0" else "FAIL (1)"
|
||||
lines.append(" Exit Status : " + status)
|
||||
|
||||
recovered = s.get("affected_pods", {}).get("recovered", [])
|
||||
unrecovered = s.get("affected_pods", {}).get("unrecovered", [])
|
||||
if recovered or unrecovered:
|
||||
lines.append(" Pods Recovered : " + str(len(recovered)))
|
||||
lines.append(" Pods Unrecovered : " + str(len(unrecovered)))
|
||||
|
||||
pod_recovery_times = [
|
||||
p.get("total_recovery_time")
|
||||
for p in recovered
|
||||
if isinstance(p, dict) and p.get("total_recovery_time") is not None
|
||||
]
|
||||
if pod_recovery_times:
|
||||
total = max(pod_recovery_times)
|
||||
reschedule_times = [p.get("pod_rescheduling_time") or 0 for p in recovered if isinstance(p, dict)]
|
||||
readiness_times = [p.get("pod_readiness_time") or 0 for p in recovered if isinstance(p, dict)]
|
||||
lines.append(f" Total Recovery Time : {total:.2f}s")
|
||||
lines.append(f" ├─ Rescheduling Time: {max(reschedule_times):.2f}s")
|
||||
lines.append(f" └─ Readiness Time : {max(readiness_times):.2f}s")
|
||||
|
||||
vmi_recovered = s.get("affected_vmis", {}).get("recovered", [])
|
||||
vmi_unrecovered = s.get("affected_vmis", {}).get("unrecovered", [])
|
||||
if vmi_recovered or vmi_unrecovered:
|
||||
lines.append(" VMIs Recovered : " + str(len(vmi_recovered)))
|
||||
lines.append(" VMIs Unrecovered : " + str(len(vmi_unrecovered)))
|
||||
vmi_recovery_times = [
|
||||
v.get("total_recovery_time")
|
||||
for v in vmi_recovered
|
||||
if isinstance(v, dict) and v.get("total_recovery_time") is not None
|
||||
]
|
||||
if vmi_recovery_times:
|
||||
total = max(vmi_recovery_times)
|
||||
reschedule_times = [v.get("vmi_rescheduling_time") or 0 for v in vmi_recovered if isinstance(v, dict)]
|
||||
readiness_times = [v.get("vmi_readiness_time") or 0 for v in vmi_recovered if isinstance(v, dict)]
|
||||
lines.append(f" VMI Recovery Time : {total:.2f}s")
|
||||
lines.append(f" ├─ Rescheduling Time: {max(reschedule_times):.2f}s")
|
||||
lines.append(f" └─ Readiness Time : {max(readiness_times):.2f}s")
|
||||
|
||||
affected_nodes = s.get("affected_nodes", [])
|
||||
if affected_nodes:
|
||||
lines.append(f" Nodes Affected : {len(affected_nodes)}")
|
||||
for node in affected_nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
node_label = node.get("node_name", "N/A")
|
||||
node_id = node.get("node_id", "")
|
||||
if node_id:
|
||||
node_label += f" ({node_id})"
|
||||
lines.append(f" Node: {node_label}")
|
||||
timings = []
|
||||
for key, label in [
|
||||
("stopped_time", "Stopped Time"),
|
||||
("running_time", "Running Time"),
|
||||
("terminating_time", "Terminating Time"),
|
||||
("not_ready_time", "Not Ready Time"),
|
||||
("ready_time", "Ready Time"),
|
||||
]:
|
||||
val = node.get(key)
|
||||
if val is not None and val > 0:
|
||||
timings.append((label, val))
|
||||
for i, (label, val) in enumerate(timings):
|
||||
connector = "└─" if i == len(timings) - 1 else "├─"
|
||||
lines.append(f" {connector} {label:<17}: {val:.2f}s")
|
||||
|
||||
# Additional telemetry (HTTP load test / Vegeta metrics)
|
||||
additional = s.get("additional_telemetry")
|
||||
if additional and isinstance(additional, dict):
|
||||
lines.append(" Load Test Metrics:")
|
||||
for metric_key, metric_val in additional.items():
|
||||
lines.append(f" {metric_key}: {metric_val}")
|
||||
|
||||
cluster_events = s.get("cluster_events") or []
|
||||
if cluster_events:
|
||||
lines.append(f" Cluster Events : {len(cluster_events)}")
|
||||
for event in cluster_events[:10]:
|
||||
if isinstance(event, dict):
|
||||
etype = event.get("type", "")
|
||||
reason = event.get("reason", "N/A")
|
||||
msg = event.get("message", "N/A")
|
||||
obj_kind = event.get("involved_object_kind", "")
|
||||
obj_name = event.get("involved_object_name", "")
|
||||
ns = event.get("namespace", "")
|
||||
prefix = f"[{etype}] " if etype else ""
|
||||
obj_ref = f" ({obj_kind}/{obj_name})" if obj_kind else ""
|
||||
ns_ref = f" in {ns}" if ns else ""
|
||||
lines.append(f" - {prefix}{reason}: {msg}{obj_ref}{ns_ref}")
|
||||
else:
|
||||
lines.append(f" - {event}")
|
||||
if len(cluster_events) > 10:
|
||||
lines.append(f" ... and {len(cluster_events) - 10} more")
|
||||
|
||||
# --- Health Checks ---
|
||||
health_checks = telemetry.get("health_checks")
|
||||
if health_checks:
|
||||
lines.append("HEALTH CHECKS")
|
||||
for check in health_checks:
|
||||
if isinstance(check, dict):
|
||||
url = check.get("url", "")
|
||||
status_code = check.get("status_code", "")
|
||||
duration = check.get("duration")
|
||||
passed = check.get("status") or check.get("passed")
|
||||
status_str = "PASS" if passed else "FAIL"
|
||||
detail = url or check.get("name") or check.get("check_name", "N/A")
|
||||
extra = []
|
||||
if status_code:
|
||||
extra.append(f"HTTP {status_code}")
|
||||
if duration is not None and duration != "":
|
||||
extra.append(f"{float(duration):.2f}s")
|
||||
suffix = f" ({', '.join(extra)})" if extra else ""
|
||||
lines.append(f" {status_str:<6} {detail}{suffix}")
|
||||
else:
|
||||
lines.append(f" {check}")
|
||||
|
||||
# --- KubeVirt Health Checks ---
|
||||
virt_checks = telemetry.get("virt_checks")
|
||||
if virt_checks:
|
||||
lines.append("KUBEVIRT HEALTH CHECKS (pre-chaos)")
|
||||
for check in virt_checks:
|
||||
if isinstance(check, dict):
|
||||
vm = check.get("vm_name") or check.get("vmi_name") or check.get("name", "N/A")
|
||||
ns = check.get("namespace", "")
|
||||
node = check.get("node_name", "")
|
||||
ip = check.get("ip_address", "")
|
||||
passed = check.get("status", True)
|
||||
duration = check.get("duration")
|
||||
status_str = "PASS" if passed else "FAIL"
|
||||
label = f"{ns}/{vm}" if ns else vm
|
||||
extra = []
|
||||
if ip:
|
||||
extra.append(ip)
|
||||
if node:
|
||||
extra.append(f"on {node}")
|
||||
if duration is not None and duration != "":
|
||||
extra.append(f"{float(duration):.2f}s")
|
||||
suffix = f" ({', '.join(extra)})" if extra else ""
|
||||
lines.append(f" {status_str:<6} {label}{suffix}")
|
||||
else:
|
||||
lines.append(f" {check}")
|
||||
|
||||
post_virt_checks = telemetry.get("post_virt_checks")
|
||||
if post_virt_checks:
|
||||
lines.append("KUBEVIRT HEALTH CHECKS (post-chaos)")
|
||||
for check in post_virt_checks:
|
||||
if isinstance(check, dict):
|
||||
vm = check.get("vm_name") or check.get("vmi_name") or check.get("name", "N/A")
|
||||
ns = check.get("namespace", "")
|
||||
node = check.get("node_name", "")
|
||||
ip = check.get("ip_address", "")
|
||||
new_ip = check.get("new_ip_address", "")
|
||||
passed = check.get("status", True)
|
||||
duration = check.get("duration")
|
||||
status_str = "PASS" if passed else "FAIL"
|
||||
label = f"{ns}/{vm}" if ns else vm
|
||||
extra = []
|
||||
if ip:
|
||||
ip_str = ip
|
||||
if new_ip and new_ip != ip:
|
||||
ip_str += f" → {new_ip}"
|
||||
extra.append(ip_str)
|
||||
if node:
|
||||
extra.append(f"on {node}")
|
||||
if duration is not None and duration != "":
|
||||
extra.append(f"{float(duration):.2f}s")
|
||||
suffix = f" ({', '.join(extra)})" if extra else ""
|
||||
lines.append(f" {status_str:<6} {label}{suffix}")
|
||||
else:
|
||||
lines.append(f" {check}")
|
||||
|
||||
# --- Alerts & SLOs ---
|
||||
|
||||
scenario_slo_details = chaos_output.get("scenario_slo_details", [])
|
||||
resiliency = telemetry.get("overall_resiliency_report", {})
|
||||
total_slos = resiliency.get("total_slos", 0)
|
||||
passed_slos = resiliency.get("passed_slos", 0)
|
||||
failed_slos = total_slos - passed_slos
|
||||
critical_alerts_raw = chaos_output.get("critical_alerts") or {}
|
||||
chaos_alerts, post_chaos_alerts = _extract_critical_alerts(critical_alerts_raw)
|
||||
error_logs = telemetry.get("error_logs") or []
|
||||
|
||||
lines.append("ALERTS & SLOs")
|
||||
lines.append(" SLOs Evaluated : " + str(total_slos))
|
||||
lines.append(f" SLOs Passed : {passed_slos} / {total_slos}")
|
||||
lines.append(" SLOs Failed : " + str(failed_slos))
|
||||
|
||||
total_alert_count = len(chaos_alerts) + len(post_chaos_alerts)
|
||||
lines.append(" Critical Alerts : " + (str(total_alert_count) if total_alert_count else "None"))
|
||||
if chaos_alerts:
|
||||
lines.append(" During Chaos:")
|
||||
for alert in chaos_alerts:
|
||||
if isinstance(alert, dict):
|
||||
lines.append(
|
||||
f" - {alert.get('alertname', 'N/A')} "
|
||||
f"[{alert.get('severity', 'N/A')}] "
|
||||
f"ns={alert.get('namespace', 'N/A')} "
|
||||
f"state={alert.get('alertstate', 'N/A')}"
|
||||
)
|
||||
else:
|
||||
lines.append(f" - {alert}")
|
||||
if post_chaos_alerts:
|
||||
lines.append(" Post Chaos:")
|
||||
for alert in post_chaos_alerts:
|
||||
if isinstance(alert, dict):
|
||||
lines.append(
|
||||
f" - {alert.get('alertname', 'N/A')} "
|
||||
f"[{alert.get('severity', 'N/A')}] "
|
||||
f"ns={alert.get('namespace', 'N/A')} "
|
||||
f"state={alert.get('alertstate', 'N/A')}"
|
||||
)
|
||||
else:
|
||||
lines.append(f" - {alert}")
|
||||
|
||||
if error_logs:
|
||||
lines.append(f" Error Logs : {len(error_logs)}")
|
||||
for log_entry in error_logs[:20]:
|
||||
if isinstance(log_entry, dict):
|
||||
ts = log_entry.get("timestamp", "")
|
||||
msg = log_entry.get("message", str(log_entry))
|
||||
lines.append(f" [{ts}] {msg}" if ts else f" {msg}")
|
||||
else:
|
||||
lines.append(f" {log_entry}")
|
||||
if len(error_logs) > 20:
|
||||
lines.append(f" ... and {len(error_logs) - 20} more")
|
||||
|
||||
# --- Failed SLOs ---
|
||||
if scenario_slo_details:
|
||||
has_failures = any(
|
||||
not s["passed"]
|
||||
for entry in scenario_slo_details
|
||||
for s in entry.get("slo_details", [])
|
||||
)
|
||||
if has_failures:
|
||||
lines.append("FAILED SLOs (per scenario)")
|
||||
for entry in scenario_slo_details:
|
||||
failed = [s for s in entry.get("slo_details", []) if not s["passed"]]
|
||||
if not failed:
|
||||
continue
|
||||
lines.append(f" Scenario: {entry['scenario']}")
|
||||
for slo in failed:
|
||||
lines.append(f" FAIL [{slo.get('severity', 'unknown'):<8}] {slo['name']}")
|
||||
|
||||
# --- Resiliency Score ---
|
||||
lines.append("RESILIENCY SCORE")
|
||||
per_scenario_scores = resiliency.get("scenarios", {})
|
||||
for scenario_name, score in per_scenario_scores.items():
|
||||
lines.append(f" {scenario_name:<28} : {score} / 100")
|
||||
|
||||
overall_score = resiliency.get("resiliency_score", "N/A")
|
||||
emoji = ""
|
||||
if isinstance(overall_score, (int, float)):
|
||||
emoji = " ✅" if overall_score >= 90 else (" ⚠️" if overall_score >= 70 else " ❌")
|
||||
lines.append(f" Overall Score : {overall_score} / 100{emoji}")
|
||||
lines.append("=" * 80)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_chaos_report_pdf(chaos_output: dict, output_path: str) -> str:
|
||||
from weasyprint import HTML
|
||||
logging.getLogger("weasyprint").setLevel(logging.WARNING)
|
||||
|
||||
telemetry = chaos_output.get("telemetry", {})
|
||||
scenarios_raw = telemetry.get("scenarios", [])
|
||||
|
||||
starts = [s["start_timestamp"] for s in scenarios_raw if s.get("start_timestamp")]
|
||||
ends = [s["end_timestamp"] for s in scenarios_raw if s.get("end_timestamp")]
|
||||
time_window = format_window(min(starts), max(ends)) if starts and ends else "N/A"
|
||||
|
||||
scenarios = []
|
||||
for s in scenarios_raw:
|
||||
selectors, ns_list, exclude_labels, cloud_types = _extract_scenario_params(s.get("parameters", {}))
|
||||
|
||||
recovered = s.get("affected_pods", {}).get("recovered", [])
|
||||
unrecovered = s.get("affected_pods", {}).get("unrecovered", [])
|
||||
all_pods = [_extract_pod_name(p) for p in unrecovered + recovered]
|
||||
pods_error = s.get("affected_pods", {}).get("error")
|
||||
|
||||
pod_recovery_times = [
|
||||
p.get("total_recovery_time")
|
||||
for p in recovered
|
||||
if isinstance(p, dict) and p.get("total_recovery_time") is not None
|
||||
]
|
||||
if pod_recovery_times:
|
||||
total_recovery = max(pod_recovery_times)
|
||||
reschedule = max(p.get("pod_rescheduling_time") or 0 for p in recovered if isinstance(p, dict))
|
||||
readiness = max(p.get("pod_readiness_time") or 0 for p in recovered if isinstance(p, dict))
|
||||
else:
|
||||
total_recovery = None
|
||||
reschedule = None
|
||||
readiness = None
|
||||
|
||||
vmi_recovered = s.get("affected_vmis", {}).get("recovered", [])
|
||||
vmi_unrecovered = s.get("affected_vmis", {}).get("unrecovered", [])
|
||||
all_vmis = [_extract_vmi_name(v) for v in vmi_unrecovered + vmi_recovered]
|
||||
vmis_error = s.get("affected_vmis", {}).get("error")
|
||||
|
||||
vmi_recovery_times = [
|
||||
v.get("total_recovery_time")
|
||||
for v in vmi_recovered
|
||||
if isinstance(v, dict) and v.get("total_recovery_time") is not None
|
||||
]
|
||||
if vmi_recovery_times:
|
||||
vmi_total_recovery = max(vmi_recovery_times)
|
||||
vmi_reschedule = max(v.get("vmi_rescheduling_time") or 0 for v in vmi_recovered if isinstance(v, dict))
|
||||
vmi_readiness = max(v.get("vmi_readiness_time") or 0 for v in vmi_recovered if isinstance(v, dict))
|
||||
else:
|
||||
vmi_total_recovery = None
|
||||
vmi_reschedule = None
|
||||
vmi_readiness = None
|
||||
|
||||
affected_nodes = s.get("affected_nodes", [])
|
||||
node_list = []
|
||||
for node in affected_nodes:
|
||||
if isinstance(node, dict):
|
||||
node_list.append({
|
||||
"node_name": node.get("node_name", "N/A"),
|
||||
"node_id": node.get("node_id", ""),
|
||||
"stopped_time": node.get("stopped_time"),
|
||||
"running_time": node.get("running_time"),
|
||||
"terminating_time": node.get("terminating_time"),
|
||||
"not_ready_time": node.get("not_ready_time"),
|
||||
"ready_time": node.get("ready_time"),
|
||||
})
|
||||
|
||||
# Cluster events with full detail
|
||||
raw_events = s.get("cluster_events") or []
|
||||
cluster_events = []
|
||||
for event in raw_events:
|
||||
if isinstance(event, dict):
|
||||
cluster_events.append({
|
||||
"reason": event.get("reason", "N/A"),
|
||||
"message": event.get("message", "N/A"),
|
||||
"type": event.get("type", ""),
|
||||
"namespace": event.get("namespace", ""),
|
||||
"source_component": event.get("source_component", ""),
|
||||
"involved_object_kind": event.get("involved_object_kind", ""),
|
||||
"involved_object_name": event.get("involved_object_name", ""),
|
||||
"creation": event.get("creation", ""),
|
||||
})
|
||||
else:
|
||||
cluster_events.append({"reason": "Event", "message": str(event)})
|
||||
|
||||
# Additional telemetry (HTTP load test metrics)
|
||||
additional_telemetry = s.get("additional_telemetry")
|
||||
|
||||
scenarios.append({
|
||||
"scenario": s.get("scenario", "N/A"),
|
||||
"scenario_type": s.get("scenario_type", "N/A"),
|
||||
"selectors": selectors,
|
||||
"namespaces": ns_list,
|
||||
"exclude_labels": exclude_labels,
|
||||
"cloud_types": cloud_types,
|
||||
"all_pods": all_pods,
|
||||
"pods_error": pods_error,
|
||||
"exit_status": str(s.get("exit_status", "1")),
|
||||
"recovered_count": len(recovered),
|
||||
"unrecovered_count": len(unrecovered),
|
||||
"total_recovery_time": total_recovery,
|
||||
"rescheduling_time": reschedule,
|
||||
"readiness_time": readiness,
|
||||
"all_vmis": all_vmis,
|
||||
"vmis_error": vmis_error,
|
||||
"vmi_recovered_count": len(vmi_recovered),
|
||||
"vmi_unrecovered_count": len(vmi_unrecovered),
|
||||
"vmi_total_recovery_time": vmi_total_recovery,
|
||||
"vmi_rescheduling_time": vmi_reschedule,
|
||||
"vmi_readiness_time": vmi_readiness,
|
||||
"affected_nodes": node_list,
|
||||
"cluster_events": cluster_events,
|
||||
"additional_telemetry": additional_telemetry,
|
||||
})
|
||||
|
||||
resiliency = telemetry.get("overall_resiliency_report", {})
|
||||
total_slos = resiliency.get("total_slos", 0)
|
||||
passed_slos = resiliency.get("passed_slos", 0)
|
||||
|
||||
critical_alerts_raw = chaos_output.get("critical_alerts") or {}
|
||||
chaos_alerts, post_chaos_alerts = _extract_critical_alerts(critical_alerts_raw)
|
||||
total_alert_count = len(chaos_alerts) + len(post_chaos_alerts)
|
||||
|
||||
error_logs = telemetry.get("error_logs") or []
|
||||
|
||||
template_dir = Path(__file__).parent / "templates"
|
||||
env = Environment(loader=FileSystemLoader(str(template_dir)), autoescape=True)
|
||||
template = env.get_template("report.html")
|
||||
|
||||
scenario_slo_details = chaos_output.get("scenario_slo_details", [])
|
||||
|
||||
security_flags = []
|
||||
if telemetry.get("fips_enabled"):
|
||||
security_flags.append("FIPS")
|
||||
if telemetry.get("etcd_encryption_enabled"):
|
||||
security_flags.append("etcd encryption")
|
||||
if telemetry.get("ipsec_enabled"):
|
||||
security_flags.append("IPSec")
|
||||
|
||||
node_infos = telemetry.get("node_summary_infos") or []
|
||||
health_checks = telemetry.get("health_checks")
|
||||
virt_checks = telemetry.get("virt_checks")
|
||||
post_virt_checks = telemetry.get("post_virt_checks")
|
||||
|
||||
SCENARIO_TYPE_DOCS = {
|
||||
"hog_scenarios": "https://krkn-chaos.dev/docs/scenarios/hog-scenarios/",
|
||||
"application_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/application-outages/",
|
||||
"container_scenarios": "https://krkn-chaos.dev/docs/scenarios/container-scenarios/",
|
||||
"pod_network_scenarios": "https://krkn-chaos.dev/docs/scenarios/pod-network-scenario/",
|
||||
"pod_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-disruption-scenarios/",
|
||||
"node_scenarios": "https://krkn-chaos.dev/docs/scenarios/node-scenarios/",
|
||||
"time_scenarios": "https://krkn-chaos.dev/docs/scenarios/time-scenarios/",
|
||||
"cluster_shut_down_scenarios": "https://krkn-chaos.dev/docs/scenarios/power-outage-scenarios/",
|
||||
"service_disruption_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-disruption-scenarios/",
|
||||
"zone_outages_scenarios": "https://krkn-chaos.dev/docs/scenarios/zone-outage-scenarios/",
|
||||
"pvc_scenarios": "https://krkn-chaos.dev/docs/scenarios/pvc-scenario/",
|
||||
"storage_throttle_scenarios": "https://krkn-chaos.dev/docs/scenarios/storage-throttle-scenario/",
|
||||
"network_chaos_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos-scenario/",
|
||||
"service_hijacking_scenarios": "https://krkn-chaos.dev/docs/scenarios/service-hijacking-scenario/",
|
||||
"syn_flood_scenarios": "https://krkn-chaos.dev/docs/scenarios/syn-flood-scenario/",
|
||||
"network_chaos_ng_scenarios": "https://krkn-chaos.dev/docs/scenarios/network-chaos-ng-scenarios/",
|
||||
"kubevirt_vm_outage": "https://krkn-chaos.dev/docs/scenarios/kubevirt-vm-outage-scenario/",
|
||||
"http_load_scenarios": "https://krkn-chaos.dev/docs/scenarios/http-load-scenario/"
|
||||
}
|
||||
|
||||
html_content = template.render(
|
||||
generated_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
run_uuid=telemetry.get("run_uuid", "N/A"),
|
||||
cluster_version=telemetry.get("cluster_version", "N/A"),
|
||||
cloud_infrastructure=telemetry.get("cloud_infrastructure", "N/A"),
|
||||
cloud_type=telemetry.get("cloud_type", "N/A"),
|
||||
time_window=time_window,
|
||||
total_node_count=telemetry.get("total_node_count", "N/A"),
|
||||
network_plugins=telemetry.get("network_plugins") or [],
|
||||
security_flags=security_flags,
|
||||
node_summary_infos=node_infos,
|
||||
health_checks=health_checks,
|
||||
virt_checks=virt_checks,
|
||||
post_virt_checks=post_virt_checks,
|
||||
scenarios=scenarios,
|
||||
total_slos=total_slos,
|
||||
passed_slos=passed_slos,
|
||||
failed_slos=total_slos - passed_slos,
|
||||
scenario_slo_details=scenario_slo_details,
|
||||
critical_alert_count=total_alert_count,
|
||||
chaos_alerts=chaos_alerts,
|
||||
post_chaos_alerts=post_chaos_alerts,
|
||||
error_logs=error_logs,
|
||||
per_scenario_scores=resiliency.get("scenarios", {}),
|
||||
overall_score=resiliency.get("resiliency_score", "N/A"),
|
||||
scenario_type_docs=SCENARIO_TYPE_DOCS
|
||||
)
|
||||
|
||||
HTML(string=html_content).write_pdf(output_path)
|
||||
return output_path
|
||||
@@ -16,6 +16,7 @@ ibm-cloud-sdk-core>=3.24.4 # Requires requests>=2.32.4
|
||||
ibm_vpc==0.26.3 # Requires ibm_cloud_sdk_core
|
||||
jinja2==3.1.6
|
||||
jaraco-context>=6.1.0 # Fixes GHSA-58pv-8j8x-9vj2
|
||||
weasyprint>=63.0
|
||||
cbor2<5.7.0 # Pinned by arcaflow-plugin-sdk
|
||||
lxml==6.1.0
|
||||
kubernetes>=35.0.0,<36.0.0
|
||||
|
||||
@@ -72,6 +72,7 @@ from krkn.rollback.command import (
|
||||
list_rollback as list_rollback_command,
|
||||
execute_rollback as execute_rollback_command,
|
||||
)
|
||||
from krkn.summarized_reports.transform import build_chaos_report, build_chaos_report_pdf
|
||||
from krkn.scenario_plugins.triggers.trigger_manager import TriggerManager
|
||||
|
||||
# removes TripleDES warning
|
||||
@@ -104,6 +105,7 @@ def main(options, command: Optional[str]) -> int:
|
||||
config["kraken"], "publish_kraken_status", False
|
||||
)
|
||||
port = get_yaml_item_value(config["kraken"], "port", 8081)
|
||||
generate_pdf_report = get_yaml_item_value(config["kraken"], "generate_pdf_report", True)
|
||||
rollback_versions_dir = get_yaml_item_value(
|
||||
config["kraken"],
|
||||
"rollback_versions_directory",
|
||||
@@ -670,6 +672,26 @@ def main(options, command: Optional[str]) -> int:
|
||||
)
|
||||
chaos_output.telemetry = decoded_chaos_run_telemetry
|
||||
logging.info(f"Chaos data:\n{chaos_output.to_json()}")
|
||||
|
||||
chaos_output_dict = json.loads(chaos_output.to_json())
|
||||
if resiliency_obj and hasattr(resiliency_obj, 'scenario_reports') and resiliency_obj.scenario_reports:
|
||||
chaos_output_dict["scenario_slo_details"] = resiliency_obj.get_scenario_slo_details()
|
||||
try:
|
||||
text_summary = build_chaos_report(chaos_output_dict)
|
||||
logging.info(f"\n{text_summary}")
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to build text summary: {e}")
|
||||
|
||||
if generate_pdf_report:
|
||||
pdf_path = report_file + ".pdf"
|
||||
try:
|
||||
abs_pdf_path = os.path.abspath(pdf_path)
|
||||
build_chaos_report_pdf(chaos_output_dict, abs_pdf_path)
|
||||
logging.info("PDF report generated: %s", abs_pdf_path)
|
||||
print(f"\nfile://{abs_pdf_path}\n")
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to generate PDF report: {e}")
|
||||
|
||||
if enable_elastic:
|
||||
result = elastic_search.push_telemetry(
|
||||
decoded_chaos_run_telemetry, elastic_telemetry_index
|
||||
|
||||
@@ -0,0 +1,962 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Test suite for krkn.summarized_reports.transform module
|
||||
|
||||
Usage:
|
||||
python -m coverage run -a -m unittest tests/test_summarized_reports.py -v
|
||||
|
||||
Assisted By: Claude Code
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from krkn.summarized_reports.transform import (
|
||||
build_chaos_report,
|
||||
build_chaos_report_pdf,
|
||||
format_ts,
|
||||
format_window,
|
||||
_extract_scenario_params,
|
||||
_extract_pod_name,
|
||||
_extract_vmi_name,
|
||||
_extract_critical_alerts,
|
||||
)
|
||||
|
||||
|
||||
def _minimal_chaos_output(**overrides):
|
||||
base = {
|
||||
"telemetry": {
|
||||
"run_uuid": "test-uuid-1234",
|
||||
"cluster_version": "4.22.0",
|
||||
"cloud_infrastructure": "AWS",
|
||||
"cloud_type": "self-managed",
|
||||
"total_node_count": 6,
|
||||
"network_plugins": ["OVNKubernetes"],
|
||||
"scenarios": [],
|
||||
"overall_resiliency_report": {
|
||||
"total_slos": 0,
|
||||
"passed_slos": 0,
|
||||
"resiliency_score": 100,
|
||||
"scenarios": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
def _make_scenario(scenario_name="test.yml", scenario_type="pod_disruption_scenarios",
|
||||
exit_status=0, parameters=None, affected_pods=None,
|
||||
affected_vmis=None, affected_nodes=None,
|
||||
additional_telemetry=None, cluster_events=None,
|
||||
start_timestamp=1000000, end_timestamp=1000060):
|
||||
s = {
|
||||
"scenario": scenario_name,
|
||||
"scenario_type": scenario_type,
|
||||
"exit_status": exit_status,
|
||||
"parameters": parameters or {},
|
||||
"start_timestamp": start_timestamp,
|
||||
"end_timestamp": end_timestamp,
|
||||
}
|
||||
if affected_pods is not None:
|
||||
s["affected_pods"] = affected_pods
|
||||
if affected_vmis is not None:
|
||||
s["affected_vmis"] = affected_vmis
|
||||
if affected_nodes is not None:
|
||||
s["affected_nodes"] = affected_nodes
|
||||
if additional_telemetry is not None:
|
||||
s["additional_telemetry"] = additional_telemetry
|
||||
if cluster_events is not None:
|
||||
s["cluster_events"] = cluster_events
|
||||
return s
|
||||
|
||||
|
||||
class TestFormatTs(unittest.TestCase):
|
||||
|
||||
def test_formats_unix_timestamp(self):
|
||||
result = format_ts(1700000000)
|
||||
self.assertIn("2023", result)
|
||||
self.assertIn(":", result)
|
||||
|
||||
|
||||
class TestFormatWindow(unittest.TestCase):
|
||||
|
||||
def test_same_day_omits_date_on_end(self):
|
||||
result = format_window(1000000, 1000060)
|
||||
parts = result.split(" – ")
|
||||
self.assertEqual(len(parts), 2)
|
||||
self.assertIn("-", parts[0])
|
||||
self.assertNotIn("-", parts[1])
|
||||
|
||||
def test_different_days_shows_full_dates(self):
|
||||
result = format_window(1000000, 1000000 + 86400 * 2)
|
||||
parts = result.split(" – ")
|
||||
self.assertIn("-", parts[0])
|
||||
self.assertIn("-", parts[1])
|
||||
|
||||
|
||||
class TestExtractScenarioParams(unittest.TestCase):
|
||||
|
||||
def test_empty_params(self):
|
||||
selectors, ns, exclude, cloud = _extract_scenario_params({})
|
||||
self.assertEqual(selectors, [])
|
||||
self.assertEqual(ns, [])
|
||||
self.assertEqual(exclude, [])
|
||||
self.assertEqual(cloud, [])
|
||||
|
||||
def test_flat_label_selector(self):
|
||||
params = {"label_selector": "app=etcd"}
|
||||
selectors, _, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(selectors, ["app=etcd"])
|
||||
|
||||
def test_node_selector_hyphenated_key(self):
|
||||
params = {"node-selector": "node-role.kubernetes.io/worker"}
|
||||
selectors, _, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(selectors, ["node-role.kubernetes.io/worker"])
|
||||
|
||||
def test_node_label_selector(self):
|
||||
params = {"node_label_selector": "node-role.kubernetes.io/worker"}
|
||||
selectors, _, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(selectors, ["node-role.kubernetes.io/worker"])
|
||||
|
||||
def test_namespace_pattern_priority(self):
|
||||
params = {"namespace_pattern": "^openshift-etcd$", "namespace": "openshift-etcd"}
|
||||
_, ns, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(ns, ["^openshift-etcd$"])
|
||||
|
||||
def test_namespace_fallback(self):
|
||||
params = {"namespace": "default"}
|
||||
_, ns, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(ns, ["default"])
|
||||
|
||||
def test_service_namespace(self):
|
||||
params = {"service_namespace": "openshift-console"}
|
||||
_, ns, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(ns, ["openshift-console"])
|
||||
|
||||
def test_exclude_label(self):
|
||||
params = {"exclude_label": "component=downloads"}
|
||||
_, _, exclude, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(exclude, ["component=downloads"])
|
||||
|
||||
def test_empty_exclude_label_ignored(self):
|
||||
params = {"exclude_label": ""}
|
||||
_, _, exclude, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(exclude, [])
|
||||
|
||||
def test_cloud_type(self):
|
||||
params = {"cloud_type": "aws"}
|
||||
_, _, _, cloud = _extract_scenario_params(params)
|
||||
self.assertEqual(cloud, ["aws"])
|
||||
|
||||
def test_nested_pod_disruption_format(self):
|
||||
params = [{"id": "kill-pods", "config": {
|
||||
"namespace_pattern": "^openshift-etcd$",
|
||||
"label_selector": "k8s-app=etcd",
|
||||
"exclude_label": "app=etcd-backup",
|
||||
"node_label_selector": "node-role.kubernetes.io/worker",
|
||||
}}]
|
||||
selectors, ns, exclude, _ = _extract_scenario_params(params)
|
||||
self.assertIn("k8s-app=etcd", selectors)
|
||||
self.assertIn("node-role.kubernetes.io/worker", selectors)
|
||||
self.assertEqual(ns, ["^openshift-etcd$"])
|
||||
self.assertEqual(exclude, ["app=etcd-backup"])
|
||||
|
||||
def test_node_scenario_format(self):
|
||||
params = {"node_scenarios": [{"actions": [
|
||||
{"node-selector": "node-role.kubernetes.io/worker", "cloud_type": "aws"}
|
||||
]}]}
|
||||
selectors, _, _, cloud = _extract_scenario_params(params)
|
||||
self.assertIn("node-role.kubernetes.io/worker", selectors)
|
||||
self.assertIn("aws", cloud)
|
||||
|
||||
def test_deduplicates_selectors(self):
|
||||
params = [
|
||||
{"config": {"label_selector": "app=etcd"}},
|
||||
{"config": {"label_selector": "app=etcd"}},
|
||||
]
|
||||
selectors, _, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(selectors, ["app=etcd"])
|
||||
|
||||
def test_non_string_label_selector_ignored(self):
|
||||
params = {"label_selector": 123}
|
||||
selectors, _, _, _ = _extract_scenario_params(params)
|
||||
self.assertEqual(selectors, [])
|
||||
|
||||
def test_none_params(self):
|
||||
selectors, ns, exclude, cloud = _extract_scenario_params(None)
|
||||
self.assertEqual(selectors, [])
|
||||
|
||||
|
||||
class TestExtractPodName(unittest.TestCase):
|
||||
|
||||
def test_dict_with_namespace(self):
|
||||
pod = {"namespace": "openshift-etcd", "pod_name": "etcd-0"}
|
||||
self.assertEqual(_extract_pod_name(pod), "openshift-etcd/etcd-0")
|
||||
|
||||
def test_dict_without_namespace(self):
|
||||
pod = {"pod_name": "etcd-0"}
|
||||
self.assertEqual(_extract_pod_name(pod), "etcd-0")
|
||||
|
||||
def test_string_pod(self):
|
||||
self.assertEqual(_extract_pod_name("my-pod"), "my-pod")
|
||||
|
||||
|
||||
class TestExtractVmiName(unittest.TestCase):
|
||||
|
||||
def test_dict_with_namespace(self):
|
||||
vmi = {"namespace": "kubevirt", "vmi_name": "test-vm"}
|
||||
self.assertEqual(_extract_vmi_name(vmi), "kubevirt/test-vm")
|
||||
|
||||
def test_string_vmi(self):
|
||||
self.assertEqual(_extract_vmi_name("my-vmi"), "my-vmi")
|
||||
|
||||
|
||||
class TestExtractCriticalAlerts(unittest.TestCase):
|
||||
|
||||
def test_dict_format(self):
|
||||
raw = {"chaos_alerts": [{"alertname": "a1"}], "post_chaos_alerts": [{"alertname": "a2"}]}
|
||||
chaos, post = _extract_critical_alerts(raw)
|
||||
self.assertEqual(len(chaos), 1)
|
||||
self.assertEqual(len(post), 1)
|
||||
|
||||
def test_list_format(self):
|
||||
raw = [{"alertname": "a1"}]
|
||||
chaos, post = _extract_critical_alerts(raw)
|
||||
self.assertEqual(len(chaos), 1)
|
||||
self.assertEqual(post, [])
|
||||
|
||||
def test_empty_dict(self):
|
||||
chaos, post = _extract_critical_alerts({})
|
||||
self.assertEqual(chaos, [])
|
||||
self.assertEqual(post, [])
|
||||
|
||||
def test_none_input(self):
|
||||
chaos, post = _extract_critical_alerts(None)
|
||||
self.assertEqual(chaos, [])
|
||||
self.assertEqual(post, [])
|
||||
|
||||
|
||||
class TestBuildChaosReportStructure(unittest.TestCase):
|
||||
|
||||
def test_contains_all_sections(self):
|
||||
output = _minimal_chaos_output()
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("KRKN RUN SUMMARY", report)
|
||||
self.assertIn("Run UUID", report)
|
||||
self.assertIn("TARGETS", report)
|
||||
self.assertIn("KEY METRICS", report)
|
||||
self.assertIn("ALERTS & SLOs", report)
|
||||
self.assertIn("RESILIENCY SCORE", report)
|
||||
|
||||
def test_run_metadata(self):
|
||||
output = _minimal_chaos_output()
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("test-uuid-1234", report)
|
||||
self.assertIn("4.22.0", report)
|
||||
self.assertIn("AWS", report)
|
||||
self.assertIn("OVNKubernetes", report)
|
||||
|
||||
def test_empty_scenarios(self):
|
||||
output = _minimal_chaos_output()
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("TARGETS", report)
|
||||
self.assertIn("KEY METRICS", report)
|
||||
|
||||
def test_missing_telemetry_key(self):
|
||||
report = build_chaos_report({})
|
||||
self.assertIn("KRKN RUN SUMMARY", report)
|
||||
self.assertIn("N/A", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportExitStatusOnly(unittest.TestCase):
|
||||
"""Scenarios like hog_scenarios that only have exit status, no pod/node recovery."""
|
||||
|
||||
def test_pass(self):
|
||||
scenario = _make_scenario(scenario_name="cpu-hog.yml", scenario_type="hog_scenarios", exit_status=0)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("PASS (0)", report)
|
||||
self.assertNotIn("Pods Recovered", report)
|
||||
self.assertNotIn("Nodes Affected", report)
|
||||
|
||||
def test_fail(self):
|
||||
scenario = _make_scenario(exit_status=1)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("FAIL (1)", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportPodRecovery(unittest.TestCase):
|
||||
|
||||
def _pod_scenario(self, rescheduling=0.5, readiness=3.0, total=3.5):
|
||||
return _make_scenario(
|
||||
scenario_name="etcd.yml",
|
||||
exit_status=0,
|
||||
parameters=[{"id": "kill-pods", "config": {
|
||||
"namespace_pattern": "^openshift-etcd$",
|
||||
"label_selector": "k8s-app=etcd",
|
||||
}}],
|
||||
affected_pods={
|
||||
"recovered": [{
|
||||
"namespace": "openshift-etcd",
|
||||
"pod_name": "etcd-0",
|
||||
"total_recovery_time": total,
|
||||
"pod_rescheduling_time": rescheduling,
|
||||
"pod_readiness_time": readiness,
|
||||
}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
)
|
||||
|
||||
def test_recovery_times_displayed(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [self._pod_scenario()]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Pods Recovered : 1", report)
|
||||
self.assertIn("Pods Unrecovered : 0", report)
|
||||
self.assertIn("Total Recovery Time", report)
|
||||
self.assertIn("Rescheduling Time", report)
|
||||
self.assertIn("Readiness Time", report)
|
||||
|
||||
def test_pod_listed_in_targets(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [self._pod_scenario()]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("openshift-etcd/etcd-0", report)
|
||||
self.assertIn("k8s-app=etcd", report)
|
||||
self.assertIn("^openshift-etcd$", report)
|
||||
|
||||
def test_none_rescheduling_time_coalesced(self):
|
||||
scenario = _make_scenario(
|
||||
affected_pods={
|
||||
"recovered": [{
|
||||
"total_recovery_time": 5.0,
|
||||
"pod_rescheduling_time": None,
|
||||
"pod_readiness_time": None,
|
||||
}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Rescheduling Time: 0.00s", report)
|
||||
self.assertIn("Readiness Time : 0.00s", report)
|
||||
|
||||
def test_unrecovered_pods(self):
|
||||
scenario = _make_scenario(
|
||||
affected_pods={
|
||||
"recovered": [],
|
||||
"unrecovered": [{"namespace": "ns", "pod_name": "dead-pod"}],
|
||||
},
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Pods Recovered : 0", report)
|
||||
self.assertIn("Pods Unrecovered : 1", report)
|
||||
self.assertNotIn("Total Recovery Time", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportNodeRecovery(unittest.TestCase):
|
||||
|
||||
def test_node_timings(self):
|
||||
scenario = _make_scenario(
|
||||
scenario_name="cluster_shut_down.yml",
|
||||
scenario_type="cluster_shut_down_scenarios",
|
||||
parameters={"cloud_type": "aws"},
|
||||
affected_nodes=[{
|
||||
"node_name": "ip-10-0-1-1.compute.internal",
|
||||
"node_id": "i-abc123",
|
||||
"stopped_time": 138.5,
|
||||
"running_time": 17.0,
|
||||
}],
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Nodes Affected : 1", report)
|
||||
self.assertIn("ip-10-0-1-1.compute.internal (i-abc123)", report)
|
||||
self.assertIn("Stopped Time", report)
|
||||
self.assertIn("138.50s", report)
|
||||
self.assertIn("Running Time", report)
|
||||
self.assertIn("Cloud Type : aws", report)
|
||||
|
||||
def test_node_without_id(self):
|
||||
scenario = _make_scenario(
|
||||
affected_nodes=[{"node_name": "worker-1", "node_id": "", "stopped_time": 10.0}],
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("worker-1", report)
|
||||
self.assertNotIn("()", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportVMIRecovery(unittest.TestCase):
|
||||
|
||||
def test_vmi_recovery_times(self):
|
||||
scenario = _make_scenario(
|
||||
affected_vmis={
|
||||
"recovered": [{
|
||||
"namespace": "kubevirt",
|
||||
"vmi_name": "test-vm",
|
||||
"total_recovery_time": 10.0,
|
||||
"vmi_rescheduling_time": 2.0,
|
||||
"vmi_readiness_time": 8.0,
|
||||
}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("VMIs Recovered : 1", report)
|
||||
self.assertIn("VMI Recovery Time", report)
|
||||
self.assertIn("kubevirt/test-vm", report)
|
||||
|
||||
def test_none_vmi_times_coalesced(self):
|
||||
scenario = _make_scenario(
|
||||
affected_vmis={
|
||||
"recovered": [{
|
||||
"total_recovery_time": 5.0,
|
||||
"vmi_rescheduling_time": None,
|
||||
"vmi_readiness_time": None,
|
||||
}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Rescheduling Time: 0.00s", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportLoadTestMetrics(unittest.TestCase):
|
||||
|
||||
def test_additional_telemetry_displayed(self):
|
||||
scenario = _make_scenario(
|
||||
scenario_name="http_load.yml",
|
||||
scenario_type="http_load_scenarios",
|
||||
additional_telemetry={
|
||||
"requests_per_sec": 150.5,
|
||||
"p99_latency_ms": 42.3,
|
||||
},
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Load Test Metrics", report)
|
||||
self.assertIn("requests_per_sec", report)
|
||||
self.assertIn("150.5", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportSLOs(unittest.TestCase):
|
||||
|
||||
def test_slo_counts(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["overall_resiliency_report"] = {
|
||||
"total_slos": 27,
|
||||
"passed_slos": 13,
|
||||
"resiliency_score": 93,
|
||||
"scenarios": {"etcd.yml": 100},
|
||||
}
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("SLOs Evaluated : 27", report)
|
||||
self.assertIn("SLOs Passed : 13 / 27", report)
|
||||
self.assertIn("SLOs Failed : 14", report)
|
||||
|
||||
def test_failed_slos_listed(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["scenario_slo_details"] = [{
|
||||
"scenario": "etcd.yml",
|
||||
"slo_details": [
|
||||
{"name": "etcd latency too high", "severity": "warning", "passed": False},
|
||||
{"name": "api latency ok", "severity": "warning", "passed": True},
|
||||
],
|
||||
}]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("FAILED SLOs (per scenario)", report)
|
||||
self.assertIn("etcd latency too high", report)
|
||||
self.assertNotIn("api latency ok", report)
|
||||
|
||||
def test_no_failed_slos_section_when_all_pass(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["scenario_slo_details"] = [{
|
||||
"scenario": "etcd.yml",
|
||||
"slo_details": [
|
||||
{"name": "check1", "severity": "warning", "passed": True},
|
||||
],
|
||||
}]
|
||||
report = build_chaos_report(output)
|
||||
self.assertNotIn("FAILED SLOs", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportResiliencyScore(unittest.TestCase):
|
||||
|
||||
def test_high_score_checkmark(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["overall_resiliency_report"]["resiliency_score"] = 95
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("95 / 100", report)
|
||||
|
||||
def test_per_scenario_scores(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["overall_resiliency_report"]["scenarios"] = {
|
||||
"etcd.yml": 100,
|
||||
"node.yml": 75,
|
||||
}
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("etcd.yml", report)
|
||||
self.assertIn("100 / 100", report)
|
||||
self.assertIn("node.yml", report)
|
||||
self.assertIn("75 / 100", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportAlerts(unittest.TestCase):
|
||||
|
||||
def test_critical_alerts_displayed(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["critical_alerts"] = {
|
||||
"chaos_alerts": [{"alertname": "EtcdDown", "severity": "critical",
|
||||
"namespace": "openshift-etcd", "alertstate": "firing"}],
|
||||
"post_chaos_alerts": [],
|
||||
}
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Critical Alerts : 1", report)
|
||||
self.assertIn("EtcdDown", report)
|
||||
|
||||
def test_no_alerts(self):
|
||||
output = _minimal_chaos_output()
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Critical Alerts : None", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportErrorLogs(unittest.TestCase):
|
||||
|
||||
def test_error_logs_displayed(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["error_logs"] = [
|
||||
{"timestamp": "2026-07-27T10:00:00Z", "message": "Pod creation failed"},
|
||||
]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Error Logs : 1", report)
|
||||
self.assertIn("Pod creation failed", report)
|
||||
|
||||
def test_error_logs_truncated_at_20(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["error_logs"] = [
|
||||
{"timestamp": f"ts-{i}", "message": f"error {i}"} for i in range(25)
|
||||
]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("... and 5 more", report)
|
||||
|
||||
def test_string_error_logs(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["error_logs"] = ["raw error string"]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("raw error string", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportSecurityFlags(unittest.TestCase):
|
||||
|
||||
def test_fips_shown(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["fips_enabled"] = True
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Security : FIPS", report)
|
||||
|
||||
def test_no_security_when_all_false(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["fips_enabled"] = False
|
||||
report = build_chaos_report(output)
|
||||
self.assertNotIn("Security", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportClusterOverview(unittest.TestCase):
|
||||
|
||||
def test_node_summary_table(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["node_summary_infos"] = [{
|
||||
"nodes_type": "worker",
|
||||
"count": 3,
|
||||
"instance_type": "m6i.xlarge",
|
||||
"architecture": "amd64",
|
||||
"kubelet_version": "v1.35.5",
|
||||
"os_version": "RHCOS 9.8",
|
||||
}]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("CLUSTER OVERVIEW", report)
|
||||
self.assertIn("worker", report)
|
||||
self.assertIn("m6i.xlarge", report)
|
||||
|
||||
def test_no_overview_when_empty(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["node_summary_infos"] = []
|
||||
report = build_chaos_report(output)
|
||||
self.assertNotIn("CLUSTER OVERVIEW", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportClusterEvents(unittest.TestCase):
|
||||
|
||||
def test_events_shown(self):
|
||||
scenario = _make_scenario(
|
||||
cluster_events=[{
|
||||
"type": "Warning",
|
||||
"reason": "BackOff",
|
||||
"message": "Back-off restarting failed container",
|
||||
"involved_object_kind": "Pod",
|
||||
"involved_object_name": "etcd-0",
|
||||
"namespace": "openshift-etcd",
|
||||
}],
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Cluster Events", report)
|
||||
self.assertIn("BackOff", report)
|
||||
|
||||
def test_no_events_section_when_empty(self):
|
||||
scenario = _make_scenario(cluster_events=[])
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertNotIn("Cluster Events", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportMultipleScenarios(unittest.TestCase):
|
||||
|
||||
def test_scenarios_numbered(self):
|
||||
s1 = _make_scenario(scenario_name="etcd.yml")
|
||||
s2 = _make_scenario(scenario_name="cpu-hog.yml")
|
||||
s3 = _make_scenario(scenario_name="node.yml")
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [s1, s2, s3]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("[1] Scenario", report)
|
||||
self.assertIn("[2] Scenario", report)
|
||||
self.assertIn("[3] Scenario", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportHealthChecks(unittest.TestCase):
|
||||
|
||||
def test_health_checks_shown(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["health_checks"] = [
|
||||
{"url": "https://api.cluster.local:6443", "status_code": 200,
|
||||
"status": True, "duration": 0.12},
|
||||
]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("HEALTH CHECKS", report)
|
||||
self.assertIn("PASS", report)
|
||||
self.assertIn("api.cluster.local", report)
|
||||
|
||||
def test_no_health_checks_when_null(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["health_checks"] = None
|
||||
report = build_chaos_report(output)
|
||||
self.assertNotIn("HEALTH CHECKS", report)
|
||||
|
||||
def test_string_health_check(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["health_checks"] = ["raw check string"]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("raw check string", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportKubevirtChecks(unittest.TestCase):
|
||||
|
||||
def test_virt_checks_shown(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["virt_checks"] = [{
|
||||
"vm_name": "test-vm",
|
||||
"namespace": "kubevirt",
|
||||
"node_name": "worker-1",
|
||||
"ip_address": "10.0.0.5",
|
||||
"status": True,
|
||||
"duration": 1.23,
|
||||
}]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("KUBEVIRT HEALTH CHECKS (pre-chaos)", report)
|
||||
self.assertIn("kubevirt/test-vm", report)
|
||||
self.assertIn("10.0.0.5", report)
|
||||
|
||||
def test_post_virt_checks_shown(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["post_virt_checks"] = [{
|
||||
"vmi_name": "test-vmi",
|
||||
"namespace": "kubevirt",
|
||||
"node_name": "worker-2",
|
||||
"ip_address": "10.0.0.6",
|
||||
"new_ip_address": "10.0.0.7",
|
||||
"status": True,
|
||||
"duration": 2.5,
|
||||
}]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("KUBEVIRT HEALTH CHECKS (post-chaos)", report)
|
||||
self.assertIn("10.0.0.6", report)
|
||||
self.assertIn("10.0.0.7", report)
|
||||
|
||||
def test_string_virt_check(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["virt_checks"] = ["raw virt check"]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("raw virt check", report)
|
||||
|
||||
def test_post_virt_string_check(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["post_virt_checks"] = ["raw post check"]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("raw post check", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportEdgeCases(unittest.TestCase):
|
||||
|
||||
def test_security_flags_etcd_and_ipsec(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["etcd_encryption_enabled"] = True
|
||||
output["telemetry"]["ipsec_enabled"] = True
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("etcd encryption", report)
|
||||
self.assertIn("IPSec", report)
|
||||
|
||||
def test_exclude_label_in_targets(self):
|
||||
scenario = _make_scenario(
|
||||
parameters=[{"id": "kill", "config": {
|
||||
"label_selector": "app=etcd",
|
||||
"exclude_label": "component=backup",
|
||||
}}],
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Exclude Label : component=backup", report)
|
||||
|
||||
def test_pod_monitoring_error(self):
|
||||
scenario = _make_scenario(
|
||||
affected_pods={"recovered": [], "unrecovered": [], "error": "timeout reached"},
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("Pod Monitoring Error: timeout reached", report)
|
||||
|
||||
def test_vmi_monitoring_error(self):
|
||||
scenario = _make_scenario(
|
||||
affected_vmis={"recovered": [], "unrecovered": [], "error": "ssh failed"},
|
||||
)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("VMI Monitoring Error: ssh failed", report)
|
||||
|
||||
def test_string_node_in_affected_nodes(self):
|
||||
scenario = _make_scenario(affected_nodes=["node-as-string"])
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("node-as-string", report)
|
||||
|
||||
def test_cluster_events_truncated_beyond_10(self):
|
||||
events = [{"type": "Warning", "reason": f"Reason{i}",
|
||||
"message": f"msg{i}"} for i in range(15)]
|
||||
scenario = _make_scenario(cluster_events=events)
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("... and 5 more", report)
|
||||
|
||||
def test_string_cluster_event(self):
|
||||
scenario = _make_scenario(cluster_events=["raw event string"])
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [scenario]
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("raw event string", report)
|
||||
|
||||
def test_post_chaos_alerts(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["critical_alerts"] = {
|
||||
"chaos_alerts": [{"alertname": "A1", "severity": "critical",
|
||||
"namespace": "ns1", "alertstate": "firing"}],
|
||||
"post_chaos_alerts": [{"alertname": "A2", "severity": "warning",
|
||||
"namespace": "ns2", "alertstate": "pending"}],
|
||||
}
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("During Chaos:", report)
|
||||
self.assertIn("Post Chaos:", report)
|
||||
self.assertIn("A1", report)
|
||||
self.assertIn("A2", report)
|
||||
self.assertIn("Critical Alerts : 2", report)
|
||||
|
||||
def test_string_alert(self):
|
||||
output = _minimal_chaos_output()
|
||||
output["critical_alerts"] = {
|
||||
"chaos_alerts": ["raw alert string"],
|
||||
"post_chaos_alerts": ["raw post alert"],
|
||||
}
|
||||
report = build_chaos_report(output)
|
||||
self.assertIn("raw alert string", report)
|
||||
self.assertIn("raw post alert", report)
|
||||
|
||||
|
||||
class TestBuildChaosReportPdf(unittest.TestCase):
|
||||
|
||||
@patch("weasyprint.HTML")
|
||||
def test_pdf_generated(self, mock_html_cls):
|
||||
mock_html_instance = MagicMock()
|
||||
mock_html_cls.return_value = mock_html_instance
|
||||
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [_make_scenario(
|
||||
parameters=[{"id": "kill", "config": {"label_selector": "app=etcd",
|
||||
"namespace_pattern": "^ns$"}}],
|
||||
affected_pods={
|
||||
"recovered": [{"namespace": "ns", "pod_name": "p1",
|
||||
"total_recovery_time": 3.0,
|
||||
"pod_rescheduling_time": 0.5,
|
||||
"pod_readiness_time": 2.5}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
)]
|
||||
output["telemetry"]["overall_resiliency_report"]["scenarios"] = {"test.yml": 100}
|
||||
output["scenario_slo_details"] = [{
|
||||
"scenario": "test.yml",
|
||||
"slo_details": [{"name": "slo1", "severity": "warning", "passed": False}],
|
||||
}]
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
pdf_path = f.name
|
||||
try:
|
||||
result = build_chaos_report_pdf(output, pdf_path)
|
||||
self.assertEqual(result, pdf_path)
|
||||
mock_html_cls.assert_called_once()
|
||||
mock_html_instance.write_pdf.assert_called_once_with(pdf_path)
|
||||
finally:
|
||||
if os.path.exists(pdf_path):
|
||||
os.unlink(pdf_path)
|
||||
|
||||
@patch("weasyprint.HTML")
|
||||
def test_pdf_with_node_recovery(self, mock_html_cls):
|
||||
mock_html_cls.return_value = MagicMock()
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [_make_scenario(
|
||||
affected_nodes=[{
|
||||
"node_name": "worker-1", "node_id": "i-123",
|
||||
"stopped_time": 100.0, "running_time": 15.0,
|
||||
}],
|
||||
)]
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
pdf_path = f.name
|
||||
try:
|
||||
build_chaos_report_pdf(output, pdf_path)
|
||||
mock_html_cls.assert_called_once()
|
||||
finally:
|
||||
if os.path.exists(pdf_path):
|
||||
os.unlink(pdf_path)
|
||||
|
||||
@patch("weasyprint.HTML")
|
||||
def test_pdf_with_vmi_recovery(self, mock_html_cls):
|
||||
mock_html_cls.return_value = MagicMock()
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [_make_scenario(
|
||||
affected_vmis={
|
||||
"recovered": [{"namespace": "kv", "vmi_name": "vm1",
|
||||
"total_recovery_time": 8.0,
|
||||
"vmi_rescheduling_time": 1.0,
|
||||
"vmi_readiness_time": 7.0}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
)]
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
pdf_path = f.name
|
||||
try:
|
||||
build_chaos_report_pdf(output, pdf_path)
|
||||
mock_html_cls.assert_called_once()
|
||||
finally:
|
||||
if os.path.exists(pdf_path):
|
||||
os.unlink(pdf_path)
|
||||
|
||||
@patch("weasyprint.HTML")
|
||||
def test_pdf_with_none_recovery_times(self, mock_html_cls):
|
||||
mock_html_cls.return_value = MagicMock()
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [_make_scenario(
|
||||
affected_pods={
|
||||
"recovered": [{"total_recovery_time": 5.0,
|
||||
"pod_rescheduling_time": None,
|
||||
"pod_readiness_time": None}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
affected_vmis={
|
||||
"recovered": [{"total_recovery_time": 3.0,
|
||||
"vmi_rescheduling_time": None,
|
||||
"vmi_readiness_time": None}],
|
||||
"unrecovered": [],
|
||||
},
|
||||
)]
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
pdf_path = f.name
|
||||
try:
|
||||
build_chaos_report_pdf(output, pdf_path)
|
||||
mock_html_cls.assert_called_once()
|
||||
finally:
|
||||
if os.path.exists(pdf_path):
|
||||
os.unlink(pdf_path)
|
||||
|
||||
@patch("weasyprint.HTML")
|
||||
def test_pdf_with_cluster_events(self, mock_html_cls):
|
||||
mock_html_cls.return_value = MagicMock()
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [_make_scenario(
|
||||
cluster_events=[
|
||||
{"reason": "Pulled", "message": "image pulled", "type": "Normal"},
|
||||
"string event",
|
||||
],
|
||||
)]
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
pdf_path = f.name
|
||||
try:
|
||||
build_chaos_report_pdf(output, pdf_path)
|
||||
mock_html_cls.assert_called_once()
|
||||
finally:
|
||||
if os.path.exists(pdf_path):
|
||||
os.unlink(pdf_path)
|
||||
|
||||
@patch("weasyprint.HTML")
|
||||
def test_pdf_with_additional_telemetry(self, mock_html_cls):
|
||||
mock_html_cls.return_value = MagicMock()
|
||||
output = _minimal_chaos_output()
|
||||
output["telemetry"]["scenarios"] = [_make_scenario(
|
||||
additional_telemetry={"rps": 100},
|
||||
)]
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
pdf_path = f.name
|
||||
try:
|
||||
build_chaos_report_pdf(output, pdf_path)
|
||||
mock_html_cls.assert_called_once()
|
||||
finally:
|
||||
if os.path.exists(pdf_path):
|
||||
os.unlink(pdf_path)
|
||||
|
||||
@patch("weasyprint.HTML")
|
||||
def test_pdf_empty_scenarios(self, mock_html_cls):
|
||||
mock_html_cls.return_value = MagicMock()
|
||||
output = _minimal_chaos_output()
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
pdf_path = f.name
|
||||
try:
|
||||
build_chaos_report_pdf(output, pdf_path)
|
||||
mock_html_cls.assert_called_once()
|
||||
finally:
|
||||
if os.path.exists(pdf_path):
|
||||
os.unlink(pdf_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user