github/krkn
1
0
Fork 0
mirror of https://github.com/krkn-chaos/krkn.git synced 2026-08-25 09:27:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding critical or error alerts to fail run (#1503)

Browse Source 
Signed-off-by: Paige Patton <prubenda@redhat.com>
This commit is contained in:
Paige Patton
2026-07-23 16:57:15 -04:00
committed by GitHub
parent 249b07b250
commit 5973f56d20
3 changed files with 317 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
krkn/prometheus/client.py
+18 -15
View File
@@ -58,6 +58,8 @@ def alerts(
)
sys.exit(1)
# Will fail run if error or critical alerts are firing
failure_alert_count = 0
for alert in profile_yaml:
if sorted(alert.keys()) != sorted(["expr", "description", "severity"]):
logging.error(f"wrong alert {alert}, skipping")
@@ -68,21 +70,22 @@ def alerts(
datetime.datetime.fromtimestamp(start_time),
datetime.datetime.fromtimestamp(end_time),
)
if (
processed_alert[0]
and processed_alert[1]
and elastic
):
elastic_alert = ElasticAlert(
run_uuid=run_uuid,
severity=alert["severity"],
alert=processed_alert[1],
created_at=datetime.datetime.fromtimestamp(processed_alert[0]),
)
result = elastic.push_alert(elastic_alert, elastic_alerts_index)
if result == -1:
logging.error("failed to save alert on ElasticSearch")
pass
if processed_alert[0] and processed_alert[1]:
if alert["severity"] == "critical":
failure_alert_count += 1
if alert["severity"] == "error":
failure_alert_count += 1
if elastic:
elastic_alert = ElasticAlert(
run_uuid=run_uuid,
severity=alert["severity"],
alert=processed_alert[1],
created_at=datetime.datetime.fromtimestamp(processed_alert[0]),
)
result = elastic.push_alert(elastic_alert, elastic_alerts_index)
if result == -1:
logging.error("failed to save alert on ElasticSearch")
return failure_alert_count
def critical_alerts(
run_kraken.py
+36 -20
View File
@@ -363,6 +363,7 @@ def main(options, command: Optional[str]) -> int:
# Capture the start time
start_time = int(time.time())
post_critical_alerts = 0
profile_critical_alerts = 0
chaos_output = ChaosRunOutput()
chaos_telemetry = ChaosRunTelemetry()
chaos_telemetry.run_uuid = run_uuid
@@ -594,6 +595,26 @@ def main(options, command: Optional[str]) -> int:
logging.error("Failed to finalize resiliency scoring: %s", e)
# Check for the alerts specified before telemetry so job_status is included in output
if enable_alerts:
logging.info("Alerts checking is enabled")
if alert_profile:
profile_critical_alerts = prometheus_plugin.alerts(
prometheus,
elastic_search,
run_uuid,
start_time,
end_time,
alert_profile,
elastic_alerts_index
)
else:
logging.error("Alert profile is not defined")
return -1
if post_critical_alerts > 0 or profile_critical_alerts > 0:
chaos_telemetry.job_status = False
telemetry_json = chaos_telemetry.to_json()
decoded_chaos_run_telemetry = ChaosRunTelemetry(json.loads(telemetry_json))
if resiliency_obj and hasattr(resiliency_obj, "summary") and resiliency_obj.summary is not None:
@@ -680,24 +701,6 @@ def main(options, command: Optional[str]) -> int:
else:
logging.info("api_url not set, skipping telemetry upload.")
# Check for the alerts specified
if enable_alerts:
logging.info("Alerts checking is enabled")
if alert_profile:
prometheus_plugin.alerts(
prometheus,
elastic_search,
run_uuid,
start_time,
end_time,
alert_profile,
elastic_alerts_index
)
else:
logging.error("Alert profile is not defined")
return -1
# sys.exit(1)
if enable_metrics:
logging.info(f'Capturing metrics using file {metrics_profile}')
prometheus_plugin.metrics(
@@ -711,6 +714,11 @@ def main(options, command: Optional[str]) -> int:
telemetry_json
)
logging.info(
"Kraken UUID for the run: "
"%s. Report generated at %s." % (run_uuid, report_file)
)
# Exit code priority (lowest wins, checked first):
# 1 = post-scenario failure
# 2 = critical Prometheus alerts
@@ -733,10 +741,18 @@ def main(options, command: Optional[str]) -> int:
logging.error("Critical alerts are firing, please check; exiting")
return 2
if profile_critical_alerts > 0:
logging.error("Critical or Error alerts from alert profile are firing, please check; exiting")
return 2
if not chaos_telemetry.job_status:
logging.error("job_status is false, please check; exiting")
return 1
logging.info(
"Successfully finished running Kraken. UUID for the run: "
"%s. Report generated at %s. Exiting" % (run_uuid, report_file)
"Successfully finished running Kraken, exiting"
)
else:
logging.error("Cannot find a config at %s, please check" % (cfg))
# sys.exit(1)
tests/test_prometheus_client.py
+263 -1
View File
@@ -53,7 +53,7 @@ class TestAlertsKeyValidation(unittest.TestCase):
)
self.elastic.push_alert.return_value = 0
client.alerts(
result = client.alerts(
self.prom_cli,
self.elastic,
self.run_uuid,
@@ -64,6 +64,7 @@ class TestAlertsKeyValidation(unittest.TestCase):
)
self.prom_cli.process_alert.assert_called_once()
self.assertEqual(result, 1)
finally:
os.unlink(profile_path)
@@ -138,6 +139,267 @@ class TestAlertsKeyValidation(unittest.TestCase):
os.unlink(profile_path)
class TestAlertsFailureCount(unittest.TestCase):
"""Tests that alerts() returns the correct failure count for critical/error severity."""
def setUp(self):
self.prom_cli = MagicMock()
self.elastic = MagicMock()
self.run_uuid = "test-uuid"
self.start_time = 1000000.0
self.end_time = 1000060.0
self.elastic_alerts_index = "test-index"
def _write_alert_profile(self, content):
f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False)
f.write(content)
f.close()
return f.name
def _call_alerts(self, profile_path):
return client.alerts(
self.prom_cli,
self.elastic,
self.run_uuid,
self.start_time,
self.end_time,
profile_path,
self.elastic_alerts_index,
)
def test_returns_zero_when_no_alerts_fire(self):
"""Returns 0 when process_alert returns (None, None) for all alerts."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "target down"\n'
' severity: "critical"\n'
)
try:
self.prom_cli.process_alert.return_value = (None, None)
result = self._call_alerts(profile_path)
self.assertEqual(result, 0)
finally:
os.unlink(profile_path)
def test_returns_one_for_single_critical_alert(self):
"""Returns 1 when one critical-severity alert fires."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "target down"\n'
' severity: "critical"\n'
)
try:
self.prom_cli.process_alert.return_value = (self.start_time, "target down")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 1)
finally:
os.unlink(profile_path)
def test_returns_one_for_single_error_alert(self):
"""Returns 1 when one error-severity alert fires."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "target down"\n'
' severity: "error"\n'
)
try:
self.prom_cli.process_alert.return_value = (self.start_time, "target down")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 1)
finally:
os.unlink(profile_path)
def test_warning_alert_does_not_increment_count(self):
"""Returns 0 when only warning/info/debug alerts fire."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "slow"\n'
' severity: "warning"\n'
'- expr: "up == 1"\n'
' description: "info"\n'
' severity: "info"\n'
)
try:
self.prom_cli.process_alert.return_value = (self.start_time, "alert fired")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 0)
finally:
os.unlink(profile_path)
def test_counts_multiple_critical_and_error_alerts(self):
"""Returns correct total when multiple critical and error alerts fire."""
profile_path = self._write_alert_profile(
'- expr: "a == 0"\n'
' description: "a"\n'
' severity: "critical"\n'
'- expr: "b == 0"\n'
' description: "b"\n'
' severity: "error"\n'
'- expr: "c == 0"\n'
' description: "c"\n'
' severity: "warning"\n'
'- expr: "d == 0"\n'
' description: "d"\n'
' severity: "critical"\n'
)
try:
self.prom_cli.process_alert.return_value = (self.start_time, "fired")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 3)
finally:
os.unlink(profile_path)
def test_non_firing_critical_alerts_not_counted(self):
"""Critical alerts that don't fire (return None) are not counted."""
profile_path = self._write_alert_profile(
'- expr: "a == 0"\n'
' description: "fired"\n'
' severity: "critical"\n'
'- expr: "b == 0"\n'
' description: "not fired"\n'
' severity: "critical"\n'
)
try:
self.prom_cli.process_alert.side_effect = [
(self.start_time, "fired"),
(None, None),
]
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 1)
finally:
os.unlink(profile_path)
def test_elastic_push_still_called_for_non_critical_firing_alerts(self):
"""Elastic push is called for warning/info alerts that fire even though they don't fail the run."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "slow"\n'
' severity: "warning"\n'
)
try:
self.prom_cli.process_alert.return_value = (self.start_time, "slow")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 0)
self.elastic.push_alert.assert_called_once()
finally:
os.unlink(profile_path)
class TestJobStatusComputation(unittest.TestCase):
"""
Tests the job_status logic from run_kraken.py:
if post_critical_alerts > 0 or profile_critical_alerts > 0:
chaos_telemetry.job_status = False
chaos_output.job_status = (
chaos_telemetry.job_status
and post_critical_alerts == 0
and profile_critical_alerts == 0
)
Mirrors the logic directly so that regressions in the computation are caught.
"""
def _compute(self, telemetry_job_status, post_critical_alerts, profile_critical_alerts):
if post_critical_alerts > 0 or profile_critical_alerts > 0:
telemetry_job_status = False
return (
telemetry_job_status
and post_critical_alerts == 0
and profile_critical_alerts == 0
)
def test_true_when_no_failures(self):
self.assertTrue(self._compute(True, 0, 0))
def test_false_on_post_critical_alerts(self):
self.assertFalse(self._compute(True, 1, 0))
def test_false_on_profile_critical_alerts(self):
self.assertFalse(self._compute(True, 0, 1))
def test_false_on_scenario_failure(self):
"""chaos_telemetry.job_status starts False when a scenario failed (set by krkn-lib)."""
self.assertFalse(self._compute(False, 0, 0))
def test_false_when_all_fail(self):
self.assertFalse(self._compute(False, 2, 3))
def test_profile_alerts_override_passing_telemetry(self):
"""Even if no scenarios failed, profile critical alerts must flip job_status False."""
self.assertFalse(self._compute(True, 0, 1))
def test_post_critical_alerts_override_passing_telemetry(self):
"""Even if no scenarios failed, post-chaos critical alerts must flip job_status False."""
self.assertFalse(self._compute(True, 1, 0))
def test_telemetry_job_status_mutated_by_alerts(self):
"""Verifies that chaos_telemetry.job_status is set to False when alerts fire,
not just chaos_output.job_status — both fields in the output must be false."""
telemetry_job_status = True
post_critical_alerts = 0
profile_critical_alerts = 1
if post_critical_alerts > 0 or profile_critical_alerts > 0:
telemetry_job_status = False
self.assertFalse(telemetry_job_status)
self.assertFalse(
telemetry_job_status
and post_critical_alerts == 0
and profile_critical_alerts == 0
)
# --- exit guard: not chaos_output.job_status ---
def test_exit_guard_triggers_when_job_status_false(self):
"""not chaos_output.job_status is True when job_status is False — guard fires."""
job_status = self._compute(False, 0, 0)
self.assertTrue(not job_status)
def test_exit_guard_does_not_trigger_when_job_status_true(self):
"""not chaos_output.job_status is False when everything passed — guard skipped."""
job_status = self._compute(True, 0, 0)
self.assertFalse(not job_status)
def test_exit_guard_catches_scenario_failure_not_in_failed_post_scenarios(self):
"""The guard catches chaos_telemetry.job_status=False (set by krkn-lib for a scenario
failure) even when post_critical_alerts==0, profile_critical_alerts==0, and
failed_post_scenarios is empty — i.e. when prior specific checks all pass."""
post_critical_alerts = 0
profile_critical_alerts = 0
failed_post_scenarios = []
# Prior checks do not fire
self.assertFalse(bool(failed_post_scenarios))
self.assertEqual(post_critical_alerts, 0)
self.assertEqual(profile_critical_alerts, 0)
# krkn-lib set telemetry job_status=False due to a scenario exit_status > 0
job_status = self._compute(
telemetry_job_status=False,
post_critical_alerts=post_critical_alerts,
profile_critical_alerts=profile_critical_alerts,
)
# The catch-all guard fires
self.assertTrue(not job_status)
def test_exit_guard_does_not_double_trigger_on_alert_failure(self):
"""When alerts already caused an earlier return (post/profile > 0), job_status
is also False — the guard would fire too, but order ensures alerts return first."""
job_status = self._compute(True, 1, 0)
self.assertFalse(job_status)
# guard would trigger, but post_critical_alerts > 0 returns 2 before reaching it
self.assertTrue(not job_status)
class TestMetricsQueryRouting(unittest.TestCase):
"""Tests for metric query routing in the metrics() function."""