From 5973f56d20fb148c45b35ea75c757deb0146af11 Mon Sep 17 00:00:00 2001 From: Paige Patton <64206430+paigerube14@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:57:15 -0400 Subject: [PATCH] adding critical or error alerts to fail run (#1503) Signed-off-by: Paige Patton --- krkn/prometheus/client.py | 33 ++-- run_kraken.py | 56 ++++--- tests/test_prometheus_client.py | 264 +++++++++++++++++++++++++++++++- 3 files changed, 317 insertions(+), 36 deletions(-) diff --git a/krkn/prometheus/client.py b/krkn/prometheus/client.py index 55f5dfac..73a3104d 100644 --- a/krkn/prometheus/client.py +++ b/krkn/prometheus/client.py @@ -58,6 +58,8 @@ def alerts( ) sys.exit(1) + # Will fail run if error or critical alerts are firing + failure_alert_count = 0 for alert in profile_yaml: if sorted(alert.keys()) != sorted(["expr", "description", "severity"]): logging.error(f"wrong alert {alert}, skipping") @@ -68,21 +70,22 @@ def alerts( datetime.datetime.fromtimestamp(start_time), datetime.datetime.fromtimestamp(end_time), ) - if ( - processed_alert[0] - and processed_alert[1] - and elastic - ): - elastic_alert = ElasticAlert( - run_uuid=run_uuid, - severity=alert["severity"], - alert=processed_alert[1], - created_at=datetime.datetime.fromtimestamp(processed_alert[0]), - ) - result = elastic.push_alert(elastic_alert, elastic_alerts_index) - if result == -1: - logging.error("failed to save alert on ElasticSearch") - pass + if processed_alert[0] and processed_alert[1]: + if alert["severity"] == "critical": + failure_alert_count += 1 + if alert["severity"] == "error": + failure_alert_count += 1 + if elastic: + elastic_alert = ElasticAlert( + run_uuid=run_uuid, + severity=alert["severity"], + alert=processed_alert[1], + created_at=datetime.datetime.fromtimestamp(processed_alert[0]), + ) + result = elastic.push_alert(elastic_alert, elastic_alerts_index) + if result == -1: + logging.error("failed to save alert on ElasticSearch") + return failure_alert_count def critical_alerts( diff --git a/run_kraken.py b/run_kraken.py index 015dd666..84c8f22a 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -363,6 +363,7 @@ def main(options, command: Optional[str]) -> int: # Capture the start time start_time = int(time.time()) post_critical_alerts = 0 + profile_critical_alerts = 0 chaos_output = ChaosRunOutput() chaos_telemetry = ChaosRunTelemetry() chaos_telemetry.run_uuid = run_uuid @@ -594,6 +595,26 @@ def main(options, command: Optional[str]) -> int: logging.error("Failed to finalize resiliency scoring: %s", e) + # Check for the alerts specified before telemetry so job_status is included in output + if enable_alerts: + logging.info("Alerts checking is enabled") + if alert_profile: + profile_critical_alerts = prometheus_plugin.alerts( + prometheus, + elastic_search, + run_uuid, + start_time, + end_time, + alert_profile, + elastic_alerts_index + ) + else: + logging.error("Alert profile is not defined") + return -1 + + if post_critical_alerts > 0 or profile_critical_alerts > 0: + chaos_telemetry.job_status = False + telemetry_json = chaos_telemetry.to_json() decoded_chaos_run_telemetry = ChaosRunTelemetry(json.loads(telemetry_json)) if resiliency_obj and hasattr(resiliency_obj, "summary") and resiliency_obj.summary is not None: @@ -680,24 +701,6 @@ def main(options, command: Optional[str]) -> int: else: logging.info("api_url not set, skipping telemetry upload.") - # Check for the alerts specified - if enable_alerts: - logging.info("Alerts checking is enabled") - if alert_profile: - prometheus_plugin.alerts( - prometheus, - elastic_search, - run_uuid, - start_time, - end_time, - alert_profile, - elastic_alerts_index - ) - - else: - logging.error("Alert profile is not defined") - return -1 - # sys.exit(1) if enable_metrics: logging.info(f'Capturing metrics using file {metrics_profile}') prometheus_plugin.metrics( @@ -711,6 +714,11 @@ def main(options, command: Optional[str]) -> int: telemetry_json ) + logging.info( + "Kraken UUID for the run: " + "%s. Report generated at %s." % (run_uuid, report_file) + ) + # Exit code priority (lowest wins, checked first): # 1 = post-scenario failure # 2 = critical Prometheus alerts @@ -733,10 +741,18 @@ def main(options, command: Optional[str]) -> int: logging.error("Critical alerts are firing, please check; exiting") return 2 + if profile_critical_alerts > 0: + logging.error("Critical or Error alerts from alert profile are firing, please check; exiting") + return 2 + + if not chaos_telemetry.job_status: + logging.error("job_status is false, please check; exiting") + return 1 + logging.info( - "Successfully finished running Kraken. UUID for the run: " - "%s. Report generated at %s. Exiting" % (run_uuid, report_file) + "Successfully finished running Kraken, exiting" ) + else: logging.error("Cannot find a config at %s, please check" % (cfg)) # sys.exit(1) diff --git a/tests/test_prometheus_client.py b/tests/test_prometheus_client.py index 7fa2f908..95b1ce6c 100644 --- a/tests/test_prometheus_client.py +++ b/tests/test_prometheus_client.py @@ -53,7 +53,7 @@ class TestAlertsKeyValidation(unittest.TestCase): ) self.elastic.push_alert.return_value = 0 - client.alerts( + result = client.alerts( self.prom_cli, self.elastic, self.run_uuid, @@ -64,6 +64,7 @@ class TestAlertsKeyValidation(unittest.TestCase): ) self.prom_cli.process_alert.assert_called_once() + self.assertEqual(result, 1) finally: os.unlink(profile_path) @@ -138,6 +139,267 @@ class TestAlertsKeyValidation(unittest.TestCase): os.unlink(profile_path) +class TestAlertsFailureCount(unittest.TestCase): + """Tests that alerts() returns the correct failure count for critical/error severity.""" + + def setUp(self): + self.prom_cli = MagicMock() + self.elastic = MagicMock() + self.run_uuid = "test-uuid" + self.start_time = 1000000.0 + self.end_time = 1000060.0 + self.elastic_alerts_index = "test-index" + + def _write_alert_profile(self, content): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) + f.write(content) + f.close() + return f.name + + def _call_alerts(self, profile_path): + return client.alerts( + self.prom_cli, + self.elastic, + self.run_uuid, + self.start_time, + self.end_time, + profile_path, + self.elastic_alerts_index, + ) + + def test_returns_zero_when_no_alerts_fire(self): + """Returns 0 when process_alert returns (None, None) for all alerts.""" + profile_path = self._write_alert_profile( + '- expr: "up == 0"\n' + ' description: "target down"\n' + ' severity: "critical"\n' + ) + try: + self.prom_cli.process_alert.return_value = (None, None) + result = self._call_alerts(profile_path) + self.assertEqual(result, 0) + finally: + os.unlink(profile_path) + + def test_returns_one_for_single_critical_alert(self): + """Returns 1 when one critical-severity alert fires.""" + profile_path = self._write_alert_profile( + '- expr: "up == 0"\n' + ' description: "target down"\n' + ' severity: "critical"\n' + ) + try: + self.prom_cli.process_alert.return_value = (self.start_time, "target down") + self.elastic.push_alert.return_value = 0 + result = self._call_alerts(profile_path) + self.assertEqual(result, 1) + finally: + os.unlink(profile_path) + + def test_returns_one_for_single_error_alert(self): + """Returns 1 when one error-severity alert fires.""" + profile_path = self._write_alert_profile( + '- expr: "up == 0"\n' + ' description: "target down"\n' + ' severity: "error"\n' + ) + try: + self.prom_cli.process_alert.return_value = (self.start_time, "target down") + self.elastic.push_alert.return_value = 0 + result = self._call_alerts(profile_path) + self.assertEqual(result, 1) + finally: + os.unlink(profile_path) + + def test_warning_alert_does_not_increment_count(self): + """Returns 0 when only warning/info/debug alerts fire.""" + profile_path = self._write_alert_profile( + '- expr: "up == 0"\n' + ' description: "slow"\n' + ' severity: "warning"\n' + '- expr: "up == 1"\n' + ' description: "info"\n' + ' severity: "info"\n' + ) + try: + self.prom_cli.process_alert.return_value = (self.start_time, "alert fired") + self.elastic.push_alert.return_value = 0 + result = self._call_alerts(profile_path) + self.assertEqual(result, 0) + finally: + os.unlink(profile_path) + + def test_counts_multiple_critical_and_error_alerts(self): + """Returns correct total when multiple critical and error alerts fire.""" + profile_path = self._write_alert_profile( + '- expr: "a == 0"\n' + ' description: "a"\n' + ' severity: "critical"\n' + '- expr: "b == 0"\n' + ' description: "b"\n' + ' severity: "error"\n' + '- expr: "c == 0"\n' + ' description: "c"\n' + ' severity: "warning"\n' + '- expr: "d == 0"\n' + ' description: "d"\n' + ' severity: "critical"\n' + ) + try: + self.prom_cli.process_alert.return_value = (self.start_time, "fired") + self.elastic.push_alert.return_value = 0 + result = self._call_alerts(profile_path) + self.assertEqual(result, 3) + finally: + os.unlink(profile_path) + + def test_non_firing_critical_alerts_not_counted(self): + """Critical alerts that don't fire (return None) are not counted.""" + profile_path = self._write_alert_profile( + '- expr: "a == 0"\n' + ' description: "fired"\n' + ' severity: "critical"\n' + '- expr: "b == 0"\n' + ' description: "not fired"\n' + ' severity: "critical"\n' + ) + try: + self.prom_cli.process_alert.side_effect = [ + (self.start_time, "fired"), + (None, None), + ] + self.elastic.push_alert.return_value = 0 + result = self._call_alerts(profile_path) + self.assertEqual(result, 1) + finally: + os.unlink(profile_path) + + def test_elastic_push_still_called_for_non_critical_firing_alerts(self): + """Elastic push is called for warning/info alerts that fire even though they don't fail the run.""" + profile_path = self._write_alert_profile( + '- expr: "up == 0"\n' + ' description: "slow"\n' + ' severity: "warning"\n' + ) + try: + self.prom_cli.process_alert.return_value = (self.start_time, "slow") + self.elastic.push_alert.return_value = 0 + result = self._call_alerts(profile_path) + self.assertEqual(result, 0) + self.elastic.push_alert.assert_called_once() + finally: + os.unlink(profile_path) + + +class TestJobStatusComputation(unittest.TestCase): + """ + Tests the job_status logic from run_kraken.py: + + if post_critical_alerts > 0 or profile_critical_alerts > 0: + chaos_telemetry.job_status = False + + chaos_output.job_status = ( + chaos_telemetry.job_status + and post_critical_alerts == 0 + and profile_critical_alerts == 0 + ) + + Mirrors the logic directly so that regressions in the computation are caught. + """ + + def _compute(self, telemetry_job_status, post_critical_alerts, profile_critical_alerts): + if post_critical_alerts > 0 or profile_critical_alerts > 0: + telemetry_job_status = False + return ( + telemetry_job_status + and post_critical_alerts == 0 + and profile_critical_alerts == 0 + ) + + def test_true_when_no_failures(self): + self.assertTrue(self._compute(True, 0, 0)) + + def test_false_on_post_critical_alerts(self): + self.assertFalse(self._compute(True, 1, 0)) + + def test_false_on_profile_critical_alerts(self): + self.assertFalse(self._compute(True, 0, 1)) + + def test_false_on_scenario_failure(self): + """chaos_telemetry.job_status starts False when a scenario failed (set by krkn-lib).""" + self.assertFalse(self._compute(False, 0, 0)) + + def test_false_when_all_fail(self): + self.assertFalse(self._compute(False, 2, 3)) + + def test_profile_alerts_override_passing_telemetry(self): + """Even if no scenarios failed, profile critical alerts must flip job_status False.""" + self.assertFalse(self._compute(True, 0, 1)) + + def test_post_critical_alerts_override_passing_telemetry(self): + """Even if no scenarios failed, post-chaos critical alerts must flip job_status False.""" + self.assertFalse(self._compute(True, 1, 0)) + + def test_telemetry_job_status_mutated_by_alerts(self): + """Verifies that chaos_telemetry.job_status is set to False when alerts fire, + not just chaos_output.job_status — both fields in the output must be false.""" + telemetry_job_status = True + post_critical_alerts = 0 + profile_critical_alerts = 1 + + if post_critical_alerts > 0 or profile_critical_alerts > 0: + telemetry_job_status = False + + self.assertFalse(telemetry_job_status) + self.assertFalse( + telemetry_job_status + and post_critical_alerts == 0 + and profile_critical_alerts == 0 + ) + + # --- exit guard: not chaos_output.job_status --- + + def test_exit_guard_triggers_when_job_status_false(self): + """not chaos_output.job_status is True when job_status is False — guard fires.""" + job_status = self._compute(False, 0, 0) + self.assertTrue(not job_status) + + def test_exit_guard_does_not_trigger_when_job_status_true(self): + """not chaos_output.job_status is False when everything passed — guard skipped.""" + job_status = self._compute(True, 0, 0) + self.assertFalse(not job_status) + + def test_exit_guard_catches_scenario_failure_not_in_failed_post_scenarios(self): + """The guard catches chaos_telemetry.job_status=False (set by krkn-lib for a scenario + failure) even when post_critical_alerts==0, profile_critical_alerts==0, and + failed_post_scenarios is empty — i.e. when prior specific checks all pass.""" + post_critical_alerts = 0 + profile_critical_alerts = 0 + failed_post_scenarios = [] + + # Prior checks do not fire + self.assertFalse(bool(failed_post_scenarios)) + self.assertEqual(post_critical_alerts, 0) + self.assertEqual(profile_critical_alerts, 0) + + # krkn-lib set telemetry job_status=False due to a scenario exit_status > 0 + job_status = self._compute( + telemetry_job_status=False, + post_critical_alerts=post_critical_alerts, + profile_critical_alerts=profile_critical_alerts, + ) + # The catch-all guard fires + self.assertTrue(not job_status) + + def test_exit_guard_does_not_double_trigger_on_alert_failure(self): + """When alerts already caused an earlier return (post/profile > 0), job_status + is also False — the guard would fire too, but order ensures alerts return first.""" + job_status = self._compute(True, 1, 0) + self.assertFalse(job_status) + # guard would trigger, but post_critical_alerts > 0 returns 2 before reaching it + self.assertTrue(not job_status) + + class TestMetricsQueryRouting(unittest.TestCase): """Tests for metric query routing in the metrics() function."""