diff --git a/krkn/prometheus/client.py b/krkn/prometheus/client.py index 73a3104d..440139b6 100644 --- a/krkn/prometheus/client.py +++ b/krkn/prometheus/client.py @@ -28,6 +28,7 @@ import yaml from krkn_lib.elastic.krkn_elastic import KrknElastic from krkn_lib.models.elastic.models import ElasticAlert from krkn_lib.models.krkn import ChaosRunAlertSummary, ChaosRunAlert +from krkn_lib.models.telemetry import FailedAlert from krkn_lib.prometheus.krkn_prometheus import KrknPrometheus @@ -58,8 +59,7 @@ def alerts( ) sys.exit(1) - # Will fail run if error or critical alerts are firing - failure_alert_count = 0 + fired: list[FailedAlert] = [] for alert in profile_yaml: if sorted(alert.keys()) != sorted(["expr", "description", "severity"]): logging.error(f"wrong alert {alert}, skipping") @@ -71,21 +71,27 @@ def alerts( datetime.datetime.fromtimestamp(end_time), ) if processed_alert[0] and processed_alert[1]: - if alert["severity"] == "critical": - failure_alert_count += 1 - if alert["severity"] == "error": - failure_alert_count += 1 + severity = alert["severity"] + created_datetime = datetime.datetime.fromtimestamp(processed_alert[0]) + if severity in ("critical", "error"): + fired.append(FailedAlert({ + "name": alert["description"], + "severity": severity, + "message": processed_alert[1], + "namespace": "", + "starts_at": str(created_datetime), + })) if elastic: elastic_alert = ElasticAlert( run_uuid=run_uuid, - severity=alert["severity"], + severity=severity, alert=processed_alert[1], - created_at=datetime.datetime.fromtimestamp(processed_alert[0]), + created_at=created_datetime, ) result = elastic.push_alert(elastic_alert, elastic_alerts_index) if result == -1: logging.error("failed to save alert on ElasticSearch") - return failure_alert_count + return fired def critical_alerts( @@ -181,8 +187,10 @@ def critical_alerts( if not firing_alerts: logging.info("No critical alerts are firing!!") - - + + + + def metrics( prom_cli: KrknPrometheus, elastic: KrknElastic, diff --git a/run_kraken.py b/run_kraken.py index 901fec6c..0622df22 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -645,11 +645,13 @@ def main(options, command: Optional[str], out: Optional[dict] = None) -> int: alert_profile, elastic_alerts_index ) + if profile_critical_alerts: + chaos_telemetry.failed_alerts = profile_critical_alerts else: logging.error("Alert profile is not defined") return -1 - if post_critical_alerts > 0 or profile_critical_alerts > 0: + if post_critical_alerts > 0 or len(profile_critical_alerts) > 0: chaos_telemetry.job_status = False telemetry_json = chaos_telemetry.to_json() @@ -809,7 +811,7 @@ def main(options, command: Optional[str], out: Optional[dict] = None) -> int: logging.error("Critical alerts are firing, please check; exiting") return 2 - if profile_critical_alerts > 0: + if len(profile_critical_alerts) > 0: logging.error("Critical or Error alerts from alert profile are firing, please check; exiting") return 2 diff --git a/tests/test_prometheus_client.py b/tests/test_prometheus_client.py index 95b1ce6c..e0bbbf02 100644 --- a/tests/test_prometheus_client.py +++ b/tests/test_prometheus_client.py @@ -64,7 +64,7 @@ class TestAlertsKeyValidation(unittest.TestCase): ) self.prom_cli.process_alert.assert_called_once() - self.assertEqual(result, 1) + self.assertEqual(len(result), 1) finally: os.unlink(profile_path) @@ -140,7 +140,7 @@ class TestAlertsKeyValidation(unittest.TestCase): class TestAlertsFailureCount(unittest.TestCase): - """Tests that alerts() returns the correct failure count for critical/error severity.""" + """Tests that alerts() returns the correct FailedAlert list for critical/error severity.""" def setUp(self): self.prom_cli = MagicMock() @@ -168,7 +168,7 @@ class TestAlertsFailureCount(unittest.TestCase): ) def test_returns_zero_when_no_alerts_fire(self): - """Returns 0 when process_alert returns (None, None) for all alerts.""" + """Returns empty list when process_alert returns (None, None) for all alerts.""" profile_path = self._write_alert_profile( '- expr: "up == 0"\n' ' description: "target down"\n' @@ -177,12 +177,12 @@ class TestAlertsFailureCount(unittest.TestCase): try: self.prom_cli.process_alert.return_value = (None, None) result = self._call_alerts(profile_path) - self.assertEqual(result, 0) + self.assertEqual(result, []) finally: os.unlink(profile_path) def test_returns_one_for_single_critical_alert(self): - """Returns 1 when one critical-severity alert fires.""" + """Returns one FailedAlert when one critical-severity alert fires.""" profile_path = self._write_alert_profile( '- expr: "up == 0"\n' ' description: "target down"\n' @@ -192,12 +192,12 @@ class TestAlertsFailureCount(unittest.TestCase): self.prom_cli.process_alert.return_value = (self.start_time, "target down") self.elastic.push_alert.return_value = 0 result = self._call_alerts(profile_path) - self.assertEqual(result, 1) + self.assertEqual(len(result), 1) finally: os.unlink(profile_path) def test_returns_one_for_single_error_alert(self): - """Returns 1 when one error-severity alert fires.""" + """Returns one FailedAlert when one error-severity alert fires.""" profile_path = self._write_alert_profile( '- expr: "up == 0"\n' ' description: "target down"\n' @@ -207,12 +207,12 @@ class TestAlertsFailureCount(unittest.TestCase): self.prom_cli.process_alert.return_value = (self.start_time, "target down") self.elastic.push_alert.return_value = 0 result = self._call_alerts(profile_path) - self.assertEqual(result, 1) + self.assertEqual(len(result), 1) finally: os.unlink(profile_path) def test_warning_alert_does_not_increment_count(self): - """Returns 0 when only warning/info/debug alerts fire.""" + """Returns empty list when only warning/info/debug alerts fire.""" profile_path = self._write_alert_profile( '- expr: "up == 0"\n' ' description: "slow"\n' @@ -225,12 +225,12 @@ class TestAlertsFailureCount(unittest.TestCase): self.prom_cli.process_alert.return_value = (self.start_time, "alert fired") self.elastic.push_alert.return_value = 0 result = self._call_alerts(profile_path) - self.assertEqual(result, 0) + self.assertEqual(result, []) finally: os.unlink(profile_path) def test_counts_multiple_critical_and_error_alerts(self): - """Returns correct total when multiple critical and error alerts fire.""" + """Returns 3 FailedAlerts when multiple critical and error alerts fire.""" profile_path = self._write_alert_profile( '- expr: "a == 0"\n' ' description: "a"\n' @@ -249,12 +249,12 @@ class TestAlertsFailureCount(unittest.TestCase): self.prom_cli.process_alert.return_value = (self.start_time, "fired") self.elastic.push_alert.return_value = 0 result = self._call_alerts(profile_path) - self.assertEqual(result, 3) + self.assertEqual(len(result), 3) finally: os.unlink(profile_path) def test_non_firing_critical_alerts_not_counted(self): - """Critical alerts that don't fire (return None) are not counted.""" + """Critical alerts that don't fire (return None) produce no FailedAlert.""" profile_path = self._write_alert_profile( '- expr: "a == 0"\n' ' description: "fired"\n' @@ -270,7 +270,7 @@ class TestAlertsFailureCount(unittest.TestCase): ] self.elastic.push_alert.return_value = 0 result = self._call_alerts(profile_path) - self.assertEqual(result, 1) + self.assertEqual(len(result), 1) finally: os.unlink(profile_path) @@ -285,7 +285,7 @@ class TestAlertsFailureCount(unittest.TestCase): self.prom_cli.process_alert.return_value = (self.start_time, "slow") self.elastic.push_alert.return_value = 0 result = self._call_alerts(profile_path) - self.assertEqual(result, 0) + self.assertEqual(result, []) self.elastic.push_alert.assert_called_once() finally: os.unlink(profile_path)