github/krkn
1
0
Fork 0
mirror of https://github.com/krkn-chaos/krkn.git synced 2026-08-25 09:27:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding failed alerts to telemetery (#1552)

Browse Source
This commit is contained in:
Paige Patton
2026-08-13 13:07:01 -04:00
committed by GitHub
parent e86ac5b802
commit fef4f2d862
3 changed files with 38 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
krkn/prometheus/client.py
+19 -11
View File
@@ -28,6 +28,7 @@ import yaml
from krkn_lib.elastic.krkn_elastic import KrknElastic
from krkn_lib.models.elastic.models import ElasticAlert
from krkn_lib.models.krkn import ChaosRunAlertSummary, ChaosRunAlert
from krkn_lib.models.telemetry import FailedAlert
from krkn_lib.prometheus.krkn_prometheus import KrknPrometheus
@@ -58,8 +59,7 @@ def alerts(
)
sys.exit(1)
# Will fail run if error or critical alerts are firing
failure_alert_count = 0
fired: list[FailedAlert] = []
for alert in profile_yaml:
if sorted(alert.keys()) != sorted(["expr", "description", "severity"]):
logging.error(f"wrong alert {alert}, skipping")
@@ -71,21 +71,27 @@ def alerts(
datetime.datetime.fromtimestamp(end_time),
)
if processed_alert[0] and processed_alert[1]:
if alert["severity"] == "critical":
failure_alert_count += 1
if alert["severity"] == "error":
failure_alert_count += 1
severity = alert["severity"]
created_datetime = datetime.datetime.fromtimestamp(processed_alert[0])
if severity in ("critical", "error"):
fired.append(FailedAlert({
"name": alert["description"],
"severity": severity,
"message": processed_alert[1],
"namespace": "",
"starts_at": str(created_datetime),
}))
if elastic:
elastic_alert = ElasticAlert(
run_uuid=run_uuid,
severity=alert["severity"],
severity=severity,
alert=processed_alert[1],
created_at=datetime.datetime.fromtimestamp(processed_alert[0]),
created_at=created_datetime,
)
result = elastic.push_alert(elastic_alert, elastic_alerts_index)
if result == -1:
logging.error("failed to save alert on ElasticSearch")
return failure_alert_count
return fired
def critical_alerts(
@@ -181,8 +187,10 @@ def critical_alerts(
if not firing_alerts:
logging.info("No critical alerts are firing!!")
def metrics(
prom_cli: KrknPrometheus,
elastic: KrknElastic,
run_kraken.py
+4 -2
View File
@@ -645,11 +645,13 @@ def main(options, command: Optional[str], out: Optional[dict] = None) -> int:
alert_profile,
elastic_alerts_index
)
if profile_critical_alerts:
chaos_telemetry.failed_alerts = profile_critical_alerts
else:
logging.error("Alert profile is not defined")
return -1
if post_critical_alerts > 0 or profile_critical_alerts > 0:
if post_critical_alerts > 0 or len(profile_critical_alerts) > 0:
chaos_telemetry.job_status = False
telemetry_json = chaos_telemetry.to_json()
@@ -809,7 +811,7 @@ def main(options, command: Optional[str], out: Optional[dict] = None) -> int:
logging.error("Critical alerts are firing, please check; exiting")
return 2
if profile_critical_alerts > 0:
if len(profile_critical_alerts) > 0:
logging.error("Critical or Error alerts from alert profile are firing, please check; exiting")
return 2
tests/test_prometheus_client.py
+15 -15
View File
@@ -64,7 +64,7 @@ class TestAlertsKeyValidation(unittest.TestCase):
)
self.prom_cli.process_alert.assert_called_once()
self.assertEqual(result, 1)
self.assertEqual(len(result), 1)
finally:
os.unlink(profile_path)
@@ -140,7 +140,7 @@ class TestAlertsKeyValidation(unittest.TestCase):
class TestAlertsFailureCount(unittest.TestCase):
"""Tests that alerts() returns the correct failure count for critical/error severity."""
"""Tests that alerts() returns the correct FailedAlert list for critical/error severity."""
def setUp(self):
self.prom_cli = MagicMock()
@@ -168,7 +168,7 @@ class TestAlertsFailureCount(unittest.TestCase):
)
def test_returns_zero_when_no_alerts_fire(self):
"""Returns 0 when process_alert returns (None, None) for all alerts."""
"""Returns empty list when process_alert returns (None, None) for all alerts."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "target down"\n'
@@ -177,12 +177,12 @@ class TestAlertsFailureCount(unittest.TestCase):
try:
self.prom_cli.process_alert.return_value = (None, None)
result = self._call_alerts(profile_path)
self.assertEqual(result, 0)
self.assertEqual(result, [])
finally:
os.unlink(profile_path)
def test_returns_one_for_single_critical_alert(self):
"""Returns 1 when one critical-severity alert fires."""
"""Returns one FailedAlert when one critical-severity alert fires."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "target down"\n'
@@ -192,12 +192,12 @@ class TestAlertsFailureCount(unittest.TestCase):
self.prom_cli.process_alert.return_value = (self.start_time, "target down")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 1)
self.assertEqual(len(result), 1)
finally:
os.unlink(profile_path)
def test_returns_one_for_single_error_alert(self):
"""Returns 1 when one error-severity alert fires."""
"""Returns one FailedAlert when one error-severity alert fires."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "target down"\n'
@@ -207,12 +207,12 @@ class TestAlertsFailureCount(unittest.TestCase):
self.prom_cli.process_alert.return_value = (self.start_time, "target down")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 1)
self.assertEqual(len(result), 1)
finally:
os.unlink(profile_path)
def test_warning_alert_does_not_increment_count(self):
"""Returns 0 when only warning/info/debug alerts fire."""
"""Returns empty list when only warning/info/debug alerts fire."""
profile_path = self._write_alert_profile(
'- expr: "up == 0"\n'
' description: "slow"\n'
@@ -225,12 +225,12 @@ class TestAlertsFailureCount(unittest.TestCase):
self.prom_cli.process_alert.return_value = (self.start_time, "alert fired")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 0)
self.assertEqual(result, [])
finally:
os.unlink(profile_path)
def test_counts_multiple_critical_and_error_alerts(self):
"""Returns correct total when multiple critical and error alerts fire."""
"""Returns 3 FailedAlerts when multiple critical and error alerts fire."""
profile_path = self._write_alert_profile(
'- expr: "a == 0"\n'
' description: "a"\n'
@@ -249,12 +249,12 @@ class TestAlertsFailureCount(unittest.TestCase):
self.prom_cli.process_alert.return_value = (self.start_time, "fired")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 3)
self.assertEqual(len(result), 3)
finally:
os.unlink(profile_path)
def test_non_firing_critical_alerts_not_counted(self):
"""Critical alerts that don't fire (return None) are not counted."""
"""Critical alerts that don't fire (return None) produce no FailedAlert."""
profile_path = self._write_alert_profile(
'- expr: "a == 0"\n'
' description: "fired"\n'
@@ -270,7 +270,7 @@ class TestAlertsFailureCount(unittest.TestCase):
]
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 1)
self.assertEqual(len(result), 1)
finally:
os.unlink(profile_path)
@@ -285,7 +285,7 @@ class TestAlertsFailureCount(unittest.TestCase):
self.prom_cli.process_alert.return_value = (self.start_time, "slow")
self.elastic.push_alert.return_value = 0
result = self._call_alerts(profile_path)
self.assertEqual(result, 0)
self.assertEqual(result, [])
self.elastic.push_alert.assert_called_once()
finally:
os.unlink(profile_path)