mirror of
https://github.com/krkn-chaos/krkn.git
synced 2026-08-25 09:27:36 +00:00
adding critical alerts to post to elastic search
Signed-off-by: Paige Patton <prubenda@redhat.com>
This commit is contained in:
committed by
Naga Ravi Chaitanya Elluri
parent
d434bb26fa
commit
a5459792ef
@@ -75,10 +75,12 @@ def alerts(
|
||||
def critical_alerts(
|
||||
prom_cli: KrknPrometheus,
|
||||
summary: ChaosRunAlertSummary,
|
||||
elastic: KrknElastic,
|
||||
run_id,
|
||||
scenario,
|
||||
start_time,
|
||||
end_time,
|
||||
elastic_alerts_index
|
||||
):
|
||||
summary.scenario = scenario
|
||||
summary.run_id = run_id
|
||||
@@ -113,7 +115,6 @@ def critical_alerts(
|
||||
summary.chaos_alerts.append(alert)
|
||||
|
||||
post_critical_alerts = prom_cli.process_query(query)
|
||||
|
||||
for alert in post_critical_alerts:
|
||||
if "metric" in alert:
|
||||
alertname = (
|
||||
@@ -136,6 +137,21 @@ def critical_alerts(
|
||||
)
|
||||
alert = ChaosRunAlert(alertname, alertstate, namespace, severity)
|
||||
summary.post_chaos_alerts.append(alert)
|
||||
if elastic:
|
||||
elastic_alert = ElasticAlert(
|
||||
run_uuid=run_id,
|
||||
severity=severity,
|
||||
alert=alertname,
|
||||
created_at=end_time,
|
||||
namespace=namespace,
|
||||
alertstate=alertstate,
|
||||
phase="post_chaos"
|
||||
)
|
||||
result = elastic.push_alert(elastic_alert, elastic_alerts_index)
|
||||
if result == -1:
|
||||
logging.error("failed to save alert on ElasticSearch")
|
||||
pass
|
||||
|
||||
|
||||
during_critical_alerts_count = len(during_critical_alerts)
|
||||
post_critical_alerts_count = len(post_critical_alerts)
|
||||
@@ -149,8 +165,8 @@ def critical_alerts(
|
||||
|
||||
if not firing_alerts:
|
||||
logging.info("No critical alerts are firing!!")
|
||||
|
||||
|
||||
|
||||
|
||||
def metrics(
|
||||
prom_cli: KrknPrometheus,
|
||||
elastic: KrknElastic,
|
||||
|
||||
@@ -375,10 +375,12 @@ def main(options, command: Optional[str]) -> int:
|
||||
prometheus_plugin.critical_alerts(
|
||||
prometheus,
|
||||
summary,
|
||||
elastic_search,
|
||||
run_uuid,
|
||||
scenario_type,
|
||||
start_time,
|
||||
datetime.datetime.now(),
|
||||
elastic_alerts_index
|
||||
)
|
||||
|
||||
chaos_output.critical_alerts = summary
|
||||
|
||||
Reference in New Issue
Block a user