From 87c2b3c8fde297dad08dff4195b982b736a20c97 Mon Sep 17 00:00:00 2001 From: Paige Patton Date: Wed, 7 May 2025 18:54:33 -0400 Subject: [PATCH] adding recovery times to metrics Signed-off-by: Paige Patton --- krkn/prometheus/client.py | 32 ++++++++++++++++++++++++++++++-- run_kraken.py | 3 ++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/krkn/prometheus/client.py b/krkn/prometheus/client.py index 383c4191..095c77c1 100644 --- a/krkn/prometheus/client.py +++ b/krkn/prometheus/client.py @@ -8,6 +8,7 @@ from typing import Optional, List, Dict, Any import logging import urllib3 import sys +import json import yaml from krkn_lib.elastic.krkn_elastic import KrknElastic @@ -156,7 +157,8 @@ def metrics( start_time, end_time, metrics_profile, - elastic_metrics_index + elastic_metrics_index, + telemetry_json ) -> list[dict[str, list[(int, float)] | str]]: if metrics_profile is None or os.path.exists(metrics_profile) is False: @@ -222,7 +224,33 @@ def metrics( metrics_list.append(metric.copy()) except ValueError: pass - + telemetry_json = json.loads(telemetry_json) + for scenario in telemetry_json['scenarios']: + for k,v in scenario["affected_pods"].items(): + metric_name = "affected_pods_recovery" + metric = {"metricName": metric_name, "type": k} + if type(v) is list: + for pod in v: + for k,v in pod.items(): + metric[k] = v + metric['timestamp'] = str(datetime.datetime.now()) + print('adding pod' + str(metric)) + metrics_list.append(metric.copy()) + for affected_node in scenario["affected_nodes"]: + metric_name = "affected_nodes_recovery" + metric = {"metricName": metric_name} + for k,v in affected_node.items(): + metric[k] = v + metric['timestamp'] = str(datetime.datetime.now()) + metrics_list.append(metric.copy()) + if telemetry_json['health_checks']: + for health_check in telemetry_json["health_checks"]: + metric_name = "health_check_recovery" + metric = {"metricName": metric_name} + for k,v in health_check.items(): + metric[k] = v + metric['timestamp'] = str(datetime.datetime.now()) + metrics_list.append(metric.copy()) if elastic: result = elastic.upload_metrics_to_elasticsearch( run_uuid=run_uuid, index=elastic_metrics_index, raw_data=metrics_list diff --git a/run_kraken.py b/run_kraken.py index e2052566..06ad2210 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -499,7 +499,8 @@ def main(cfg) -> int: start_time, end_time, metrics_profile, - elastic_metrics_index + elastic_metrics_index, + telemetry_json ) if post_critical_alerts > 0: