From 9df727ccf5996d573e68486fcdc0735244024cbf Mon Sep 17 00:00:00 2001 From: Anshuman Panda Date: Tue, 3 Jun 2025 05:41:01 +0530 Subject: [PATCH] Ensure metrics are always saved with improved local fallback Signed-off-by: Anshuman Panda Signed-off-by: Paige Patton --- krkn/prometheus/client.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/krkn/prometheus/client.py b/krkn/prometheus/client.py index 095c77c1..0acc2a0b 100644 --- a/krkn/prometheus/client.py +++ b/krkn/prometheus/client.py @@ -9,6 +9,7 @@ import logging import urllib3 import sys import json +import tempfile import yaml from krkn_lib.elastic.krkn_elastic import KrknElastic @@ -251,11 +252,29 @@ def metrics( metric[k] = v metric['timestamp'] = str(datetime.datetime.now()) metrics_list.append(metric.copy()) - if elastic: + + save_metrics = False + if elastic is not None and elastic_metrics_index is not None: result = elastic.upload_metrics_to_elasticsearch( run_uuid=run_uuid, index=elastic_metrics_index, raw_data=metrics_list ) if result == -1: logging.error("failed to save metrics on ElasticSearch") + save_metrics = True + else: + save_metrics = True + if save_metrics: + local_dir = os.path.join(tempfile.gettempdir(), "krkn_metrics") + os.makedirs(local_dir, exist_ok=True) + local_file = os.path.join(local_dir, f"{elastic_metrics_index}_{run_uuid}.json") + try: + with open(local_file, "w") as f: + json.dump({ + "run_uuid": run_uuid, + "metrics": metrics_list + }, f, indent=2) + logging.info(f"Metrics saved to {local_file}") + except Exception as e: + logging.error(f"Failed to save metrics to {local_file}: {e}") return metrics_list