diff --git a/krkn/resiliency/__init__.py b/krkn/resiliency/__init__.py index 7b963495..75246d0c 100644 --- a/krkn/resiliency/__init__.py +++ b/krkn/resiliency/__init__.py @@ -16,3 +16,8 @@ from .resiliency import Resiliency # noqa: F401 from .score import calculate_resiliency_score # noqa: F401 +from .history import ( # noqa: F401 + HistoryWindow, + parse_history_window, + apply_historical_resiliency, +) diff --git a/krkn/resiliency/history.py b/krkn/resiliency/history.py new file mode 100644 index 00000000..ca1f6c03 --- /dev/null +++ b/krkn/resiliency/history.py @@ -0,0 +1,204 @@ +# Copyright 2025 The Krkn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Historical resiliency-score queries. + +Provides helpers to parse a CLI time-window (duration string or explicit +start/end timestamps), query Prometheus over that window, and populate the +overall_resiliency_report field of a ChaosRunTelemetry object. +""" + +from __future__ import annotations + +import datetime +import logging +from dataclasses import dataclass +from typing import Optional + +from krkn_lib.models.k8s import ResiliencyReport +from krkn_lib.models.telemetry import ChaosRunTelemetry +from krkn_lib.prometheus.krkn_prometheus import KrknPrometheus + +from krkn.resiliency.resiliency import Resiliency + + +# --------------------------------------------------------------------------- +# Parsing helpers +# --------------------------------------------------------------------------- + +def parse_duration(duration_str: str) -> datetime.timedelta: + """Parse a human-friendly duration string into a timedelta. + + Supported units: s (seconds), m (minutes), h (hours), d (days), w (weeks). + Examples: '30s', '5m', '24h', '7d', '2w' + """ + units = { + 's': 'seconds', + 'm': 'minutes', + 'h': 'hours', + 'd': 'days', + 'w': 'weeks', + } + s = duration_str.strip().lower() + if len(s) < 2: + raise ValueError( + f"Invalid duration '{duration_str}': expected a number followed by a unit (s, m, h, d, w)" + ) + unit = s[-1] + if unit not in units: + raise ValueError( + f"Unknown duration unit '{unit}' in '{duration_str}'. Supported units: s, m, h, d, w" + ) + try: + value = float(s[:-1]) + except ValueError: + raise ValueError(f"Invalid numeric value '{s[:-1]}' in '{duration_str}'") + if value <= 0: + raise ValueError(f"Duration must be positive, got '{duration_str}'") + return datetime.timedelta(**{units[unit]: value}) + + +def parse_datetime(dt_str: str) -> datetime.datetime: + """Parse a datetime string into a UTC-aware datetime object. + + The input is always interpreted as UTC regardless of the host timezone. + + Supported formats: + YYYY-MM-DDTHH:MM:SS (ISO 8601) + YYYY-MM-DD HH:MM:SS + YYYY-MM-DD (midnight assumed) + """ + for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): + try: + naive = datetime.datetime.strptime(dt_str.strip(), fmt) + return naive.replace(tzinfo=datetime.timezone.utc) + except ValueError: + continue + raise ValueError( + f"Cannot parse datetime '{dt_str}'. " + "Use ISO 8601 format: YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD (times are UTC)" + ) + + +# --------------------------------------------------------------------------- +# Window representation +# --------------------------------------------------------------------------- + +@dataclass +class HistoryWindow: + """A resolved start/end time window for a historical resiliency query.""" + start: datetime.datetime + end: datetime.datetime + label: str + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def parse_history_window( + past_resiliency_score: Optional[str], + hist_start_str: Optional[str], + hist_end_str: Optional[str], + resiliency_score_flag: bool = False, +) -> Optional[HistoryWindow]: + """Parse and validate historical resiliency CLI options. + + Returns a :class:`HistoryWindow` when any window option was provided, or + ``None`` when none were supplied (normal chaos run). + + Args: + past_resiliency_score: Trailing duration string (e.g. '1h', '24h'). + hist_start_str: Explicit window start datetime string. + hist_end_str: Explicit window end datetime string. + resiliency_score_flag: Must be ``True`` when using ``--start-time``/ + ``--end-time``; set by the ``--resiliency-score`` flag or the + ``resiliency-score`` command. + + Raises: + ValueError: with a human-readable message when options are invalid. + """ + if past_resiliency_score is not None and (hist_start_str or hist_end_str): + raise ValueError( + "--past-resiliency-score and --start-time/--end-time are mutually exclusive" + ) + + if (hist_start_str or hist_end_str) and not resiliency_score_flag: + raise ValueError( + "--start-time/--end-time require the --resiliency-score flag " + "or the resiliency-score command" + ) + + if past_resiliency_score is not None: + duration = parse_duration(past_resiliency_score) + end = datetime.datetime.now(datetime.timezone.utc) + start = end - duration + return HistoryWindow(start=start, end=end, label=past_resiliency_score) + + if hist_start_str or hist_end_str: + if not hist_start_str or not hist_end_str: + raise ValueError("--start-time and --end-time must both be provided together") + start = parse_datetime(hist_start_str) + end = parse_datetime(hist_end_str) + if end <= start: + raise ValueError("--end-time must be after --start-time") + return HistoryWindow(start=start, end=end, label=f"{start} → {end}") + + return None + + +def apply_historical_resiliency( + window: HistoryWindow, + resiliency_obj: Resiliency, + prometheus: KrknPrometheus, + telemetry: ChaosRunTelemetry, +) -> None: + """Query Prometheus over *window* and populate ``telemetry.overall_resiliency_report``. + + Raises: + RuntimeError: when Prometheus or the resiliency object is unavailable. + """ + if resiliency_obj is None or prometheus is None: + raise RuntimeError( + "Prometheus is required for historical resiliency scoring but is not available. " + "Ensure prometheus_url is set in config and Prometheus is reachable." + ) + + logging.info( + "Querying historical resiliency score for window %s → %s", window.start, window.end + ) + resiliency_obj.add_scenario_report( + scenario_name="historical", + prom_cli=prometheus, + start_time=window.start, + end_time=window.end, + ) + hist_report = resiliency_obj.scenario_reports[-1] + hist_score = hist_report["score"] + hist_breakdown = hist_report.get("breakdown", {}) + passed = hist_breakdown.get("passed", 0) + total = passed + hist_breakdown.get("failed", 0) + history_summary = { + "scenarios": {"historical": hist_score}, + "resiliency_score": hist_score, + "passed_slos": passed, + "total_slos": total, + } + telemetry.overall_resiliency_report = ResiliencyReport( + json_object=history_summary, + resiliency_score=hist_score, + passed_slos=passed, + total_slos=total, + ) + logging.info("Historical resiliency score (%s): %d", window.label, hist_score) diff --git a/run_kraken.py b/run_kraken.py index 84c8f22a..f8ff0bb0 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -46,6 +46,11 @@ import server as server from krkn.resiliency.resiliency import ( Resiliency ) +from krkn.resiliency.history import ( + HistoryWindow, + parse_history_window, + apply_historical_resiliency, +) from krkn_lib.k8s import KrknKubernetes from krkn_lib.ocp import KrknOpenshift from krkn_lib.telemetry.k8s import KrknTelemetryKubernetes @@ -74,6 +79,7 @@ warnings.filterwarnings(action='ignore', module='.*paramiko.*') report_file = "" + # Main function def main(options, command: Optional[str]) -> int: # Start kraken @@ -131,6 +137,24 @@ def main(options, command: Optional[str]) -> int: if run_mode not in valid_run_modes: logging.warning("Unknown resiliency_run_mode '%s'. Defaulting to 'standalone'", run_mode) run_mode = "standalone" + + try: + hist_window = parse_history_window( + getattr(options, "past_resiliency_score", None), + getattr(options, "hist_start_time", None), + getattr(options, "hist_end_time", None), + resiliency_score_flag=getattr(options, "resiliency_score", False) or command == "resiliency-score", + ) + except ValueError as exc: + logging.error("%s", exc) + return -1 + + if hist_window is not None: + logging.info( + "Historical resiliency window '%s' provided. Chaos scenarios will not be executed.", + hist_window.label, + ) + chaos_scenarios = [] wait_duration = get_yaml_item_value(config["tunings"], "wait_duration", 60) iterations = get_yaml_item_value(config["tunings"], "iterations", 1) daemon_mode = get_yaml_item_value(config["tunings"], "daemon_mode", False) @@ -341,6 +365,15 @@ def main(options, command: Optional[str]) -> int: telemetry_ocp, options.run_uuid, options.scenario_type ) ) + elif command == "resiliency-score": + if hist_window is None: + logging.error( + "resiliency-score command requires a time window: " + "use --past-resiliency-score (e.g. 24h) " + "or --start-time/--end-time for an explicit range" + ) + sys.exit(-1) + chaos_scenarios = [] # Initialize the start iteration to 0 iteration = 0 @@ -576,13 +609,13 @@ def main(options, command: Optional[str]) -> int: else: logging.info("No error logs collected during chaos run") chaos_telemetry.error_logs = [] - if resiliency_obj: + if resiliency_obj and hist_window is None: try: resiliency_obj.attach_compact_to_telemetry(chaos_telemetry) except Exception as exc: logging.error("Failed to embed per-scenario resiliency in telemetry: %s", exc) - if resiliency_obj: + if resiliency_obj and hist_window is None: try: resiliency_obj.finalize_and_save( prom_cli=prometheus, @@ -617,7 +650,16 @@ def main(options, command: Optional[str]) -> int: telemetry_json = chaos_telemetry.to_json() decoded_chaos_run_telemetry = ChaosRunTelemetry(json.loads(telemetry_json)) - if resiliency_obj and hasattr(resiliency_obj, "summary") and resiliency_obj.summary is not None: + if hist_window is not None: + try: + apply_historical_resiliency(hist_window, resiliency_obj, prometheus, decoded_chaos_run_telemetry) + except RuntimeError as exc: + logging.error("%s", exc) + return -1 + except Exception as exc: + logging.error("Failed to compute historical resiliency score: %s", exc) + return -1 + elif resiliency_obj and hasattr(resiliency_obj, "summary") and resiliency_obj.summary is not None: summary_dict = resiliency_obj.get_summary() decoded_chaos_run_telemetry.overall_resiliency_report = ResiliencyReport( json_object=summary_dict, @@ -767,7 +809,9 @@ if __name__ == "__main__": usage="%prog [options] [command]\n\n" "Commands:\n" " list-rollback List rollback version files in a tree-like format\n" - " execute-rollback Execute rollback version files and cleanup if successful\n\n" + " execute-rollback Execute rollback version files and cleanup if successful\n" + " resiliency-score Query historical resiliency score without running chaos scenarios.\n" + " Requires --start-time/--end-time or --past-resiliency-score.\n\n" "If no command is specified, kraken will run chaos scenarios.", ) parser.add_option( @@ -831,6 +875,42 @@ if __name__ == "__main__": default=False, ) + parser.add_option( + "--past-resiliency-score", + dest="past_resiliency_score", + help="Query historical resiliency score over a trailing window (e.g. 1h, 24h, 7d) " + "without running chaos scenarios. Mutually exclusive with --start-time/--end-time.", + default=None, + ) + + parser.add_option( + "--resiliency-score", + dest="resiliency_score", + action="store_true", + help="Indicate that --start-time/--end-time define a historical resiliency score query. " + "Required when using --start-time/--end-time. " + "Implied automatically by the resiliency-score command.", + default=False, + ) + + parser.add_option( + "--start-time", + dest="hist_start_time", + help="Start of explicit historical resiliency window (YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD, UTC). " + "Must be used together with --end-time and --resiliency-score. " + "Mutually exclusive with --past-resiliency-score.", + default=None, + ) + + parser.add_option( + "--end-time", + dest="hist_end_time", + help="End of explicit historical resiliency window (YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD, UTC). " + "Must be used together with --start-time and --resiliency-score. " + "Mutually exclusive with --past-resiliency-score.", + default=None, + ) + (options, args) = parser.parse_args() # If no command or regular execution, continue with existing logic diff --git a/tests/test_resiliency_history.py b/tests/test_resiliency_history.py new file mode 100644 index 00000000..e37860ab --- /dev/null +++ b/tests/test_resiliency_history.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +Unit tests for krkn.resiliency.history. + +Usage: + python -m unittest tests/test_resiliency_history.py -v +""" + +import datetime +import sys +import os +import unittest +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from krkn.resiliency.history import ( + HistoryWindow, + apply_historical_resiliency, + parse_datetime, + parse_duration, + parse_history_window, +) + + +class TestParseDuration(unittest.TestCase): + + def test_minutes(self): + self.assertEqual(parse_duration("5m"), datetime.timedelta(minutes=5)) + + def test_hours(self): + self.assertEqual(parse_duration("24h"), datetime.timedelta(hours=24)) + + def test_days(self): + self.assertEqual(parse_duration("7d"), datetime.timedelta(days=7)) + + def test_seconds(self): + self.assertEqual(parse_duration("30s"), datetime.timedelta(seconds=30)) + + def test_weeks(self): + self.assertEqual(parse_duration("2w"), datetime.timedelta(weeks=2)) + + def test_uppercase_unit(self): + self.assertEqual(parse_duration("1H"), datetime.timedelta(hours=1)) + + def test_fractional_value(self): + self.assertEqual(parse_duration("1.5h"), datetime.timedelta(hours=1.5)) + + def test_unknown_unit_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_duration("5x") + self.assertIn("Unknown duration unit", str(ctx.exception)) + + def test_zero_value_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_duration("0h") + self.assertIn("positive", str(ctx.exception)) + + def test_negative_value_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_duration("-1h") + self.assertIn("positive", str(ctx.exception)) + + def test_non_numeric_value_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_duration("abch") + self.assertIn("Invalid numeric value", str(ctx.exception)) + + def test_too_short_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_duration("h") + self.assertIn("expected a number", str(ctx.exception)) + + +class TestParseDatetime(unittest.TestCase): + + def test_iso_with_time(self): + result = parse_datetime("2026-05-25T08:00:00") + self.assertEqual(result, datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc)) + + def test_space_separated(self): + result = parse_datetime("2026-05-25 08:00:00") + self.assertEqual(result, datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc)) + + def test_date_only(self): + result = parse_datetime("2026-05-25") + self.assertEqual(result, datetime.datetime(2026, 5, 25, 0, 0, 0, tzinfo=datetime.timezone.utc)) + + def test_leading_trailing_whitespace(self): + result = parse_datetime(" 2026-05-25T10:30:00 ") + self.assertEqual(result, datetime.datetime(2026, 5, 25, 10, 30, 0, tzinfo=datetime.timezone.utc)) + + def test_invalid_format_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_datetime("25/05/2026") + self.assertIn("Cannot parse datetime", str(ctx.exception)) + + def test_invalid_date_raises(self): + with self.assertRaises(ValueError): + parse_datetime("2026-13-01") + + +class TestParseHistoryWindow(unittest.TestCase): + + def test_no_options_returns_none(self): + self.assertIsNone(parse_history_window(None, None, None)) + + def test_duration_mode(self): + window = parse_history_window("1h", None, None) + self.assertIsNotNone(window) + self.assertEqual(window.label, "1h") + expected_delta = datetime.timedelta(hours=1) + self.assertAlmostEqual( + (window.end - window.start).total_seconds(), + expected_delta.total_seconds(), + delta=2, + ) + + def test_explicit_range_requires_flag(self): + with self.assertRaises(ValueError) as ctx: + parse_history_window(None, "2026-05-25T08:00:00", "2026-05-25T09:00:00") + self.assertIn("--resiliency-score", str(ctx.exception)) + + def test_explicit_range_with_flag(self): + window = parse_history_window( + None, "2026-05-25T08:00:00", "2026-05-25T09:00:00", + resiliency_score_flag=True, + ) + self.assertIsNotNone(window) + self.assertEqual(window.start, datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc)) + self.assertEqual(window.end, datetime.datetime(2026, 5, 25, 9, 0, 0, tzinfo=datetime.timezone.utc)) + + def test_explicit_range_label(self): + window = parse_history_window( + None, "2026-05-25", "2026-05-26", resiliency_score_flag=True, + ) + self.assertIn("2026-05-25", window.label) + self.assertIn("2026-05-26", window.label) + + def test_mutual_exclusivity_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_history_window( + "1h", "2026-05-25T08:00:00", "2026-05-25T09:00:00", + resiliency_score_flag=True, + ) + self.assertIn("mutually exclusive", str(ctx.exception)) + + def test_missing_end_time_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_history_window(None, "2026-05-25T08:00:00", None, resiliency_score_flag=True) + self.assertIn("both be provided", str(ctx.exception)) + + def test_missing_start_time_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_history_window(None, None, "2026-05-25T09:00:00", resiliency_score_flag=True) + self.assertIn("both be provided", str(ctx.exception)) + + def test_end_before_start_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_history_window( + None, "2026-05-25T09:00:00", "2026-05-25T08:00:00", + resiliency_score_flag=True, + ) + self.assertIn("after --start-time", str(ctx.exception)) + + def test_end_equal_start_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_history_window( + None, "2026-05-25T08:00:00", "2026-05-25T08:00:00", + resiliency_score_flag=True, + ) + self.assertIn("after --start-time", str(ctx.exception)) + + def test_invalid_duration_raises(self): + with self.assertRaises(ValueError): + parse_history_window("badvalue", None, None) + + def test_invalid_start_time_raises(self): + with self.assertRaises(ValueError): + parse_history_window(None, "not-a-date", "2026-05-25T09:00:00", resiliency_score_flag=True) + + +class TestApplyHistoricalResiliency(unittest.TestCase): + + def _make_window(self): + start = datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc) + end = datetime.datetime(2026, 5, 25, 9, 0, 0, tzinfo=datetime.timezone.utc) + return HistoryWindow(start=start, end=end, label="1h") + + def test_raises_when_prometheus_is_none(self): + resiliency_obj = MagicMock() + telemetry = MagicMock() + with self.assertRaises(RuntimeError) as ctx: + apply_historical_resiliency(self._make_window(), resiliency_obj, None, telemetry) + self.assertIn("Prometheus", str(ctx.exception)) + + def test_raises_when_resiliency_obj_is_none(self): + prometheus = MagicMock() + telemetry = MagicMock() + with self.assertRaises(RuntimeError) as ctx: + apply_historical_resiliency(self._make_window(), None, prometheus, telemetry) + self.assertIn("Prometheus", str(ctx.exception)) + + def test_populates_overall_resiliency_report(self): + window = self._make_window() + resiliency_obj = MagicMock() + resiliency_obj.scenario_reports = [ + {"score": 85, "breakdown": {"passed": 20, "failed": 5}} + ] + prometheus = MagicMock() + telemetry = MagicMock() + + apply_historical_resiliency(window, resiliency_obj, prometheus, telemetry) + + resiliency_obj.add_scenario_report.assert_called_once_with( + scenario_name="historical", + prom_cli=prometheus, + start_time=window.start, + end_time=window.end, + ) + self.assertIsNotNone(telemetry.overall_resiliency_report) + report = telemetry.overall_resiliency_report + self.assertEqual(report.resiliency_score, 85) + self.assertEqual(report.passed_slos, 20) + self.assertEqual(report.total_slos, 25) + + def test_uses_last_scenario_report(self): + window = self._make_window() + resiliency_obj = MagicMock() + resiliency_obj.scenario_reports = [ + {"score": 50, "breakdown": {"passed": 10, "failed": 10}}, + {"score": 90, "breakdown": {"passed": 18, "failed": 2}}, + ] + prometheus = MagicMock() + telemetry = MagicMock() + + apply_historical_resiliency(window, resiliency_obj, prometheus, telemetry) + + self.assertEqual(telemetry.overall_resiliency_report.resiliency_score, 90) + + +if __name__ == "__main__": + unittest.main()