github/krkn
1
0
Fork 0
mirror of https://github.com/krkn-chaos/krkn.git synced 2026-08-25 09:27:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding resiliency score (#1364)

Browse Source 
Assisted By: Claude Code:

Signed-off-by: Paige Patton <prubenda@redhat.com>
This commit is contained in:
Paige Patton
2026-07-28 16:50:08 -04:00
committed by GitHub
parent 3bec95ae1c
commit 5017fbb47d
4 changed files with 536 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
krkn/resiliency/__init__.py
+5
View File
@@ -16,3 +16,8 @@
from .resiliency import Resiliency # noqa: F401
from .score import calculate_resiliency_score # noqa: F401
from .history import ( # noqa: F401
HistoryWindow,
parse_history_window,
apply_historical_resiliency,
)
krkn/resiliency/history.py
+204
View File
@@ -0,0 +1,204 @@
# Copyright 2025 The Krkn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Historical resiliency-score queries.
Provides helpers to parse a CLI time-window (duration string or explicit
start/end timestamps), query Prometheus over that window, and populate the
overall_resiliency_report field of a ChaosRunTelemetry object.
"""
from __future__ import annotations
import datetime
import logging
from dataclasses import dataclass
from typing import Optional
from krkn_lib.models.k8s import ResiliencyReport
from krkn_lib.models.telemetry import ChaosRunTelemetry
from krkn_lib.prometheus.krkn_prometheus import KrknPrometheus
from krkn.resiliency.resiliency import Resiliency
# ---------------------------------------------------------------------------
# Parsing helpers
# ---------------------------------------------------------------------------
def parse_duration(duration_str: str) -> datetime.timedelta:
"""Parse a human-friendly duration string into a timedelta.
Supported units: s (seconds), m (minutes), h (hours), d (days), w (weeks).
Examples: '30s', '5m', '24h', '7d', '2w'
"""
units = {
's': 'seconds',
'm': 'minutes',
'h': 'hours',
'd': 'days',
'w': 'weeks',
}
s = duration_str.strip().lower()
if len(s) < 2:
raise ValueError(
f"Invalid duration '{duration_str}': expected a number followed by a unit (s, m, h, d, w)"
)
unit = s[-1]
if unit not in units:
raise ValueError(
f"Unknown duration unit '{unit}' in '{duration_str}'. Supported units: s, m, h, d, w"
)
try:
value = float(s[:-1])
except ValueError:
raise ValueError(f"Invalid numeric value '{s[:-1]}' in '{duration_str}'")
if value <= 0:
raise ValueError(f"Duration must be positive, got '{duration_str}'")
return datetime.timedelta(**{units[unit]: value})
def parse_datetime(dt_str: str) -> datetime.datetime:
"""Parse a datetime string into a UTC-aware datetime object.
The input is always interpreted as UTC regardless of the host timezone.
Supported formats:
YYYY-MM-DDTHH:MM:SS (ISO 8601)
YYYY-MM-DD HH:MM:SS
YYYY-MM-DD (midnight assumed)
"""
for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
try:
naive = datetime.datetime.strptime(dt_str.strip(), fmt)
return naive.replace(tzinfo=datetime.timezone.utc)
except ValueError:
continue
raise ValueError(
f"Cannot parse datetime '{dt_str}'. "
"Use ISO 8601 format: YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD (times are UTC)"
)
# ---------------------------------------------------------------------------
# Window representation
# ---------------------------------------------------------------------------
@dataclass
class HistoryWindow:
"""A resolved start/end time window for a historical resiliency query."""
start: datetime.datetime
end: datetime.datetime
label: str
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def parse_history_window(
past_resiliency_score: Optional[str],
hist_start_str: Optional[str],
hist_end_str: Optional[str],
resiliency_score_flag: bool = False,
) -> Optional[HistoryWindow]:
"""Parse and validate historical resiliency CLI options.
Returns a :class:`HistoryWindow` when any window option was provided, or
``None`` when none were supplied (normal chaos run).
Args:
past_resiliency_score: Trailing duration string (e.g. '1h', '24h').
hist_start_str: Explicit window start datetime string.
hist_end_str: Explicit window end datetime string.
resiliency_score_flag: Must be ``True`` when using ``--start-time``/
``--end-time``; set by the ``--resiliency-score`` flag or the
``resiliency-score`` command.
Raises:
ValueError: with a human-readable message when options are invalid.
"""
if past_resiliency_score is not None and (hist_start_str or hist_end_str):
raise ValueError(
"--past-resiliency-score and --start-time/--end-time are mutually exclusive"
)
if (hist_start_str or hist_end_str) and not resiliency_score_flag:
raise ValueError(
"--start-time/--end-time require the --resiliency-score flag "
"or the resiliency-score command"
)
if past_resiliency_score is not None:
duration = parse_duration(past_resiliency_score)
end = datetime.datetime.now(datetime.timezone.utc)
start = end - duration
return HistoryWindow(start=start, end=end, label=past_resiliency_score)
if hist_start_str or hist_end_str:
if not hist_start_str or not hist_end_str:
raise ValueError("--start-time and --end-time must both be provided together")
start = parse_datetime(hist_start_str)
end = parse_datetime(hist_end_str)
if end <= start:
raise ValueError("--end-time must be after --start-time")
return HistoryWindow(start=start, end=end, label=f"{start} → {end}")
return None
def apply_historical_resiliency(
window: HistoryWindow,
resiliency_obj: Resiliency,
prometheus: KrknPrometheus,
telemetry: ChaosRunTelemetry,
) -> None:
"""Query Prometheus over *window* and populate ``telemetry.overall_resiliency_report``.
Raises:
RuntimeError: when Prometheus or the resiliency object is unavailable.
"""
if resiliency_obj is None or prometheus is None:
raise RuntimeError(
"Prometheus is required for historical resiliency scoring but is not available. "
"Ensure prometheus_url is set in config and Prometheus is reachable."
)
logging.info(
"Querying historical resiliency score for window %s → %s", window.start, window.end
)
resiliency_obj.add_scenario_report(
scenario_name="historical",
prom_cli=prometheus,
start_time=window.start,
end_time=window.end,
)
hist_report = resiliency_obj.scenario_reports[-1]
hist_score = hist_report["score"]
hist_breakdown = hist_report.get("breakdown", {})
passed = hist_breakdown.get("passed", 0)
total = passed + hist_breakdown.get("failed", 0)
history_summary = {
"scenarios": {"historical": hist_score},
"resiliency_score": hist_score,
"passed_slos": passed,
"total_slos": total,
}
telemetry.overall_resiliency_report = ResiliencyReport(
json_object=history_summary,
resiliency_score=hist_score,
passed_slos=passed,
total_slos=total,
)
logging.info("Historical resiliency score (%s): %d", window.label, hist_score)
run_kraken.py
+84 -4
View File
@@ -46,6 +46,11 @@ import server as server
from krkn.resiliency.resiliency import (
Resiliency
)
from krkn.resiliency.history import (
HistoryWindow,
parse_history_window,
apply_historical_resiliency,
)
from krkn_lib.k8s import KrknKubernetes
from krkn_lib.ocp import KrknOpenshift
from krkn_lib.telemetry.k8s import KrknTelemetryKubernetes
@@ -74,6 +79,7 @@ warnings.filterwarnings(action='ignore', module='.*paramiko.*')
report_file = ""
# Main function
def main(options, command: Optional[str]) -> int:
# Start kraken
@@ -131,6 +137,24 @@ def main(options, command: Optional[str]) -> int:
if run_mode not in valid_run_modes:
logging.warning("Unknown resiliency_run_mode '%s'. Defaulting to 'standalone'", run_mode)
run_mode = "standalone"
try:
hist_window = parse_history_window(
getattr(options, "past_resiliency_score", None),
getattr(options, "hist_start_time", None),
getattr(options, "hist_end_time", None),
resiliency_score_flag=getattr(options, "resiliency_score", False) or command == "resiliency-score",
)
except ValueError as exc:
logging.error("%s", exc)
return -1
if hist_window is not None:
logging.info(
"Historical resiliency window '%s' provided. Chaos scenarios will not be executed.",
hist_window.label,
)
chaos_scenarios = []
wait_duration = get_yaml_item_value(config["tunings"], "wait_duration", 60)
iterations = get_yaml_item_value(config["tunings"], "iterations", 1)
daemon_mode = get_yaml_item_value(config["tunings"], "daemon_mode", False)
@@ -341,6 +365,15 @@ def main(options, command: Optional[str]) -> int:
telemetry_ocp, options.run_uuid, options.scenario_type
)
)
elif command == "resiliency-score":
if hist_window is None:
logging.error(
"resiliency-score command requires a time window: "
"use --past-resiliency-score <duration> (e.g. 24h) "
"or --start-time/--end-time for an explicit range"
)
sys.exit(-1)
chaos_scenarios = []
# Initialize the start iteration to 0
iteration = 0
@@ -576,13 +609,13 @@ def main(options, command: Optional[str]) -> int:
else:
logging.info("No error logs collected during chaos run")
chaos_telemetry.error_logs = []
if resiliency_obj:
if resiliency_obj and hist_window is None:
try:
resiliency_obj.attach_compact_to_telemetry(chaos_telemetry)
except Exception as exc:
logging.error("Failed to embed per-scenario resiliency in telemetry: %s", exc)
if resiliency_obj:
if resiliency_obj and hist_window is None:
try:
resiliency_obj.finalize_and_save(
prom_cli=prometheus,
@@ -617,7 +650,16 @@ def main(options, command: Optional[str]) -> int:
telemetry_json = chaos_telemetry.to_json()
decoded_chaos_run_telemetry = ChaosRunTelemetry(json.loads(telemetry_json))
if resiliency_obj and hasattr(resiliency_obj, "summary") and resiliency_obj.summary is not None:
if hist_window is not None:
try:
apply_historical_resiliency(hist_window, resiliency_obj, prometheus, decoded_chaos_run_telemetry)
except RuntimeError as exc:
logging.error("%s", exc)
return -1
except Exception as exc:
logging.error("Failed to compute historical resiliency score: %s", exc)
return -1
elif resiliency_obj and hasattr(resiliency_obj, "summary") and resiliency_obj.summary is not None:
summary_dict = resiliency_obj.get_summary()
decoded_chaos_run_telemetry.overall_resiliency_report = ResiliencyReport(
json_object=summary_dict,
@@ -767,7 +809,9 @@ if __name__ == "__main__":
usage="%prog [options] [command]\n\n"
"Commands:\n"
" list-rollback List rollback version files in a tree-like format\n"
" execute-rollback Execute rollback version files and cleanup if successful\n\n"
" execute-rollback Execute rollback version files and cleanup if successful\n"
" resiliency-score Query historical resiliency score without running chaos scenarios.\n"
" Requires --start-time/--end-time or --past-resiliency-score.\n\n"
"If no command is specified, kraken will run chaos scenarios.",
)
parser.add_option(
@@ -831,6 +875,42 @@ if __name__ == "__main__":
default=False,
)
parser.add_option(
"--past-resiliency-score",
dest="past_resiliency_score",
help="Query historical resiliency score over a trailing window (e.g. 1h, 24h, 7d) "
"without running chaos scenarios. Mutually exclusive with --start-time/--end-time.",
default=None,
)
parser.add_option(
"--resiliency-score",
dest="resiliency_score",
action="store_true",
help="Indicate that --start-time/--end-time define a historical resiliency score query. "
"Required when using --start-time/--end-time. "
"Implied automatically by the resiliency-score command.",
default=False,
)
parser.add_option(
"--start-time",
dest="hist_start_time",
help="Start of explicit historical resiliency window (YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD, UTC). "
"Must be used together with --end-time and --resiliency-score. "
"Mutually exclusive with --past-resiliency-score.",
default=None,
)
parser.add_option(
"--end-time",
dest="hist_end_time",
help="End of explicit historical resiliency window (YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD, UTC). "
"Must be used together with --start-time and --resiliency-score. "
"Mutually exclusive with --past-resiliency-score.",
default=None,
)
(options, args) = parser.parse_args()
# If no command or regular execution, continue with existing logic
tests/test_resiliency_history.py
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
"""
Unit tests for krkn.resiliency.history.
Usage:
python -m unittest tests/test_resiliency_history.py -v
"""
import datetime
import sys
import os
import unittest
from unittest.mock import MagicMock, patch
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from krkn.resiliency.history import (
HistoryWindow,
apply_historical_resiliency,
parse_datetime,
parse_duration,
parse_history_window,
)
class TestParseDuration(unittest.TestCase):
def test_minutes(self):
self.assertEqual(parse_duration("5m"), datetime.timedelta(minutes=5))
def test_hours(self):
self.assertEqual(parse_duration("24h"), datetime.timedelta(hours=24))
def test_days(self):
self.assertEqual(parse_duration("7d"), datetime.timedelta(days=7))
def test_seconds(self):
self.assertEqual(parse_duration("30s"), datetime.timedelta(seconds=30))
def test_weeks(self):
self.assertEqual(parse_duration("2w"), datetime.timedelta(weeks=2))
def test_uppercase_unit(self):
self.assertEqual(parse_duration("1H"), datetime.timedelta(hours=1))
def test_fractional_value(self):
self.assertEqual(parse_duration("1.5h"), datetime.timedelta(hours=1.5))
def test_unknown_unit_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_duration("5x")
self.assertIn("Unknown duration unit", str(ctx.exception))
def test_zero_value_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_duration("0h")
self.assertIn("positive", str(ctx.exception))
def test_negative_value_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_duration("-1h")
self.assertIn("positive", str(ctx.exception))
def test_non_numeric_value_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_duration("abch")
self.assertIn("Invalid numeric value", str(ctx.exception))
def test_too_short_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_duration("h")
self.assertIn("expected a number", str(ctx.exception))
class TestParseDatetime(unittest.TestCase):
def test_iso_with_time(self):
result = parse_datetime("2026-05-25T08:00:00")
self.assertEqual(result, datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc))
def test_space_separated(self):
result = parse_datetime("2026-05-25 08:00:00")
self.assertEqual(result, datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc))
def test_date_only(self):
result = parse_datetime("2026-05-25")
self.assertEqual(result, datetime.datetime(2026, 5, 25, 0, 0, 0, tzinfo=datetime.timezone.utc))
def test_leading_trailing_whitespace(self):
result = parse_datetime(" 2026-05-25T10:30:00 ")
self.assertEqual(result, datetime.datetime(2026, 5, 25, 10, 30, 0, tzinfo=datetime.timezone.utc))
def test_invalid_format_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_datetime("25/05/2026")
self.assertIn("Cannot parse datetime", str(ctx.exception))
def test_invalid_date_raises(self):
with self.assertRaises(ValueError):
parse_datetime("2026-13-01")
class TestParseHistoryWindow(unittest.TestCase):
def test_no_options_returns_none(self):
self.assertIsNone(parse_history_window(None, None, None))
def test_duration_mode(self):
window = parse_history_window("1h", None, None)
self.assertIsNotNone(window)
self.assertEqual(window.label, "1h")
expected_delta = datetime.timedelta(hours=1)
self.assertAlmostEqual(
(window.end - window.start).total_seconds(),
expected_delta.total_seconds(),
delta=2,
)
def test_explicit_range_requires_flag(self):
with self.assertRaises(ValueError) as ctx:
parse_history_window(None, "2026-05-25T08:00:00", "2026-05-25T09:00:00")
self.assertIn("--resiliency-score", str(ctx.exception))
def test_explicit_range_with_flag(self):
window = parse_history_window(
None, "2026-05-25T08:00:00", "2026-05-25T09:00:00",
resiliency_score_flag=True,
)
self.assertIsNotNone(window)
self.assertEqual(window.start, datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc))
self.assertEqual(window.end, datetime.datetime(2026, 5, 25, 9, 0, 0, tzinfo=datetime.timezone.utc))
def test_explicit_range_label(self):
window = parse_history_window(
None, "2026-05-25", "2026-05-26", resiliency_score_flag=True,
)
self.assertIn("2026-05-25", window.label)
self.assertIn("2026-05-26", window.label)
def test_mutual_exclusivity_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_history_window(
"1h", "2026-05-25T08:00:00", "2026-05-25T09:00:00",
resiliency_score_flag=True,
)
self.assertIn("mutually exclusive", str(ctx.exception))
def test_missing_end_time_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_history_window(None, "2026-05-25T08:00:00", None, resiliency_score_flag=True)
self.assertIn("both be provided", str(ctx.exception))
def test_missing_start_time_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_history_window(None, None, "2026-05-25T09:00:00", resiliency_score_flag=True)
self.assertIn("both be provided", str(ctx.exception))
def test_end_before_start_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_history_window(
None, "2026-05-25T09:00:00", "2026-05-25T08:00:00",
resiliency_score_flag=True,
)
self.assertIn("after --start-time", str(ctx.exception))
def test_end_equal_start_raises(self):
with self.assertRaises(ValueError) as ctx:
parse_history_window(
None, "2026-05-25T08:00:00", "2026-05-25T08:00:00",
resiliency_score_flag=True,
)
self.assertIn("after --start-time", str(ctx.exception))
def test_invalid_duration_raises(self):
with self.assertRaises(ValueError):
parse_history_window("badvalue", None, None)
def test_invalid_start_time_raises(self):
with self.assertRaises(ValueError):
parse_history_window(None, "not-a-date", "2026-05-25T09:00:00", resiliency_score_flag=True)
class TestApplyHistoricalResiliency(unittest.TestCase):
def _make_window(self):
start = datetime.datetime(2026, 5, 25, 8, 0, 0, tzinfo=datetime.timezone.utc)
end = datetime.datetime(2026, 5, 25, 9, 0, 0, tzinfo=datetime.timezone.utc)
return HistoryWindow(start=start, end=end, label="1h")
def test_raises_when_prometheus_is_none(self):
resiliency_obj = MagicMock()
telemetry = MagicMock()
with self.assertRaises(RuntimeError) as ctx:
apply_historical_resiliency(self._make_window(), resiliency_obj, None, telemetry)
self.assertIn("Prometheus", str(ctx.exception))
def test_raises_when_resiliency_obj_is_none(self):
prometheus = MagicMock()
telemetry = MagicMock()
with self.assertRaises(RuntimeError) as ctx:
apply_historical_resiliency(self._make_window(), None, prometheus, telemetry)
self.assertIn("Prometheus", str(ctx.exception))
def test_populates_overall_resiliency_report(self):
window = self._make_window()
resiliency_obj = MagicMock()
resiliency_obj.scenario_reports = [
{"score": 85, "breakdown": {"passed": 20, "failed": 5}}
]
prometheus = MagicMock()
telemetry = MagicMock()
apply_historical_resiliency(window, resiliency_obj, prometheus, telemetry)
resiliency_obj.add_scenario_report.assert_called_once_with(
scenario_name="historical",
prom_cli=prometheus,
start_time=window.start,
end_time=window.end,
)
self.assertIsNotNone(telemetry.overall_resiliency_report)
report = telemetry.overall_resiliency_report
self.assertEqual(report.resiliency_score, 85)
self.assertEqual(report.passed_slos, 20)
self.assertEqual(report.total_slos, 25)
def test_uses_last_scenario_report(self):
window = self._make_window()
resiliency_obj = MagicMock()
resiliency_obj.scenario_reports = [
{"score": 50, "breakdown": {"passed": 10, "failed": 10}},
{"score": 90, "breakdown": {"passed": 18, "failed": 2}},
]
prometheus = MagicMock()
telemetry = MagicMock()
apply_historical_resiliency(window, resiliency_obj, prometheus, telemetry)
self.assertEqual(telemetry.overall_resiliency_report.resiliency_score, 90)
if __name__ == "__main__":
unittest.main()