From bf59ed3c9cee6bc024ffb57d949903a15ebf8498 Mon Sep 17 00:00:00 2001 From: harshjainnn Date: Wed, 1 Jul 2026 17:40:35 +0530 Subject: [PATCH] feat(tests): migrate pod error scenario tests to v2 framework (#1440) * feat(tests): migrate pod error scenario tests to v2 framework Signed-off-by: Harsh Jain * chore: drop unrelated file from PR Signed-off-by: Harsh Jain * refactor: rename assert_no_hang_and_correct_logs to assert_failure_logs_contain Signed-off-by: Harsh Jain * docs: reference pod_disruption_scenario_plugin.py:234 in test comment Signed-off-by: Harsh Jain --------- Signed-off-by: Harsh Jain --- CI/tests_v2/lib/utils.py | 1 + CI/tests_v2/pytest.ini | 1 + .../pod_error_scenarios/resource.yaml | 19 +++ .../pod_error_scenarios/scenario_base.yaml | 6 + .../test_pod_error_scenarios.py | 129 ++++++++++++++++++ 5 files changed, 156 insertions(+) create mode 100644 CI/tests_v2/scenarios/pod_error_scenarios/resource.yaml create mode 100644 CI/tests_v2/scenarios/pod_error_scenarios/scenario_base.yaml create mode 100644 CI/tests_v2/scenarios/pod_error_scenarios/test_pod_error_scenarios.py diff --git a/CI/tests_v2/lib/utils.py b/CI/tests_v2/lib/utils.py index 5c3d701c..1a9edba7 100644 --- a/CI/tests_v2/lib/utils.py +++ b/CI/tests_v2/lib/utils.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) # no-op (e.g. a selector matched nothing) and the happy-path test should fail. SCENARIO_EXECUTION_MARKERS = { "pod_disruption": r"Deleting pod |waiting up to .* seconds for pod recovery", + "pod_error_scenarios": r"Deleting pod |waiting up to .* seconds for pod recovery", "application_outage": r"Creating the network policy|Deleting the network policy", "storage_throttle": r"Setting io\.max|Verified blkio settings|Privileged pod deployed", "namespace_deletion": r"Delete objects in selected namespace|Deleted all objects in namespace", diff --git a/CI/tests_v2/pytest.ini b/CI/tests_v2/pytest.ini index 8dede3fe..cc54c4dd 100644 --- a/CI/tests_v2/pytest.ini +++ b/CI/tests_v2/pytest.ini @@ -8,6 +8,7 @@ addopts = -v markers = functional: marks a test as a functional test (deselect with '-m "not functional"') pod_disruption: marks a test as a pod disruption scenario test + pod_error_scenarios: marks a test as a pod error/failure-mode scenario test application_outage: marks a test as an application outage scenario test storage_throttle: marks a test as a storage throttle scenario test cpu_hog: marks a test as a CPU hog scenario test diff --git a/CI/tests_v2/scenarios/pod_error_scenarios/resource.yaml b/CI/tests_v2/scenarios/pod_error_scenarios/resource.yaml new file mode 100644 index 00000000..05aa76c0 --- /dev/null +++ b/CI/tests_v2/scenarios/pod_error_scenarios/resource.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: krkn-pod-error-target +spec: + replicas: 2 + selector: + matchLabels: + app: krkn-pod-error-target + template: + metadata: + labels: + app: krkn-pod-error-target + spec: + containers: + - name: app + image: nginx:alpine + ports: + - containerPort: 80 diff --git a/CI/tests_v2/scenarios/pod_error_scenarios/scenario_base.yaml b/CI/tests_v2/scenarios/pod_error_scenarios/scenario_base.yaml new file mode 100644 index 00000000..d7b37782 --- /dev/null +++ b/CI/tests_v2/scenarios/pod_error_scenarios/scenario_base.yaml @@ -0,0 +1,6 @@ +- id: kill-pods + config: + namespace_pattern: "^default$" + label_selector: app=krkn-pod-error-target + krkn_pod_recovery_time: 120 + kill: 1 diff --git a/CI/tests_v2/scenarios/pod_error_scenarios/test_pod_error_scenarios.py b/CI/tests_v2/scenarios/pod_error_scenarios/test_pod_error_scenarios.py new file mode 100644 index 00000000..c6afcb13 --- /dev/null +++ b/CI/tests_v2/scenarios/pod_error_scenarios/test_pod_error_scenarios.py @@ -0,0 +1,129 @@ +""" +Functional test for pod error / failure-mode coverage. +Migrated from CI/tests/test_pod_error.sh with expected-failure semantics added. +""" + +import pytest + +from lib.base import BaseScenarioTest +from lib.utils import ( + assert_kraken_failure, + assert_kraken_success, + assert_scenario_executed, + get_pods_list, + pod_uids, +) + + +@pytest.mark.functional +@pytest.mark.pod_error_scenarios +class TestPodErrorScenarios(BaseScenarioTest): + WORKLOAD_MANIFEST = "CI/tests_v2/scenarios/pod_error_scenarios/resource.yaml" + WORKLOAD_IS_PATH = True + LABEL_SELECTOR = "app=krkn-pod-error-target" + SCENARIO_NAME = "pod_error_scenarios" + SCENARIO_TYPE = "pod_disruption_scenarios" + NAMESPACE_KEY_PATH = [0, "config", "namespace_pattern"] + NAMESPACE_IS_REGEX = True + OVERRIDES_KEY_PATH = [0, "config"] + + # -- Helper assertions ------------------------------------------- + + def assert_failure_logs_contain(self, result, ns, expected_reasons, expected_keywords=None): + # Timeouts / hangs are caught at the execution level by run_kraken (raises + # TimeoutExpired) or pytest timeout budgets. Reaching here guarantees the run completed. + + combined = (result.stdout or "") + "\n" + (result.stderr or "") + combined_lower = combined.lower() + + # Assert namespace is in the logs + assert ns.lower() in combined_lower, f"Expected namespace '{ns}' not found in logs" + + # Assert failure reasons are in the logs + for reason in expected_reasons: + assert reason.lower() in combined_lower, f"Expected failure reason '{reason}' not found in logs" + + if expected_keywords: + for kw in expected_keywords: + assert kw.lower() in combined_lower, f"Expected log keyword '{kw}' not found in logs" + + # -- Happy path -------------------------------------------------- + + @pytest.mark.order(1) + def test_kill_one_pod_recovers(self, wait_for_pods_running): + ns = self.ns + result = self.run_scenario(self.tmp_path, ns) + assert_kraken_success(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + assert_scenario_executed(result, self.SCENARIO_NAME, context=f"namespace={ns}", tmp_path=self.tmp_path) + wait_for_pods_running(ns, self.LABEL_SELECTOR, timeout=90) + + @pytest.mark.order(2) + def test_kill_multiple_pods(self, wait_for_pods_running): + ns = self.ns + before = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR) + before_uids = set(pod_uids(before)) + + result = self.run_scenario(self.tmp_path, ns, overrides={"kill": 2}) + assert_kraken_success(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + assert_scenario_executed(result, self.SCENARIO_NAME, context=f"namespace={ns}", tmp_path=self.tmp_path) + + wait_for_pods_running(ns, self.LABEL_SELECTOR, timeout=90) + after = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR) + after_uids = set(pod_uids(after)) + assert before_uids.isdisjoint(after_uids), ( + f"Expected both pods replaced (namespace={ns}). before={before_uids} after={after_uids}" + ) + + # -- Negative / failure-mode -------------------------------------- + + def test_excessive_kill_count_fails(self): + ns = self.ns + result = self.run_scenario(self.tmp_path, ns, overrides={"kill": 100}) + assert_kraken_failure(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + # Expected error text must match the message raised at + # pod_disruption_scenario_plugin.py:234. Update both together + # if that message wording changes. + self.assert_failure_logs_contain( + result, ns, expected_reasons=["not enough pods match", "expected 100", "found only 2 pods"] + ) + + def test_recovery_timeout_fails(self): + ns = self.ns + before = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR) + before_names = [p.metadata.name for p in before.items] + + result = self.run_scenario( + self.tmp_path, ns, overrides={"krkn_pod_recovery_time": 1} + ) + assert_kraken_failure(result, context=f"namespace={ns}", tmp_path=self.tmp_path) + + # Verify no hang and logs contain namespace and expected timeout/recovery errors + self.assert_failure_logs_contain( + result, ns, expected_reasons=["timeout", "recover"] + ) + + # Verify at least one target pod name is in the logs + combined_lower = ((result.stdout or "") + "\n" + (result.stderr or "")).lower() + found_pod = any(name.lower() in combined_lower for name in before_names) + assert found_pod, f"None of the target pods {before_names} were found in failure logs" + + @pytest.mark.no_workload + def test_invalid_namespace_pattern_fails(self): + scenario = self.load_and_patch_scenario(self.repo_root, "nonexistent-ns-xyz") + scenario_path = self.write_scenario(self.tmp_path, scenario, suffix="_bad_ns") + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="pod_error_bad_ns_config.yaml" + ) + result = self.run_kraken(config_path) + assert_kraken_failure(result, context="invalid namespace pattern", tmp_path=self.tmp_path) + self.assert_failure_logs_contain( + result, "nonexistent-ns-xyz", expected_reasons=["not enough pods match", "expected 1", "found only 0 pods"] + ) + + def test_zero_pods_matching_label_fails(self): + ns = self.ns + result = self.run_scenario(self.tmp_path, ns, overrides={"label_selector": "app=nonexistent"}) + assert_kraken_failure(result, context=f"namespace={ns}, label mismatch", tmp_path=self.tmp_path) + self.assert_failure_logs_contain( + result, ns, expected_reasons=["not enough pods match", "expected 1", "found only 0 pods"] + )