From 3a77f487dac74d6abb796840bc98488eb9a996d6 Mon Sep 17 00:00:00 2001 From: "augmentcode[bot]" <185243770+augmentcode[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 21:37:52 +0530 Subject: [PATCH] test: migrate cpu_hog functional test to tests_v2 (closes #1390) (#1391) * test: migrate cpu_hog functional test to tests_v2 (closes #1390) Migrate the legacy CI/tests/test_cpu_hog.sh to the v2 pytest framework under CI/tests_v2/scenarios/cpu_hog/, preserving parity with the legacy flow and adding stronger functional and negative coverage. - Add scenario_base.yaml (flat hog_scenarios config, patched per test) - Add test_cpu_hog.py with TestCpuHog(BaseScenarioTest): - success: hog pod created on node-selector target, run exits 0, pod cleaned up - negative: node-selector matching zero nodes fails gracefully - negative: missing mandatory hog-type fails at config parsing - Register cpu_hog marker in pytest.ini - Document cpu_hog coverage in README.md * test: kill background Kraken proc on any cpu_hog success-test failure Broaden the success-path teardown so a poll/assert failure before proc.communicate() also kills the background Kraken process, preventing a lingering cpu-hog- pod from stressing the node and racing --reruns. Addresses Deep Code Review feedback on PR #1391. --------- Signed-off-by: augmentcode[bot] <185243770+augmentcode[bot]@users.noreply.github.com> Co-authored-by: augmentcode[bot] <185243770+augmentcode[bot]@users.noreply.github.com> --- CI/tests_v2/README.md | 16 +- CI/tests_v2/pytest.ini | 1 + .../scenarios/cpu_hog/scenario_base.yaml | 15 ++ CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py | 184 ++++++++++++++++++ 4 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 CI/tests_v2/scenarios/cpu_hog/scenario_base.yaml create mode 100644 CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py diff --git a/CI/tests_v2/README.md b/CI/tests_v2/README.md index 14659d27..31f3f5c9 100644 --- a/CI/tests_v2/README.md +++ b/CI/tests_v2/README.md @@ -1,6 +1,6 @@ # Pytest Functional Tests (tests_v2) -This directory contains a pytest-based functional test framework that runs **alongside** the existing bash tests in `CI/tests/`. It covers the **pod disruption** and **application outage** scenarios with proper assertions, retries, and reporting. +This directory contains a pytest-based functional test framework that runs **alongside** the existing bash tests in `CI/tests/`. It covers the **pod disruption**, **application outage**, **storage throttle**, and **CPU hog** scenarios with proper assertions, retries, and reporting. Each test runs in its **own ephemeral Kubernetes namespace** (`krkn-test-`). Before the test, the framework creates the namespace, deploys the target workload, and waits for pods to be ready. After the test, the namespace is deleted (cascading all resources). **You do not need to deploy any workloads manually.** @@ -121,6 +121,12 @@ pytest CI/tests_v2/ -v -m pod_disruption pytest CI/tests_v2/ -v -m application_outage ``` +### Run only CPU hog tests + +```bash +pytest CI/tests_v2/ -v -m cpu_hog +``` + ### Run with verbose output and no capture ```bash @@ -174,9 +180,15 @@ Each test runs in an isolated ephemeral namespace; workloads are deployed automa - **test_invalid_scenario_fails**: Invalid scenario file (missing `application_outage` key) causes Kraken to exit non-zero. - **test_bad_namespace_fails**: Scenario targeting a non-existent namespace causes Kraken to exit non-zero. +- **scenarios/cpu_hog/** + CPU hog scenario (`hog_scenarios`), migrated from the legacy `CI/tests/test_cpu_hog.sh`. CPU hog targets nodes (not workloads): Kraken deploys a short-lived hog pod (name prefix `cpu-hog-`) onto each selected node, runs `stress-ng` for the configured duration, then deletes the pod. Tests use `@pytest.mark.no_workload` (no app deployment needed); `scenario_base.yaml` is a flat hog config patched per test. Tests include: + - **test_cpu_hog_success_lifecycle_and_targeting**: Happy path — a hog pod is created on the `node-selector` target, the run exits 0, and the pod is cleaned up afterward. + - **test_cpu_hog_invalid_selector_fails**: A `node-selector` matching zero nodes causes Kraken to exit non-zero (no available nodes to schedule). + - **test_cpu_hog_invalid_config_fails**: Omitting the mandatory `hog-type` field causes Kraken to exit non-zero at config parsing. + ## Configuration -- **pytest.ini**: Markers (`functional`, `pod_disruption`, `application_outage`, `no_workload`). Use `--timeout=300`, `--reruns=2`, `--reruns-delay=10` on the command line for full runs. +- **pytest.ini**: Markers (`functional`, `pod_disruption`, `application_outage`, `storage_throttle`, `cpu_hog`, `no_workload`). Use `--timeout=300`, `--reruns=2`, `--reruns-delay=10` on the command line for full runs. - **conftest.py**: Re-exports fixtures from `lib/k8s.py`, `lib/namespace.py`, `lib/deploy.py`, `lib/kraken.py` (e.g. `test_namespace`, `deploy_workload`, `k8s_core`, `wait_for_pods_running`, `run_kraken`, `build_config`). Configs are built from `CI/tests_v2/config/common_test_config.yaml` with monitoring disabled for local runs. Timeout constants in `lib/base.py` can be overridden via env vars. - **Cluster access**: Reads and applies use the Kubernetes Python client; `kubectl` is still used for `port-forward` and for running Kraken. - **utils.py**: Pod/network policy helpers and assertion helpers (`assert_all_pods_running_and_ready`, `assert_pod_count_unchanged`, `assert_kraken_success`, `assert_kraken_failure`, `patch_namespace_in_docs`). diff --git a/CI/tests_v2/pytest.ini b/CI/tests_v2/pytest.ini index db6bc9b6..f4557725 100644 --- a/CI/tests_v2/pytest.ini +++ b/CI/tests_v2/pytest.ini @@ -10,6 +10,7 @@ markers = pod_disruption: marks a test as a pod disruption scenario test application_outage: marks a test as an application outage scenario test storage_throttle: marks a test as a storage throttle scenario test + cpu_hog: marks a test as a CPU hog scenario test no_workload: skip workload deployment for this test (e.g. negative tests) order: set test order (pytest-order) junit_family = xunit2 diff --git a/CI/tests_v2/scenarios/cpu_hog/scenario_base.yaml b/CI/tests_v2/scenarios/cpu_hog/scenario_base.yaml new file mode 100644 index 00000000..5f48dc40 --- /dev/null +++ b/CI/tests_v2/scenarios/cpu_hog/scenario_base.yaml @@ -0,0 +1,15 @@ +# Base CPU hog scenario (hog_scenarios). Tests load this and patch namespace, node-selector, +# number-of-nodes, duration, etc. The CPU hog targets nodes (not workloads): Krkn deploys a +# short-lived hog pod (name prefix "cpu-hog-") on each selected node, runs for `duration` +# seconds, then deletes the pod. Values here are tuned for functional testing (light load, +# short duration), not performance benchmarking. +duration: 20 +workers: 1 +hog-type: cpu +image: quay.io/krkn-chaos/krkn-hog +namespace: default +cpu-load-percentage: 50 +cpu-method: all +node-selector: "node-role.kubernetes.io/worker=" +number-of-nodes: 1 +taints: [] diff --git a/CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py b/CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py new file mode 100644 index 00000000..6ea6d2be --- /dev/null +++ b/CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py @@ -0,0 +1,184 @@ +""" +Functional tests for the CPU hog scenario (hog_scenarios), migrated from the legacy +CI/tests/test_cpu_hog.sh. + +CPU hog targets nodes (not workloads): Krkn deploys a short-lived hog pod (name prefix +"cpu-hog-") onto each selected node, runs it for the configured duration, then deletes the +pod. These tests therefore use @pytest.mark.no_workload (no app deployment is needed) and +verify execution success, node-selector targeting, duration/lifecycle, cleanup, and graceful +failure on invalid selector/config. +""" + +import logging +import subprocess +import time + +import pytest + +from lib.base import BaseScenarioTest +from lib.utils import assert_kraken_failure, assert_kraken_success + +logger = logging.getLogger(__name__) + +HOG_POD_PREFIX = "cpu-hog-" +HOG_POD_CREATE_TIMEOUT = 120 +HOG_POD_CLEANUP_TIMEOUT = 60 +KRAKEN_RUN_TIMEOUT = 300 + + +def _list_hog_pods(k8s_core, namespace): + """Return hog pods (name prefix cpu-hog-) currently in the namespace.""" + pods = k8s_core.list_namespaced_pod(namespace=namespace) + return [ + p for p in pods.items + if p.metadata and p.metadata.name and p.metadata.name.startswith(HOG_POD_PREFIX) + ] + + +def _wait_for_scheduled_hog_pod(k8s_core, namespace, timeout): + """Poll until a hog pod exists and is scheduled (spec.node_name set). Return it, or the last seen pod (may be None).""" + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + for p in _list_hog_pods(k8s_core, namespace): + last = p + if p.spec and p.spec.node_name: + return p + time.sleep(0.5) + return last + + +def _wait_for_no_hog_pods(k8s_core, namespace, timeout): + """Assert all hog pods are removed from the namespace within timeout.""" + deadline = time.monotonic() + timeout + last = [] + while time.monotonic() < deadline: + last = _list_hog_pods(k8s_core, namespace) + if not last: + return + time.sleep(1) + raise AssertionError( + f"Hog pods still present in namespace={namespace} after {timeout}s: " + f"{[p.metadata.name for p in last]}" + ) + + +def _schedulable_worker_nodes(k8s_core): + """Return names of Ready nodes that are not control-plane/master and carry no NoSchedule/NoExecute taint.""" + names = [] + for node in k8s_core.list_node().items: + labels = (node.metadata.labels or {}) if node.metadata else {} + if ( + "node-role.kubernetes.io/control-plane" in labels + or "node-role.kubernetes.io/master" in labels + ): + continue + taints = (node.spec.taints or []) if node.spec else [] + if any(getattr(t, "effect", None) in ("NoSchedule", "NoExecute") for t in taints): + continue + ready = any( + c.type == "Ready" and c.status == "True" + for c in ((node.status.conditions or []) if node.status else []) + ) + if ready: + names.append(node.metadata.name) + return names + + +@pytest.mark.functional +@pytest.mark.cpu_hog +class TestCpuHog(BaseScenarioTest): + """CPU hog scenario: deploy a CPU hog pod on selected node(s), then verify success and cleanup.""" + + SCENARIO_NAME = "cpu_hog" + SCENARIO_TYPE = "hog_scenarios" + NAMESPACE_KEY_PATH = ["namespace"] + NAMESPACE_IS_REGEX = False + + def _scenario(self, namespace, overrides): + """Load scenario_base.yaml, patch namespace, then apply flat-dict overrides (hyphenated keys).""" + scenario = self.load_and_patch_scenario(self.repo_root, namespace) + scenario.update(overrides) + return scenario + + @pytest.mark.no_workload + @pytest.mark.order(1) + def test_cpu_hog_success_lifecycle_and_targeting(self): + """Happy path: a hog pod is created on the node-selector target, the run succeeds, and the pod is cleaned up.""" + nodes = _schedulable_worker_nodes(self.k8s_core) + if not nodes: + pytest.skip("No schedulable worker node available for CPU hog targeting") + node = nodes[0] + ns = self.ns + scenario = self._scenario(ns, { + "node-selector": f"kubernetes.io/hostname={node}", + "number-of-nodes": 1, + "duration": 20, + }) + scenario_path = self.write_scenario(self.tmp_path, scenario) + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="cpu_hog_success_config.yaml" + ) + proc = self.run_kraken_background(config_path) + try: + pod = _wait_for_scheduled_hog_pod(self.k8s_core, ns, timeout=HOG_POD_CREATE_TIMEOUT) + assert pod is not None, ( + f"Expected a CPU hog pod (prefix {HOG_POD_PREFIX!r}) to be created in namespace={ns}" + ) + assert pod.spec and pod.spec.node_name == node, ( + f"CPU hog pod {pod.metadata.name} scheduled on " + f"{getattr(pod.spec, 'node_name', None)!r}, expected node-selector target {node!r} " + f"(namespace={ns})" + ) + out, err = proc.communicate(timeout=KRAKEN_RUN_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + out, err = proc.communicate() + raise + except BaseException: + # Any poll/assert failure before communicate() must still tear down the + # background Kraken process, otherwise its hog pod keeps stressing the node + # and races against subsequent --reruns on the same target. + if proc.poll() is None: + proc.kill() + proc.wait() + raise + result = subprocess.CompletedProcess(args=[], returncode=proc.returncode, stdout=out, stderr=err) + assert_kraken_success(result, context=f"node={node} namespace={ns}", tmp_path=self.tmp_path) + _wait_for_no_hog_pods(self.k8s_core, ns, timeout=HOG_POD_CLEANUP_TIMEOUT) + + @pytest.mark.no_workload + @pytest.mark.order(2) + def test_cpu_hog_invalid_selector_fails(self): + """Negative: a node-selector matching zero nodes makes Krkn fail (no available nodes to schedule).""" + ns = self.ns + scenario = self._scenario(ns, { + "node-selector": "kubernetes.io/hostname=krkn-nonexistent-node-zzz", + "number-of-nodes": 1, + "duration": 20, + }) + scenario_path = self.write_scenario(self.tmp_path, scenario) + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="cpu_hog_invalid_selector_config.yaml" + ) + result = self.run_kraken(config_path, timeout=KRAKEN_RUN_TIMEOUT) + assert_kraken_failure( + result, context=f"invalid node-selector namespace={ns}", tmp_path=self.tmp_path + ) + _wait_for_no_hog_pods(self.k8s_core, ns, timeout=HOG_POD_CLEANUP_TIMEOUT) + + @pytest.mark.no_workload + @pytest.mark.order(3) + def test_cpu_hog_invalid_config_fails(self): + """Negative: omitting the mandatory hog-type field makes Krkn fail at config parsing.""" + ns = self.ns + scenario = self._scenario(ns, {"duration": 20}) + scenario.pop("hog-type", None) + scenario_path = self.write_scenario(self.tmp_path, scenario) + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="cpu_hog_invalid_config_config.yaml" + ) + result = self.run_kraken(config_path, timeout=KRAKEN_RUN_TIMEOUT) + assert_kraken_failure( + result, context=f"missing hog-type namespace={ns}", tmp_path=self.tmp_path + )