From 8265c358d37b551b4f42fc519fc0a3cfba5426c2 Mon Sep 17 00:00:00 2001 From: "augmentcode[bot]" <185243770+augmentcode[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:02:39 +0530 Subject: [PATCH] test: migrate memory_hog functional test to tests_v2 (closes #1396) (#1397) * test: migrate memory_hog functional test to tests_v2 (closes #1396) Add a pytest v2 functional test for the memory hog scenario (hog_scenarios), mirroring the cpu_hog migration. Covers execution success, node-selector targeting, duration/memory-size parameter handling, hog pod lifecycle and cleanup, and graceful failure on invalid selector / invalid config. - CI/tests_v2/scenarios/memory_hog/test_memory_hog.py: TestMemoryHog with functional + memory_hog markers and three no_workload tests. - CI/tests_v2/scenarios/memory_hog/scenario_base.yaml: flat hog config tuned for functional testing (light fixed memory-vm-bytes, short duration). - Register the memory_hog marker in pytest.ini and document the scenario in the tests_v2 README. * test: extract shared hog-pod/node helpers into tests_v2 lib/utils Lift the duplicated pod-prefix and schedulable-node helpers out of the cpu_hog and memory_hog test modules into CI/tests_v2/lib/utils.py as parameterized, reusable functions (list_pods_by_prefix, wait_for_scheduled_pod_by_prefix, wait_for_no_pods_by_prefix, schedulable_worker_nodes) and reuse them from both scenarios. Addresses review feedback on #1397. --------- Co-authored-by: augmentcode[bot] <185243770+augmentcode[bot]@users.noreply.github.com> --- CI/tests_v2/README.md | 16 ++- CI/tests_v2/lib/utils.py | 66 +++++++++ CI/tests_v2/pytest.ini | 1 + CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py | 76 ++-------- .../scenarios/memory_hog/scenario_base.yaml | 16 +++ .../scenarios/memory_hog/test_memory_hog.py | 131 ++++++++++++++++++ 6 files changed, 239 insertions(+), 67 deletions(-) create mode 100644 CI/tests_v2/scenarios/memory_hog/scenario_base.yaml create mode 100644 CI/tests_v2/scenarios/memory_hog/test_memory_hog.py diff --git a/CI/tests_v2/README.md b/CI/tests_v2/README.md index 31f3f5c9..f8763652 100644 --- a/CI/tests_v2/README.md +++ b/CI/tests_v2/README.md @@ -1,6 +1,6 @@ # Pytest Functional Tests (tests_v2) -This directory contains a pytest-based functional test framework that runs **alongside** the existing bash tests in `CI/tests/`. It covers the **pod disruption**, **application outage**, **storage throttle**, and **CPU hog** scenarios with proper assertions, retries, and reporting. +This directory contains a pytest-based functional test framework that runs **alongside** the existing bash tests in `CI/tests/`. It covers the **pod disruption**, **application outage**, **storage throttle**, **CPU hog**, and **memory hog** scenarios with proper assertions, retries, and reporting. Each test runs in its **own ephemeral Kubernetes namespace** (`krkn-test-`). Before the test, the framework creates the namespace, deploys the target workload, and waits for pods to be ready. After the test, the namespace is deleted (cascading all resources). **You do not need to deploy any workloads manually.** @@ -127,6 +127,12 @@ pytest CI/tests_v2/ -v -m application_outage pytest CI/tests_v2/ -v -m cpu_hog ``` +### Run only memory hog tests + +```bash +pytest CI/tests_v2/ -v -m memory_hog +``` + ### Run with verbose output and no capture ```bash @@ -186,9 +192,15 @@ Each test runs in an isolated ephemeral namespace; workloads are deployed automa - **test_cpu_hog_invalid_selector_fails**: A `node-selector` matching zero nodes causes Kraken to exit non-zero (no available nodes to schedule). - **test_cpu_hog_invalid_config_fails**: Omitting the mandatory `hog-type` field causes Kraken to exit non-zero at config parsing. +- **scenarios/memory_hog/** + Memory hog scenario (`hog_scenarios`), migrated from the legacy `CI/tests/test_memory_hog.sh`. Memory hog targets nodes (not workloads): Kraken deploys a short-lived hog pod (name prefix `memory-hog-`) onto each selected node, runs `stress-ng` for the configured duration with the configured `memory-vm-bytes`, then deletes the pod. Tests use `@pytest.mark.no_workload` (no app deployment needed); `scenario_base.yaml` is a flat hog config patched per test (with a small fixed `memory-vm-bytes` instead of the production `90%`). Tests include: + - **test_memory_hog_success_lifecycle_and_targeting**: Happy path — a hog pod is created on the `node-selector` target with the configured memory size, the run exits 0, and the pod is cleaned up afterward. + - **test_memory_hog_invalid_selector_fails**: A `node-selector` matching zero nodes causes Kraken to exit non-zero (no available nodes to schedule). + - **test_memory_hog_invalid_config_fails**: Omitting the mandatory `hog-type` field causes Kraken to exit non-zero at config parsing. + ## Configuration -- **pytest.ini**: Markers (`functional`, `pod_disruption`, `application_outage`, `storage_throttle`, `cpu_hog`, `no_workload`). Use `--timeout=300`, `--reruns=2`, `--reruns-delay=10` on the command line for full runs. +- **pytest.ini**: Markers (`functional`, `pod_disruption`, `application_outage`, `storage_throttle`, `cpu_hog`, `memory_hog`, `no_workload`). Use `--timeout=300`, `--reruns=2`, `--reruns-delay=10` on the command line for full runs. - **conftest.py**: Re-exports fixtures from `lib/k8s.py`, `lib/namespace.py`, `lib/deploy.py`, `lib/kraken.py` (e.g. `test_namespace`, `deploy_workload`, `k8s_core`, `wait_for_pods_running`, `run_kraken`, `build_config`). Configs are built from `CI/tests_v2/config/common_test_config.yaml` with monitoring disabled for local runs. Timeout constants in `lib/base.py` can be overridden via env vars. - **Cluster access**: Reads and applies use the Kubernetes Python client; `kubectl` is still used for `port-forward` and for running Kraken. - **utils.py**: Pod/network policy helpers and assertion helpers (`assert_all_pods_running_and_ready`, `assert_pod_count_unchanged`, `assert_kraken_success`, `assert_kraken_failure`, `patch_namespace_in_docs`). diff --git a/CI/tests_v2/lib/utils.py b/CI/tests_v2/lib/utils.py index 9070a3ed..181d5d6f 100644 --- a/CI/tests_v2/lib/utils.py +++ b/CI/tests_v2/lib/utils.py @@ -136,6 +136,72 @@ def restart_counts(pod_list: Union[V1PodList, List[V1Pod]]) -> int: return total +def list_pods_by_prefix(k8s_core, namespace: str, name_prefix: str) -> List[V1Pod]: + """Return pods in the namespace whose name starts with name_prefix.""" + pods = k8s_core.list_namespaced_pod(namespace=namespace) + return [ + p for p in _pods(pods) + if p.metadata and p.metadata.name and p.metadata.name.startswith(name_prefix) + ] + + +def wait_for_scheduled_pod_by_prefix( + k8s_core, namespace: str, name_prefix: str, timeout: float +) -> Optional[V1Pod]: + """ + Poll until a pod with name_prefix exists and is scheduled (spec.node_name set). + Return it, or the last seen matching pod (may be None) if none get scheduled in time. + """ + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + for p in list_pods_by_prefix(k8s_core, namespace, name_prefix): + last = p + if p.spec and p.spec.node_name: + return p + time.sleep(0.5) + return last + + +def wait_for_no_pods_by_prefix( + k8s_core, namespace: str, name_prefix: str, timeout: float +) -> None: + """Assert all pods with name_prefix are removed from the namespace within timeout.""" + deadline = time.monotonic() + timeout + last = [] + while time.monotonic() < deadline: + last = list_pods_by_prefix(k8s_core, namespace, name_prefix) + if not last: + return + time.sleep(1) + raise AssertionError( + f"Pods with prefix {name_prefix!r} still present in namespace={namespace} " + f"after {timeout}s: {[p.metadata.name for p in last]}" + ) + + +def schedulable_worker_nodes(k8s_core) -> List[str]: + """Return names of Ready nodes that are not control-plane/master and carry no NoSchedule/NoExecute taint.""" + names = [] + for node in k8s_core.list_node().items: + labels = (node.metadata.labels or {}) if node.metadata else {} + if ( + "node-role.kubernetes.io/control-plane" in labels + or "node-role.kubernetes.io/master" in labels + ): + continue + taints = (node.spec.taints or []) if node.spec else [] + if any(getattr(t, "effect", None) in ("NoSchedule", "NoExecute") for t in taints): + continue + ready = any( + c.type == "Ready" and c.status == "True" + for c in ((node.status.conditions or []) if node.status else []) + ) + if ready: + names.append(node.metadata.name) + return names + + def get_network_policies_list(k8s_networking, namespace: str) -> V1NetworkPolicyList: """Return V1NetworkPolicyList from the Kubernetes API.""" return k8s_networking.list_namespaced_network_policy(namespace=namespace) diff --git a/CI/tests_v2/pytest.ini b/CI/tests_v2/pytest.ini index f4557725..f652bc20 100644 --- a/CI/tests_v2/pytest.ini +++ b/CI/tests_v2/pytest.ini @@ -11,6 +11,7 @@ markers = application_outage: marks a test as an application outage scenario test storage_throttle: marks a test as a storage throttle scenario test cpu_hog: marks a test as a CPU hog scenario test + memory_hog: marks a test as a memory hog scenario test no_workload: skip workload deployment for this test (e.g. negative tests) order: set test order (pytest-order) junit_family = xunit2 diff --git a/CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py b/CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py index 6ea6d2be..aa619b89 100644 --- a/CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py +++ b/CI/tests_v2/scenarios/cpu_hog/test_cpu_hog.py @@ -11,12 +11,17 @@ failure on invalid selector/config. import logging import subprocess -import time import pytest from lib.base import BaseScenarioTest -from lib.utils import assert_kraken_failure, assert_kraken_success +from lib.utils import ( + assert_kraken_failure, + assert_kraken_success, + schedulable_worker_nodes, + wait_for_no_pods_by_prefix, + wait_for_scheduled_pod_by_prefix, +) logger = logging.getLogger(__name__) @@ -26,65 +31,6 @@ HOG_POD_CLEANUP_TIMEOUT = 60 KRAKEN_RUN_TIMEOUT = 300 -def _list_hog_pods(k8s_core, namespace): - """Return hog pods (name prefix cpu-hog-) currently in the namespace.""" - pods = k8s_core.list_namespaced_pod(namespace=namespace) - return [ - p for p in pods.items - if p.metadata and p.metadata.name and p.metadata.name.startswith(HOG_POD_PREFIX) - ] - - -def _wait_for_scheduled_hog_pod(k8s_core, namespace, timeout): - """Poll until a hog pod exists and is scheduled (spec.node_name set). Return it, or the last seen pod (may be None).""" - deadline = time.monotonic() + timeout - last = None - while time.monotonic() < deadline: - for p in _list_hog_pods(k8s_core, namespace): - last = p - if p.spec and p.spec.node_name: - return p - time.sleep(0.5) - return last - - -def _wait_for_no_hog_pods(k8s_core, namespace, timeout): - """Assert all hog pods are removed from the namespace within timeout.""" - deadline = time.monotonic() + timeout - last = [] - while time.monotonic() < deadline: - last = _list_hog_pods(k8s_core, namespace) - if not last: - return - time.sleep(1) - raise AssertionError( - f"Hog pods still present in namespace={namespace} after {timeout}s: " - f"{[p.metadata.name for p in last]}" - ) - - -def _schedulable_worker_nodes(k8s_core): - """Return names of Ready nodes that are not control-plane/master and carry no NoSchedule/NoExecute taint.""" - names = [] - for node in k8s_core.list_node().items: - labels = (node.metadata.labels or {}) if node.metadata else {} - if ( - "node-role.kubernetes.io/control-plane" in labels - or "node-role.kubernetes.io/master" in labels - ): - continue - taints = (node.spec.taints or []) if node.spec else [] - if any(getattr(t, "effect", None) in ("NoSchedule", "NoExecute") for t in taints): - continue - ready = any( - c.type == "Ready" and c.status == "True" - for c in ((node.status.conditions or []) if node.status else []) - ) - if ready: - names.append(node.metadata.name) - return names - - @pytest.mark.functional @pytest.mark.cpu_hog class TestCpuHog(BaseScenarioTest): @@ -105,7 +51,7 @@ class TestCpuHog(BaseScenarioTest): @pytest.mark.order(1) def test_cpu_hog_success_lifecycle_and_targeting(self): """Happy path: a hog pod is created on the node-selector target, the run succeeds, and the pod is cleaned up.""" - nodes = _schedulable_worker_nodes(self.k8s_core) + nodes = schedulable_worker_nodes(self.k8s_core) if not nodes: pytest.skip("No schedulable worker node available for CPU hog targeting") node = nodes[0] @@ -121,7 +67,7 @@ class TestCpuHog(BaseScenarioTest): ) proc = self.run_kraken_background(config_path) try: - pod = _wait_for_scheduled_hog_pod(self.k8s_core, ns, timeout=HOG_POD_CREATE_TIMEOUT) + pod = wait_for_scheduled_pod_by_prefix(self.k8s_core, ns, HOG_POD_PREFIX, timeout=HOG_POD_CREATE_TIMEOUT) assert pod is not None, ( f"Expected a CPU hog pod (prefix {HOG_POD_PREFIX!r}) to be created in namespace={ns}" ) @@ -145,7 +91,7 @@ class TestCpuHog(BaseScenarioTest): raise result = subprocess.CompletedProcess(args=[], returncode=proc.returncode, stdout=out, stderr=err) assert_kraken_success(result, context=f"node={node} namespace={ns}", tmp_path=self.tmp_path) - _wait_for_no_hog_pods(self.k8s_core, ns, timeout=HOG_POD_CLEANUP_TIMEOUT) + wait_for_no_pods_by_prefix(self.k8s_core, ns, HOG_POD_PREFIX, timeout=HOG_POD_CLEANUP_TIMEOUT) @pytest.mark.no_workload @pytest.mark.order(2) @@ -165,7 +111,7 @@ class TestCpuHog(BaseScenarioTest): assert_kraken_failure( result, context=f"invalid node-selector namespace={ns}", tmp_path=self.tmp_path ) - _wait_for_no_hog_pods(self.k8s_core, ns, timeout=HOG_POD_CLEANUP_TIMEOUT) + wait_for_no_pods_by_prefix(self.k8s_core, ns, HOG_POD_PREFIX, timeout=HOG_POD_CLEANUP_TIMEOUT) @pytest.mark.no_workload @pytest.mark.order(3) diff --git a/CI/tests_v2/scenarios/memory_hog/scenario_base.yaml b/CI/tests_v2/scenarios/memory_hog/scenario_base.yaml new file mode 100644 index 00000000..151751b7 --- /dev/null +++ b/CI/tests_v2/scenarios/memory_hog/scenario_base.yaml @@ -0,0 +1,16 @@ +# Base memory hog scenario (hog_scenarios). Tests load this and patch namespace, node-selector, +# number-of-nodes, duration, memory-vm-bytes, etc. The memory hog targets nodes (not workloads): +# Krkn deploys a short-lived hog pod (name prefix "memory-hog-") on each selected node, runs +# `stress-ng` for the configured duration, then deletes the pod. Values here are tuned for +# functional testing (light load, short duration), not performance benchmarking; in particular +# memory-vm-bytes is a small fixed size rather than the production "90%" to avoid pressuring +# kind nodes. +duration: 20 +workers: 1 +hog-type: memory +image: quay.io/krkn-chaos/krkn-hog +namespace: default +memory-vm-bytes: 256m +node-selector: "node-role.kubernetes.io/worker=" +number-of-nodes: 1 +taints: [] diff --git a/CI/tests_v2/scenarios/memory_hog/test_memory_hog.py b/CI/tests_v2/scenarios/memory_hog/test_memory_hog.py new file mode 100644 index 00000000..96ded1ba --- /dev/null +++ b/CI/tests_v2/scenarios/memory_hog/test_memory_hog.py @@ -0,0 +1,131 @@ +""" +Functional tests for the memory hog scenario (hog_scenarios), migrated from the legacy +CI/tests/test_memory_hog.sh. + +Memory hog targets nodes (not workloads): Krkn deploys a short-lived hog pod (name prefix +"memory-hog-") onto each selected node, runs it for the configured duration, then deletes the +pod. These tests therefore use @pytest.mark.no_workload (no app deployment is needed) and +verify execution success, node-selector targeting, duration/lifecycle, memory size parameter +handling, cleanup, and graceful failure on invalid selector/config. +""" + +import logging +import subprocess + +import pytest + +from lib.base import BaseScenarioTest +from lib.utils import ( + assert_kraken_failure, + assert_kraken_success, + schedulable_worker_nodes, + wait_for_no_pods_by_prefix, + wait_for_scheduled_pod_by_prefix, +) + +logger = logging.getLogger(__name__) + +HOG_POD_PREFIX = "memory-hog-" +HOG_POD_CREATE_TIMEOUT = 120 +HOG_POD_CLEANUP_TIMEOUT = 60 +KRAKEN_RUN_TIMEOUT = 300 + + +@pytest.mark.functional +@pytest.mark.memory_hog +class TestMemoryHog(BaseScenarioTest): + """Memory hog scenario: deploy a memory hog pod on selected node(s), then verify success and cleanup.""" + + SCENARIO_NAME = "memory_hog" + SCENARIO_TYPE = "hog_scenarios" + NAMESPACE_KEY_PATH = ["namespace"] + NAMESPACE_IS_REGEX = False + + def _scenario(self, namespace, overrides): + """Load scenario_base.yaml, patch namespace, then apply flat-dict overrides (hyphenated keys).""" + scenario = self.load_and_patch_scenario(self.repo_root, namespace) + scenario.update(overrides) + return scenario + + @pytest.mark.no_workload + @pytest.mark.order(1) + def test_memory_hog_success_lifecycle_and_targeting(self): + """Happy path: a hog pod is created on the node-selector target with the configured memory size, the run succeeds, and the pod is cleaned up.""" + nodes = schedulable_worker_nodes(self.k8s_core) + if not nodes: + pytest.skip("No schedulable worker node available for memory hog targeting") + node = nodes[0] + ns = self.ns + scenario = self._scenario(ns, { + "node-selector": f"kubernetes.io/hostname={node}", + "number-of-nodes": 1, + "duration": 20, + "memory-vm-bytes": "256m", + }) + scenario_path = self.write_scenario(self.tmp_path, scenario) + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="memory_hog_success_config.yaml" + ) + proc = self.run_kraken_background(config_path) + try: + pod = wait_for_scheduled_pod_by_prefix(self.k8s_core, ns, HOG_POD_PREFIX, timeout=HOG_POD_CREATE_TIMEOUT) + assert pod is not None, ( + f"Expected a memory hog pod (prefix {HOG_POD_PREFIX!r}) to be created in namespace={ns}" + ) + assert pod.spec and pod.spec.node_name == node, ( + f"Memory hog pod {pod.metadata.name} scheduled on " + f"{getattr(pod.spec, 'node_name', None)!r}, expected node-selector target {node!r} " + f"(namespace={ns})" + ) + out, err = proc.communicate(timeout=KRAKEN_RUN_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + out, err = proc.communicate() + raise + except BaseException: + # Any poll/assert failure before communicate() must still tear down the + # background Kraken process, otherwise its hog pod keeps stressing the node + # and races against subsequent --reruns on the same target. + if proc.poll() is None: + proc.kill() + proc.wait() + raise + result = subprocess.CompletedProcess(args=[], returncode=proc.returncode, stdout=out, stderr=err) + assert_kraken_success(result, context=f"node={node} namespace={ns}", tmp_path=self.tmp_path) + wait_for_no_pods_by_prefix(self.k8s_core, ns, HOG_POD_PREFIX, timeout=HOG_POD_CLEANUP_TIMEOUT) + + @pytest.mark.no_workload + @pytest.mark.order(2) + def test_memory_hog_invalid_selector_fails(self): + """Negative: a node-selector matching zero nodes makes Krkn fail (no available nodes to schedule).""" + ns = self.ns + scenario = self._scenario(ns, { + "node-selector": "kubernetes.io/hostname=krkn-nonexistent-node-zzz", + "number-of-nodes": 1, + "duration": 20, + }) + scenario_path = self.write_scenario(self.tmp_path, scenario) + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="memory_hog_invalid_selector_config.yaml" + ) + result = self.run_kraken(config_path, timeout=KRAKEN_RUN_TIMEOUT) + assert_kraken_failure( + result, context=f"invalid node-selector namespace={ns}", tmp_path=self.tmp_path + ) + wait_for_no_pods_by_prefix(self.k8s_core, ns, HOG_POD_PREFIX, timeout=HOG_POD_CLEANUP_TIMEOUT) + + @pytest.mark.no_workload + @pytest.mark.order(3) + def test_memory_hog_invalid_config_fails(self): + """Negative: omitting the mandatory hog-type field makes Krkn fail at config parsing.""" + ns = self.ns + scenario = self._scenario(ns, {"duration": 20}) + scenario.pop("hog-type", None) + scenario_path = self.write_scenario(self.tmp_path, scenario) + config_path = self.build_config( + self.SCENARIO_TYPE, str(scenario_path), filename="memory_hog_invalid_config_config.yaml" + ) + result = self.run_kraken(config_path, timeout=KRAKEN_RUN_TIMEOUT) + assert_kraken_failure( + result, context=f"missing hog-type namespace={ns}", tmp_path=self.tmp_path + )