From ebe6049be9a09274e93e2de0e5fc7758bfc779a6 Mon Sep 17 00:00:00 2001 From: Paige Patton <64206430+paigerube14@users.noreply.github.com> Date: Wed, 13 May 2026 13:20:08 -0400 Subject: [PATCH] adding start of vmi network scenario (#1260) Assisted By: Claude Code: Assisted By: Claude Code: Signed-off-by: Paige Patton --- config/config.yaml | 1 + .../network_chaos_ng/models.py | 1 + .../network_chaos_ng/modules/utils.py | 53 ++- .../modules/utils_network_chaos.py | 45 +- .../modules/vmi_network_chaos.py | 313 +++++++++++++ .../network_chaos_ng/network_chaos_factory.py | 10 + scenarios/openshift/virt_network_chaos.yaml | 17 + tests/test_utils_network_chaos.py | 58 +++ tests/test_vmi_network_chaos.py | 440 ++++++++++++++++++ 9 files changed, 930 insertions(+), 8 deletions(-) create mode 100644 krkn/scenario_plugins/network_chaos_ng/modules/vmi_network_chaos.py create mode 100644 scenarios/openshift/virt_network_chaos.yaml create mode 100644 tests/test_vmi_network_chaos.py diff --git a/config/config.yaml b/config/config.yaml index b5c340ef..ab7b87ee 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -53,6 +53,7 @@ kraken: - scenarios/kube/node-network-chaos.yml - scenarios/kube/pod-network-chaos.yml - scenarios/kube/node_interface_down.yaml + - scenarios/openshift/virt_network_chaos.yaml - kubevirt_vm_outage: - scenarios/kubevirt/kubevirt-vm-outage.yaml - http_load_scenarios: diff --git a/krkn/scenario_plugins/network_chaos_ng/models.py b/krkn/scenario_plugins/network_chaos_ng/models.py index be48578e..1c45aad5 100644 --- a/krkn/scenario_plugins/network_chaos_ng/models.py +++ b/krkn/scenario_plugins/network_chaos_ng/models.py @@ -20,6 +20,7 @@ from typing import TypeVar, Optional class NetworkChaosScenarioType(Enum): Node = 1 Pod = 2 + VMI = 3 @dataclass diff --git a/krkn/scenario_plugins/network_chaos_ng/modules/utils.py b/krkn/scenario_plugins/network_chaos_ng/modules/utils.py index 24231d6a..f4f45049 100644 --- a/krkn/scenario_plugins/network_chaos_ng/modules/utils.py +++ b/krkn/scenario_plugins/network_chaos_ng/modules/utils.py @@ -13,7 +13,7 @@ # limitations under the License. import logging import os -from typing import Tuple +from typing import Optional, Tuple import yaml from jinja2 import FileSystemLoader, Environment @@ -110,6 +110,57 @@ def get_pod_default_interface( return output.replace("\n", "") +def find_virt_launcher_netns_pid( + chaos_pod_name: str, + namespace: str, + pids: list[str], + kubecli: KrknKubernetes, +) -> Optional[str]: + """Return the first PID from `pids` whose netns contains a tap device. + + Not all PIDs returned by get_pod_pids are inside the virt-launcher network + namespace — some helper processes run in the host netns. Entering one of + those would target the node's physical NIC instead of the bridge slave + inside the virt-launcher netns. + """ + for pid in pids: + try: + result = kubecli.exec_cmd_in_pod( + [f"nsenter --target {pid} --net -- ip link show type tun"], + chaos_pod_name, + namespace, + ) + if result and "tap" in result: + return pid + except Exception as e: + logging.warning(f"failed to check netns for PID {pid}: {e}") + continue + return None + + +def get_vmi_tap_interface( + chaos_pod_name: str, + namespace: str, + netns_pid: str, + kubecli: KrknKubernetes, +) -> Optional[str]: + """Return the name of the tap device inside the virt-launcher netns.""" + result = kubecli.exec_cmd_in_pod( + [f"nsenter --target {netns_pid} --net -- ip -o link show type tun"], + chaos_pod_name, + namespace, + ) + if not result: + return None + for line in result.splitlines(): + parts = line.split(":") + if len(parts) >= 2: + iface = parts[1].strip().split("@")[0].strip() + if iface.startswith("tap"): + return iface + return None + + def setup_network_chaos_ng_scenario( config: BaseNetworkChaosConfig, node_name: str, diff --git a/krkn/scenario_plugins/network_chaos_ng/modules/utils_network_chaos.py b/krkn/scenario_plugins/network_chaos_ng/modules/utils_network_chaos.py index 91f84c41..92d9fcaf 100644 --- a/krkn/scenario_plugins/network_chaos_ng/modules/utils_network_chaos.py +++ b/krkn/scenario_plugins/network_chaos_ng/modules/utils_network_chaos.py @@ -28,6 +28,37 @@ CLASS_ID = "100:1" NETEM_HANDLE = "101:" +def _normalize_rate(value: Optional[str]) -> str: + """Accept "100" or "100mbit"/"1gbit"; always return a tc-valid rate string.""" + if value is None: + return "1gbit" + s = value.strip() + try: + float(s) + return f"{s}mbit" + except ValueError: + return s + + +def _normalize_delay(value: Optional[str]) -> str: + """Accept "50" or "50ms"; always return a tc-valid delay string.""" + if value is None: + return "0ms" + s = value.strip() + try: + float(s) + return f"{s}ms" + except ValueError: + return s + + +def _normalize_loss(value: Optional[str]) -> str: + """Accept "10" or "10%"; return the bare numeric string (% appended by caller).""" + if value is None: + return "0" + return value.strip().rstrip("%") + + def run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess: return subprocess.run(cmd, check=check, text=True, capture_output=True) @@ -64,15 +95,15 @@ def get_egress_shaping_comand( ) -> list[str]: rate_commands = [] - rate = f"{rate_mbit}mbit" if rate_mbit is not None else "1gbit" - d = delay_ms if delay_ms is not None else 0 - l = loss_pct if loss_pct is not None else 0 + rate = _normalize_rate(rate_mbit) + d = _normalize_delay(delay_ms) + l = _normalize_loss(loss_pct) for dev in devices: rate_commands.append( f"tc class change dev {dev} parent {ROOT_HANDLE} classid {CLASS_ID} htb rate {rate}" ) rate_commands.append( - f"tc qdisc change dev {dev} parent {CLASS_ID} handle {NETEM_HANDLE} netem delay {d}ms loss {l}%" + f"tc qdisc change dev {dev} parent {CLASS_ID} handle {NETEM_HANDLE} netem delay {d} loss {l}%" ) return rate_commands @@ -108,12 +139,12 @@ def get_ingress_shaping_commands( ) rate_commands.append( f"tc class add dev {ifb_dev} parent {ROOT_HANDLE} classid {CLASS_ID} " - f"htb rate {rate_mbit if rate_mbit else '1gbit'} || true" + f"htb rate {_normalize_rate(rate_mbit)} || true" ) rate_commands.append( f"tc qdisc add dev {ifb_dev} parent {CLASS_ID} handle {NETEM_HANDLE} " - f"netem delay {delay_ms if delay_ms else '0ms'} " - f"loss {loss_pct if loss_pct else '0'}% || true" + f"netem delay {_normalize_delay(delay_ms)} " + f"loss {_normalize_loss(loss_pct)}% || true" ) return rate_commands diff --git a/krkn/scenario_plugins/network_chaos_ng/modules/vmi_network_chaos.py b/krkn/scenario_plugins/network_chaos_ng/modules/vmi_network_chaos.py new file mode 100644 index 00000000..e90be96d --- /dev/null +++ b/krkn/scenario_plugins/network_chaos_ng/modules/vmi_network_chaos.py @@ -0,0 +1,313 @@ +# Copyright 2025 The Krkn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import dataclasses +import queue +import re +import time +from typing import Tuple + +from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift +from krkn_lib.utils import get_random_string + +from krkn.scenario_plugins.network_chaos_ng.models import ( + NetworkChaosScenarioType, + BaseNetworkChaosConfig, + NetworkChaosConfig, +) +from krkn.scenario_plugins.network_chaos_ng.modules.abstract_network_chaos_module import ( + AbstractNetworkChaosModule, +) +from krkn.scenario_plugins.network_chaos_ng.modules.utils import ( + log_info, + log_error, + deploy_network_chaos_ng_pod, + find_virt_launcher_netns_pid, + get_vmi_tap_interface, +) +from krkn.scenario_plugins.network_chaos_ng.modules.utils_network_chaos import ( + common_set_limit_rules, + common_delete_limit_rules, +) + + +class VmiNetworkChaosModule(AbstractNetworkChaosModule): + + def __init__(self, config: NetworkChaosConfig, kubecli: KrknTelemetryOpenshift): + super().__init__(config, kubecli) + self.config = config + + def _rollback( + self, + namespace: str, + network_chaos_pod_name: str, + netns_pid: str = None, + iface: str = None, + parallel: bool = False, + vmi_name: str = "", + ): + if netns_pid and iface: + common_delete_limit_rules( + self.config.egress, + self.config.ingress, + [iface], + network_chaos_pod_name, + namespace, + self.kubecli.get_lib_kubernetes(), + [netns_pid], + parallel, + vmi_name, + ) + self.kubecli.get_lib_kubernetes().delete_pod( + network_chaos_pod_name, namespace + ) + + def run(self, target: str, error_queue: queue.Queue = None): + # target is "namespace/vmi-name" as produced by get_targets() + parallel = False + if error_queue: + parallel = True + + network_chaos_pod_name = None + netns_pid = None + iface = None + namespace = "" + vmi_name = "" + + try: + namespace, vmi_name = target.split("/", 1) + # Create a scoped config with the resolved namespace so that all + # Kubernetes calls use the actual namespace, not the regex pattern. + scoped_config = dataclasses.replace(self.config, namespace=namespace) + network_chaos_pod_name = f"vmi-network-chaos-{get_random_string(5)}" + container_name = f"fedora-container-{get_random_string(5)}" + + log_info( + f"creating workload to inject network chaos in VMI {vmi_name} " + f"latency:{str(self.config.latency) if self.config.latency else '0'}, " + f"loss:{str(self.config.loss) if self.config.loss else '0'}, " + f"bandwidth:{str(self.config.bandwidth) if self.config.bandwidth else '0'}", + parallel, + network_chaos_pod_name, + ) + + vmi = self.kubecli.get_lib_kubernetes().get_vmi(vmi_name, namespace) + if not vmi: + raise Exception( + f"VMI {vmi_name} not found in namespace {namespace}" + ) + + node_name = vmi.get("status", {}).get("nodeName") + if not node_name: + raise Exception( + f"unable to determine node for VMI {vmi_name} in namespace {namespace}; " + "VMI may not be in Running phase" + ) + + log_info( + f"VMI {vmi_name} is running on node {node_name}", + parallel, + network_chaos_pod_name, + ) + + # The virt-launcher pod carries the VMI's network namespace. + virt_launcher_pods = self.kubecli.get_lib_kubernetes().list_pods( + namespace, label_selector=f"vm.kubevirt.io/name={vmi_name}" + ) + if not virt_launcher_pods: + raise Exception( + f"no virt-launcher pod found for VMI {vmi_name} in namespace {namespace}" + ) + virt_launcher_pod_name = virt_launcher_pods[0] + + log_info( + f"resolved virt-launcher pod {virt_launcher_pod_name} for VMI {vmi_name}", + parallel, + network_chaos_pod_name, + ) + + # Deploy the privileged chaos pod onto the VMI's node. + # hostPID=True (via template) allows nsenter into the virt-launcher's + # network namespace using any of the compute container's host PIDs. + deploy_network_chaos_ng_pod( + scoped_config, + node_name, + network_chaos_pod_name, + self.kubecli.get_lib_kubernetes(), + container_name, + host_network=False, + ) + + # Prefer the 'compute' container (the QEMU process in KubeVirt). + # 'virt-launcher' is a sidecar monitor that may not be running. + pod_info = self.kubecli.get_lib_kubernetes().get_pod_info( + virt_launcher_pod_name, namespace + ) + if not pod_info: + raise Exception( + f"impossible to retrieve info for virt-launcher pod " + f"{virt_launcher_pod_name} in namespace {namespace}" + ) + + target_container_id = None + for container in pod_info.containers: + if container.name == "compute" and container.ready and container.containerId: + target_container_id = re.sub(r".*://", "", container.containerId) + break + if not target_container_id: + raise Exception( + f"compute container in virt-launcher pod {virt_launcher_pod_name} " + f"in namespace {namespace} is not ready" + ) + + log_info( + f"targeting compute container {target_container_id}", + parallel, + network_chaos_pod_name, + ) + + pids = self.kubecli.get_lib_kubernetes().get_pod_pids( + base_pod_name=network_chaos_pod_name, + base_pod_namespace=namespace, + base_pod_container_name=container_name, + pod_name=virt_launcher_pod_name, + pod_namespace=namespace, + pod_container_id=target_container_id, + ) + if not pids: + raise Exception( + f"impossible to resolve PIDs for virt-launcher pod {virt_launcher_pod_name}" + ) + + log_info( + f"resolved PIDs {pids} on node {node_name} for VMI {vmi_name}", + parallel, + network_chaos_pod_name, + ) + + # Not all PIDs are in the virt-launcher's netns — find the right one. + netns_pid = find_virt_launcher_netns_pid( + network_chaos_pod_name, + namespace, + pids, + self.kubecli.get_lib_kubernetes(), + ) + if not netns_pid: + raise Exception( + f"could not find a PID in the virt-launcher netns for VMI {vmi_name}; " + "none of the compute container PIDs contain tap" + ) + + log_info( + f"using PID {netns_pid} for netns entry (virt-launcher netns confirmed via tap)", + parallel, + network_chaos_pod_name, + ) + + # Target the tap interface rather than the bridge slave (ovn-udn1-nic). + # Shaping the bridge slave also affects OVN's BFD heartbeats and can + # cause node-wide reconvergence; the tap device only connects to QEMU. + if len(scoped_config.interfaces) == 0: + iface = get_vmi_tap_interface( + network_chaos_pod_name, + namespace, + netns_pid, + self.kubecli.get_lib_kubernetes(), + ) + if not iface: + log_error( + "could not detect tap interface in virt-launcher netns; " + "impossible to execute the VMI network chaos scenario", + parallel, + network_chaos_pod_name, + ) + self._rollback(namespace, network_chaos_pod_name) + return + else: + iface = scoped_config.interfaces[0] + + log_info( + f"targeting tap interface: {iface}", + parallel, + network_chaos_pod_name, + ) + + # Apply tc-based shaping (HTB + netem) inside the virt-launcher netns. + # Passing pids=[netns_pid] wraps each tc command with nsenter so it + # targets the VMI's network namespace, not the host or chaos pod netns. + common_set_limit_rules( + self.config.egress, + self.config.ingress, + [iface], + self.config.bandwidth, + self.config.latency, + self.config.loss, + parallel, + vmi_name, + self.kubecli.get_lib_kubernetes(), + network_chaos_pod_name, + namespace, + pids=[netns_pid], + ) + + log_info( + f"waiting {self.config.test_duration} seconds before removing tc rules", + parallel, + network_chaos_pod_name, + ) + + time.sleep(self.config.test_duration) + + log_info("removing tc rules", parallel, network_chaos_pod_name) + + self._rollback(namespace, network_chaos_pod_name, netns_pid, iface, parallel, vmi_name) + + except Exception as e: + if network_chaos_pod_name: + self._rollback( + namespace, network_chaos_pod_name, netns_pid, iface, parallel, vmi_name + ) + if error_queue is None: + raise e + else: + error_queue.put(str(e)) + + def get_config(self) -> Tuple[NetworkChaosScenarioType, BaseNetworkChaosConfig]: + return NetworkChaosScenarioType.VMI, self.config + + def get_targets(self) -> list[str]: + if not self.config.namespace: + raise Exception("namespace not specified for VMI scenario, aborting") + name_regex = self.config.target if self.config.target else ".*" + label_selector = self.config.label_selector or None + + # A valid literal k8s namespace is lowercase alphanumerics and hyphens. + # If config.namespace contains regex metacharacters, pass "" so get_vmis + # lists from all namespaces; re.match() below handles the filtering. + api_namespace = ( + self.config.namespace + if re.fullmatch(r"[a-z0-9][a-z0-9\-]*", self.config.namespace) + else "" + ) + vmis = self.kubecli.get_lib_kubernetes().get_vmis( + name_regex, api_namespace, label_selector=label_selector + ) + return [ + f"{vmi['metadata']['namespace']}/{vmi['metadata']['name']}" + for vmi in vmis + if re.match(name_regex, vmi.get("metadata", {}).get("name", "")) + and re.match( + self.config.namespace, + vmi.get("metadata", {}).get("namespace", ""), + ) + ] diff --git a/krkn/scenario_plugins/network_chaos_ng/network_chaos_factory.py b/krkn/scenario_plugins/network_chaos_ng/network_chaos_factory.py index e6d16024..50a7a38a 100644 --- a/krkn/scenario_plugins/network_chaos_ng/network_chaos_factory.py +++ b/krkn/scenario_plugins/network_chaos_ng/network_chaos_factory.py @@ -36,6 +36,9 @@ from krkn.scenario_plugins.network_chaos_ng.modules.pod_network_chaos import ( from krkn.scenario_plugins.network_chaos_ng.modules.pod_network_filter import ( PodNetworkFilterModule, ) +from krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos import ( + VmiNetworkChaosModule, +) supported_modules = [ "node_network_filter", @@ -43,6 +46,7 @@ supported_modules = [ "pod_network_chaos", "node_network_chaos", "node_interface_down", + "vmi_network_chaos", ] @@ -87,5 +91,11 @@ class NetworkChaosFactory: if len(errors) > 0: raise Exception(f"config validation errors: [{';'.join(errors)}]") return NodeInterfaceDownModule(scenario_config, kubecli) + if config["id"] == "vmi_network_chaos": + scenario_config = NetworkChaosConfig(**config) + errors = scenario_config.validate() + if len(errors) > 0: + raise Exception(f"config validation errors: [{';'.join(errors)}]") + return VmiNetworkChaosModule(scenario_config, kubecli) else: raise Exception(f"invalid network chaos id {config['id']}") diff --git a/scenarios/openshift/virt_network_chaos.yaml b/scenarios/openshift/virt_network_chaos.yaml new file mode 100644 index 00000000..baf4cafa --- /dev/null +++ b/scenarios/openshift/virt_network_chaos.yaml @@ -0,0 +1,17 @@ +- id: vmi_network_chaos + image: "quay.io/krkn-chaos/krkn-network-chaos:latest" + wait_duration: 300 + test_duration: 120 + label_selector: "" + service_account: "" + taints: [] + namespace: "" + instance_count: 1 + execution: serial + target: ".*" + interfaces: [] + ingress: true + egress: true + latency: "100ms" + loss: "10" + bandwidth: "100mbit" diff --git a/tests/test_utils_network_chaos.py b/tests/test_utils_network_chaos.py index 28385fd2..84a5dd72 100644 --- a/tests/test_utils_network_chaos.py +++ b/tests/test_utils_network_chaos.py @@ -22,12 +22,54 @@ from krkn.scenario_plugins.network_chaos_ng.modules.utils_network_chaos import ( node_qdisc_is_simple, common_set_limit_rules, common_delete_limit_rules, + _normalize_rate, + _normalize_delay, + _normalize_loss, ROOT_HANDLE, CLASS_ID, NETEM_HANDLE, ) +class TestNormalizers(unittest.TestCase): + + def test_normalize_rate_bare_integer(self): + self.assertEqual(_normalize_rate("100"), "100mbit") + + def test_normalize_rate_bare_float(self): + self.assertEqual(_normalize_rate("1.5"), "1.5mbit") + + def test_normalize_rate_with_unit(self): + self.assertEqual(_normalize_rate("100mbit"), "100mbit") + + def test_normalize_rate_gbit(self): + self.assertEqual(_normalize_rate("1gbit"), "1gbit") + + def test_normalize_rate_none(self): + self.assertEqual(_normalize_rate(None), "1gbit") + + def test_normalize_delay_bare_integer(self): + self.assertEqual(_normalize_delay("50"), "50ms") + + def test_normalize_delay_bare_float(self): + self.assertEqual(_normalize_delay("1.5"), "1.5ms") + + def test_normalize_delay_with_unit(self): + self.assertEqual(_normalize_delay("50ms"), "50ms") + + def test_normalize_delay_none(self): + self.assertEqual(_normalize_delay(None), "0ms") + + def test_normalize_loss_bare(self): + self.assertEqual(_normalize_loss("10"), "10") + + def test_normalize_loss_with_pct(self): + self.assertEqual(_normalize_loss("10%"), "10") + + def test_normalize_loss_none(self): + self.assertEqual(_normalize_loss(None), "0") + + class TestBuildTcTreeCommands(unittest.TestCase): def test_build_tc_tree_single_interface(self): @@ -150,6 +192,22 @@ class TestEgressShapingCommands(unittest.TestCase): result, ) + def test_egress_shaping_with_suffixed_params(self): + """ + Test that pre-suffixed values (e.g. "100mbit", "50ms", "10%") are passed through unchanged. + """ + devices = ["eth0"] + result = get_egress_shaping_comand(devices, "100mbit", "50ms", "10%") + + self.assertIn( + "tc class change dev eth0 parent 100: classid 100:1 htb rate 100mbit", + result, + ) + self.assertIn( + "tc qdisc change dev eth0 parent 100:1 handle 101: netem delay 50ms loss 10%", + result, + ) + def test_egress_shaping_multiple_interfaces(self): """ Test egress shaping for multiple interfaces diff --git a/tests/test_vmi_network_chaos.py b/tests/test_vmi_network_chaos.py new file mode 100644 index 00000000..da796b15 --- /dev/null +++ b/tests/test_vmi_network_chaos.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 + +""" +Test suite for VmiNetworkChaosModule + +Usage: + python -m unittest tests/test_vmi_network_chaos.py -v + python -m coverage run -a -m unittest tests/test_vmi_network_chaos.py -v +""" + +import queue +import unittest +from unittest.mock import MagicMock, patch + +from krkn.scenario_plugins.network_chaos_ng.models import ( + NetworkChaosScenarioType, + NetworkChaosConfig, +) +from krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos import ( + VmiNetworkChaosModule, +) + +MODULE = "krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos" + + +def _make_config(**overrides): + defaults = dict( + id="vmi_network_chaos", + image="quay.io/krkn-chaos/krkn-network-chaos:latest", + wait_duration=300, + test_duration=60, + label_selector="", + service_account="", + taints=[], + namespace="virt-density-udn-3", + instance_count=1, + execution="serial", + target=".*", + interfaces=[], + ingress=True, + egress=True, + latency="100ms", + loss="10", + bandwidth="100mbit", + ) + defaults.update(overrides) + return NetworkChaosConfig(**defaults) + + +def _make_container(name, ready=True, container_id="containerd://abc123"): + c = MagicMock() + c.name = name + c.ready = ready + c.containerId = container_id + return c + + +class TestVmiNetworkChaosModuleInit(unittest.TestCase): + + def setUp(self): + self.mock_kubecli = MagicMock() + self.config = _make_config() + self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli) + + def test_initialization(self): + self.assertEqual(self.module.config, self.config) + self.assertEqual(self.module.kubecli, self.mock_kubecli) + + def test_get_config(self): + scenario_type, config = self.module.get_config() + self.assertEqual(scenario_type, NetworkChaosScenarioType.VMI) + self.assertEqual(config, self.config) + + +class TestVmiNetworkChaosModuleGetTargets(unittest.TestCase): + + def setUp(self): + self.mock_kubecli = MagicMock() + self.mock_kubernetes = MagicMock() + self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes + self.config = _make_config( + namespace="virt-density-udn-3", + target="virt-server-.*", + ) + self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli) + + def test_get_targets_success(self): + vmis = [ + {"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}}, + {"metadata": {"name": "virt-server-2", "namespace": "virt-density-udn-3"}}, + ] + self.mock_kubernetes.get_vmis.return_value = vmis + + result = self.module.get_targets() + + self.assertEqual( + result, + [ + "virt-density-udn-3/virt-server-1", + "virt-density-udn-3/virt-server-2", + ], + ) + self.mock_kubernetes.get_vmis.assert_called_once_with( + "virt-server-.*", "virt-density-udn-3", label_selector=None + ) + + def test_get_targets_no_namespace_raises(self): + self.config.namespace = None + with self.assertRaises(Exception) as ctx: + self.module.get_targets() + self.assertIn("namespace not specified", str(ctx.exception)) + + def test_get_targets_no_vmis_returns_empty(self): + self.mock_kubernetes.get_vmis.return_value = [] + result = self.module.get_targets() + self.assertEqual(result, []) + + def test_get_targets_regex_filters_namespace(self): + vmis = [ + {"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}}, + {"metadata": {"name": "virt-server-2", "namespace": "other-namespace"}}, + ] + self.mock_kubernetes.get_vmis.return_value = vmis + result = self.module.get_targets() + self.assertIn("virt-density-udn-3/virt-server-1", result) + self.assertNotIn("other-namespace/virt-server-2", result) + + def test_get_targets_passes_label_selector(self): + self.config.label_selector = "app=myapp" + self.mock_kubernetes.get_vmis.return_value = [] + self.module.get_targets() + self.mock_kubernetes.get_vmis.assert_called_once_with( + "virt-server-.*", "virt-density-udn-3", label_selector="app=myapp" + ) + + def test_get_targets_empty_label_selector_passes_none(self): + self.config.label_selector = "" + self.mock_kubernetes.get_vmis.return_value = [] + self.module.get_targets() + self.mock_kubernetes.get_vmis.assert_called_once_with( + "virt-server-.*", "virt-density-udn-3", label_selector=None + ) + + def test_get_targets_regex_namespace_passes_empty_string_to_api(self): + """When namespace is a regex, get_vmis must be called with "" (all namespaces) + so the k8s API isn't asked to look up a literal namespace that doesn't exist.""" + self.config.namespace = "virt-density-.*" + vmis = [ + {"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}}, + {"metadata": {"name": "virt-server-2", "namespace": "other-namespace"}}, + ] + self.mock_kubernetes.get_vmis.return_value = vmis + result = self.module.get_targets() + + self.mock_kubernetes.get_vmis.assert_called_once_with( + "virt-server-.*", "", label_selector=None + ) + self.assertIn("virt-density-udn-3/virt-server-1", result) + self.assertNotIn("other-namespace/virt-server-2", result) + + +class TestVmiNetworkChaosModuleRun(unittest.TestCase): + + def setUp(self): + self.mock_kubecli = MagicMock() + self.mock_kubernetes = MagicMock() + self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes + self.config = _make_config( + namespace="virt-density-udn-.*", + target="virt-server-.*", + test_duration=60, + interfaces=[], + latency="100ms", + loss="10", + bandwidth="100mbit", + ) + self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli) + + self.mock_kubernetes.get_vmi.return_value = { + "status": {"nodeName": "worker-1"} + } + self.mock_kubernetes.list_pods.return_value = [ + "virt-launcher-virt-server-3-abc12" + ] + + compute = _make_container("compute", ready=True, container_id="containerd://deadbeef") + virt_launcher = _make_container("virt-launcher", ready=False, container_id="") + mock_pod_info = MagicMock() + mock_pod_info.containers = [virt_launcher, compute] + self.mock_kubernetes.get_pod_info.return_value = mock_pod_info + self.mock_kubernetes.get_pod_pids.return_value = ["100", "101", "102"] + + # ------------------------------------------------------------------ success + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.common_set_limit_rules") + @patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0") + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.time.sleep") + @patch(f"{MODULE}.log_info") + def test_run_success( + self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del + ): + self.module.run("virt-density-udn-3/virt-server-3") + + mock_deploy.assert_called_once() + mock_find.assert_called_once() + mock_tap.assert_called_once() + mock_set.assert_called_once() + mock_sleep.assert_called_once_with(60) + mock_del.assert_called_once() + self.mock_kubernetes.delete_pod.assert_called_once() + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.common_set_limit_rules") + @patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0") + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.time.sleep") + @patch(f"{MODULE}.log_info") + def test_run_uses_resolved_namespace_not_regex( + self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del + ): + """Kubernetes calls must use the real namespace, not the regex pattern.""" + self.module.run("virt-density-udn-3/virt-server-3") + + self.mock_kubernetes.get_vmi.assert_called_once_with( + "virt-server-3", "virt-density-udn-3" + ) + deploy_config = mock_deploy.call_args[0][0] + self.assertEqual(deploy_config.namespace, "virt-density-udn-3") + self.assertNotEqual(deploy_config.namespace, "virt-density-udn-.*") + + # ------------------------------------------------------------------ chaos config passed correctly + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.common_set_limit_rules") + @patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0") + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.time.sleep") + @patch(f"{MODULE}.log_info") + def test_run_passes_latency_loss_bandwidth_to_set_limit_rules( + self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del + ): + self.module.run("virt-density-udn-3/virt-server-3") + + call_kwargs = { + k: v + for k, v in zip( + ["egress", "ingress", "interfaces", "bandwidth", "latency", "loss"], + mock_set.call_args[0], + ) + } + self.assertEqual(call_kwargs["latency"], "100ms") + self.assertEqual(call_kwargs["loss"], "10") + self.assertEqual(call_kwargs["bandwidth"], "100mbit") + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.common_set_limit_rules") + @patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0") + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.time.sleep") + @patch(f"{MODULE}.log_info") + def test_run_passes_netns_pid_as_pids_list( + self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del + ): + """common_set_limit_rules must receive [netns_pid], not the full pids list.""" + self.module.run("virt-density-udn-3/virt-server-3") + + call_kwargs = mock_set.call_args[1] + self.assertEqual(call_kwargs["pids"], ["101"]) + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.common_set_limit_rules") + @patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0") + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.time.sleep") + @patch(f"{MODULE}.log_info") + def test_run_passes_tap_iface_as_interfaces_list( + self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del + ): + """common_set_limit_rules must receive [iface], not config.interfaces.""" + self.module.run("virt-density-udn-3/virt-server-3") + + iface_arg = mock_set.call_args[0][2] + self.assertEqual(iface_arg, ["tap0"]) + + # ------------------------------------------------------------------ error paths + + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_vmi_not_found_raises(self, mock_log, mock_deploy): + self.mock_kubernetes.get_vmi.return_value = None + with self.assertRaises(Exception) as ctx: + self.module.run("virt-density-udn-3/virt-server-3") + self.assertIn("not found", str(ctx.exception)) + + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_vmi_no_node_raises(self, mock_log, mock_deploy): + self.mock_kubernetes.get_vmi.return_value = {"status": {}} + with self.assertRaises(Exception) as ctx: + self.module.run("virt-density-udn-3/virt-server-3") + self.assertIn("unable to determine node", str(ctx.exception)) + + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_no_virt_launcher_pod_raises(self, mock_log, mock_deploy): + self.mock_kubernetes.list_pods.return_value = [] + with self.assertRaises(Exception) as ctx: + self.module.run("virt-density-udn-3/virt-server-3") + self.assertIn("no virt-launcher pod found", str(ctx.exception)) + + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_no_pod_info_raises(self, mock_log, mock_deploy): + self.mock_kubernetes.get_pod_info.return_value = None + with self.assertRaises(Exception) as ctx: + self.module.run("virt-density-udn-3/virt-server-3") + self.assertIn("impossible to retrieve info", str(ctx.exception)) + + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_no_pids_raises(self, mock_log, mock_deploy): + self.mock_kubernetes.get_pod_pids.return_value = None + with self.assertRaises(Exception) as ctx: + self.module.run("virt-density-udn-3/virt-server-3") + self.assertIn("impossible to resolve PIDs", str(ctx.exception)) + + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value=None) + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_no_netns_pid_raises(self, mock_log, mock_deploy, mock_find): + with self.assertRaises(Exception) as ctx: + self.module.run("virt-density-udn-3/virt-server-3") + self.assertIn("could not find a PID", str(ctx.exception)) + + # ------------------------------------------------------------------ error queue + + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_error_queue_captures_exception(self, mock_log, mock_deploy): + self.mock_kubernetes.get_vmi.return_value = None + error_queue = queue.Queue() + self.module.run("virt-density-udn-3/virt-server-3", error_queue) + self.assertFalse(error_queue.empty()) + self.assertIn("not found", error_queue.get()) + + +class TestVmiNetworkChaosModuleRollback(unittest.TestCase): + + def setUp(self): + self.mock_kubecli = MagicMock() + self.mock_kubernetes = MagicMock() + self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes + self.config = _make_config( + namespace="virt-density-udn-3", + target="virt-server-.*", + test_duration=60, + interfaces=[], + ) + self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli) + + self.mock_kubernetes.get_vmi.return_value = {"status": {"nodeName": "worker-1"}} + self.mock_kubernetes.list_pods.return_value = ["virt-launcher-virt-server-3-abc12"] + compute = _make_container("compute", ready=True, container_id="containerd://deadbeef") + mock_pod_info = MagicMock() + mock_pod_info.containers = [compute] + self.mock_kubernetes.get_pod_info.return_value = mock_pod_info + self.mock_kubernetes.get_pod_pids.return_value = ["100", "101", "102"] + + def test_rollback_calls_delete_limit_rules_then_delete_when_chaos_applied(self): + with patch(f"{MODULE}.common_delete_limit_rules") as mock_del: + self.module._rollback("ns", "chaos-pod", "101", "tap0") + mock_del.assert_called_once() + del_args = mock_del.call_args[0] + self.assertEqual(del_args[2], ["tap0"]) # interfaces + self.assertEqual(del_args[6], ["101"]) # pids + self.mock_kubernetes.delete_pod.assert_called_once_with("chaos-pod", "ns") + + def test_rollback_skips_delete_limit_rules_when_chaos_not_applied(self): + with patch(f"{MODULE}.common_delete_limit_rules") as mock_del: + self.module._rollback("ns", "chaos-pod") + mock_del.assert_not_called() + self.mock_kubernetes.delete_pod.assert_called_once_with("chaos-pod", "ns") + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.log_info") + def test_run_rollback_deletes_pod_on_error_before_chaos( + self, mock_log, mock_deploy, mock_del + ): + """Pod deployed but setup fails before chaos: delete only, no limit rules.""" + self.mock_kubernetes.get_pod_info.return_value = None + with self.assertRaises(Exception): + self.module.run("virt-density-udn-3/virt-server-3") + self.mock_kubernetes.delete_pod.assert_called_once() + mock_del.assert_not_called() + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.common_set_limit_rules") + @patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0") + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.time.sleep") + @patch(f"{MODULE}.log_info") + def test_run_rollback_calls_delete_limit_rules_on_error_after_chaos( + self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del + ): + """If interrupted after chaos is applied, delete_limit_rules and delete_pod called.""" + mock_sleep.side_effect = RuntimeError("interrupted") + with self.assertRaises(RuntimeError): + self.module.run("virt-density-udn-3/virt-server-3") + mock_del.assert_called_once() + self.mock_kubernetes.delete_pod.assert_called_once() + + @patch(f"{MODULE}.common_delete_limit_rules") + @patch(f"{MODULE}.common_set_limit_rules") + @patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0") + @patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101") + @patch(f"{MODULE}.deploy_network_chaos_ng_pod") + @patch(f"{MODULE}.time.sleep") + @patch(f"{MODULE}.log_info") + def test_run_rollback_passes_correct_pid_and_iface_on_error( + self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del + ): + mock_sleep.side_effect = RuntimeError("interrupted") + with self.assertRaises(RuntimeError): + self.module.run("virt-density-udn-3/virt-server-3") + del_args = mock_del.call_args[0] + self.assertEqual(del_args[2], ["tap0"]) # interfaces + self.assertEqual(del_args[6], ["101"]) # pids + + +if __name__ == "__main__": + unittest.main()