adding start of vmi network scenario (#1260)

Assisted By: Claude Code:
Assisted By: Claude Code:

Signed-off-by: Paige Patton <prubenda@redhat.com>
This commit is contained in:
Paige Patton
2026-05-13 13:20:08 -04:00
committed by GitHub
parent 5dc79789a3
commit ebe6049be9
9 changed files with 930 additions and 8 deletions
+1
View File
@@ -53,6 +53,7 @@ kraken:
- scenarios/kube/node-network-chaos.yml
- scenarios/kube/pod-network-chaos.yml
- scenarios/kube/node_interface_down.yaml
- scenarios/openshift/virt_network_chaos.yaml
- kubevirt_vm_outage:
- scenarios/kubevirt/kubevirt-vm-outage.yaml
- http_load_scenarios:
@@ -20,6 +20,7 @@ from typing import TypeVar, Optional
class NetworkChaosScenarioType(Enum):
Node = 1
Pod = 2
VMI = 3
@dataclass
@@ -13,7 +13,7 @@
# limitations under the License.
import logging
import os
from typing import Tuple
from typing import Optional, Tuple
import yaml
from jinja2 import FileSystemLoader, Environment
@@ -110,6 +110,57 @@ def get_pod_default_interface(
return output.replace("\n", "")
def find_virt_launcher_netns_pid(
chaos_pod_name: str,
namespace: str,
pids: list[str],
kubecli: KrknKubernetes,
) -> Optional[str]:
"""Return the first PID from `pids` whose netns contains a tap device.
Not all PIDs returned by get_pod_pids are inside the virt-launcher network
namespace — some helper processes run in the host netns. Entering one of
those would target the node's physical NIC instead of the bridge slave
inside the virt-launcher netns.
"""
for pid in pids:
try:
result = kubecli.exec_cmd_in_pod(
[f"nsenter --target {pid} --net -- ip link show type tun"],
chaos_pod_name,
namespace,
)
if result and "tap" in result:
return pid
except Exception as e:
logging.warning(f"failed to check netns for PID {pid}: {e}")
continue
return None
def get_vmi_tap_interface(
chaos_pod_name: str,
namespace: str,
netns_pid: str,
kubecli: KrknKubernetes,
) -> Optional[str]:
"""Return the name of the tap device inside the virt-launcher netns."""
result = kubecli.exec_cmd_in_pod(
[f"nsenter --target {netns_pid} --net -- ip -o link show type tun"],
chaos_pod_name,
namespace,
)
if not result:
return None
for line in result.splitlines():
parts = line.split(":")
if len(parts) >= 2:
iface = parts[1].strip().split("@")[0].strip()
if iface.startswith("tap"):
return iface
return None
def setup_network_chaos_ng_scenario(
config: BaseNetworkChaosConfig,
node_name: str,
@@ -28,6 +28,37 @@ CLASS_ID = "100:1"
NETEM_HANDLE = "101:"
def _normalize_rate(value: Optional[str]) -> str:
"""Accept "100" or "100mbit"/"1gbit"; always return a tc-valid rate string."""
if value is None:
return "1gbit"
s = value.strip()
try:
float(s)
return f"{s}mbit"
except ValueError:
return s
def _normalize_delay(value: Optional[str]) -> str:
"""Accept "50" or "50ms"; always return a tc-valid delay string."""
if value is None:
return "0ms"
s = value.strip()
try:
float(s)
return f"{s}ms"
except ValueError:
return s
def _normalize_loss(value: Optional[str]) -> str:
"""Accept "10" or "10%"; return the bare numeric string (% appended by caller)."""
if value is None:
return "0"
return value.strip().rstrip("%")
def run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(cmd, check=check, text=True, capture_output=True)
@@ -64,15 +95,15 @@ def get_egress_shaping_comand(
) -> list[str]:
rate_commands = []
rate = f"{rate_mbit}mbit" if rate_mbit is not None else "1gbit"
d = delay_ms if delay_ms is not None else 0
l = loss_pct if loss_pct is not None else 0
rate = _normalize_rate(rate_mbit)
d = _normalize_delay(delay_ms)
l = _normalize_loss(loss_pct)
for dev in devices:
rate_commands.append(
f"tc class change dev {dev} parent {ROOT_HANDLE} classid {CLASS_ID} htb rate {rate}"
)
rate_commands.append(
f"tc qdisc change dev {dev} parent {CLASS_ID} handle {NETEM_HANDLE} netem delay {d}ms loss {l}%"
f"tc qdisc change dev {dev} parent {CLASS_ID} handle {NETEM_HANDLE} netem delay {d} loss {l}%"
)
return rate_commands
@@ -108,12 +139,12 @@ def get_ingress_shaping_commands(
)
rate_commands.append(
f"tc class add dev {ifb_dev} parent {ROOT_HANDLE} classid {CLASS_ID} "
f"htb rate {rate_mbit if rate_mbit else '1gbit'} || true"
f"htb rate {_normalize_rate(rate_mbit)} || true"
)
rate_commands.append(
f"tc qdisc add dev {ifb_dev} parent {CLASS_ID} handle {NETEM_HANDLE} "
f"netem delay {delay_ms if delay_ms else '0ms'} "
f"loss {loss_pct if loss_pct else '0'}% || true"
f"netem delay {_normalize_delay(delay_ms)} "
f"loss {_normalize_loss(loss_pct)}% || true"
)
return rate_commands
@@ -0,0 +1,313 @@
# Copyright 2025 The Krkn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import dataclasses
import queue
import re
import time
from typing import Tuple
from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
from krkn_lib.utils import get_random_string
from krkn.scenario_plugins.network_chaos_ng.models import (
NetworkChaosScenarioType,
BaseNetworkChaosConfig,
NetworkChaosConfig,
)
from krkn.scenario_plugins.network_chaos_ng.modules.abstract_network_chaos_module import (
AbstractNetworkChaosModule,
)
from krkn.scenario_plugins.network_chaos_ng.modules.utils import (
log_info,
log_error,
deploy_network_chaos_ng_pod,
find_virt_launcher_netns_pid,
get_vmi_tap_interface,
)
from krkn.scenario_plugins.network_chaos_ng.modules.utils_network_chaos import (
common_set_limit_rules,
common_delete_limit_rules,
)
class VmiNetworkChaosModule(AbstractNetworkChaosModule):
def __init__(self, config: NetworkChaosConfig, kubecli: KrknTelemetryOpenshift):
super().__init__(config, kubecli)
self.config = config
def _rollback(
self,
namespace: str,
network_chaos_pod_name: str,
netns_pid: str = None,
iface: str = None,
parallel: bool = False,
vmi_name: str = "",
):
if netns_pid and iface:
common_delete_limit_rules(
self.config.egress,
self.config.ingress,
[iface],
network_chaos_pod_name,
namespace,
self.kubecli.get_lib_kubernetes(),
[netns_pid],
parallel,
vmi_name,
)
self.kubecli.get_lib_kubernetes().delete_pod(
network_chaos_pod_name, namespace
)
def run(self, target: str, error_queue: queue.Queue = None):
# target is "namespace/vmi-name" as produced by get_targets()
parallel = False
if error_queue:
parallel = True
network_chaos_pod_name = None
netns_pid = None
iface = None
namespace = ""
vmi_name = ""
try:
namespace, vmi_name = target.split("/", 1)
# Create a scoped config with the resolved namespace so that all
# Kubernetes calls use the actual namespace, not the regex pattern.
scoped_config = dataclasses.replace(self.config, namespace=namespace)
network_chaos_pod_name = f"vmi-network-chaos-{get_random_string(5)}"
container_name = f"fedora-container-{get_random_string(5)}"
log_info(
f"creating workload to inject network chaos in VMI {vmi_name} "
f"latency:{str(self.config.latency) if self.config.latency else '0'}, "
f"loss:{str(self.config.loss) if self.config.loss else '0'}, "
f"bandwidth:{str(self.config.bandwidth) if self.config.bandwidth else '0'}",
parallel,
network_chaos_pod_name,
)
vmi = self.kubecli.get_lib_kubernetes().get_vmi(vmi_name, namespace)
if not vmi:
raise Exception(
f"VMI {vmi_name} not found in namespace {namespace}"
)
node_name = vmi.get("status", {}).get("nodeName")
if not node_name:
raise Exception(
f"unable to determine node for VMI {vmi_name} in namespace {namespace}; "
"VMI may not be in Running phase"
)
log_info(
f"VMI {vmi_name} is running on node {node_name}",
parallel,
network_chaos_pod_name,
)
# The virt-launcher pod carries the VMI's network namespace.
virt_launcher_pods = self.kubecli.get_lib_kubernetes().list_pods(
namespace, label_selector=f"vm.kubevirt.io/name={vmi_name}"
)
if not virt_launcher_pods:
raise Exception(
f"no virt-launcher pod found for VMI {vmi_name} in namespace {namespace}"
)
virt_launcher_pod_name = virt_launcher_pods[0]
log_info(
f"resolved virt-launcher pod {virt_launcher_pod_name} for VMI {vmi_name}",
parallel,
network_chaos_pod_name,
)
# Deploy the privileged chaos pod onto the VMI's node.
# hostPID=True (via template) allows nsenter into the virt-launcher's
# network namespace using any of the compute container's host PIDs.
deploy_network_chaos_ng_pod(
scoped_config,
node_name,
network_chaos_pod_name,
self.kubecli.get_lib_kubernetes(),
container_name,
host_network=False,
)
# Prefer the 'compute' container (the QEMU process in KubeVirt).
# 'virt-launcher' is a sidecar monitor that may not be running.
pod_info = self.kubecli.get_lib_kubernetes().get_pod_info(
virt_launcher_pod_name, namespace
)
if not pod_info:
raise Exception(
f"impossible to retrieve info for virt-launcher pod "
f"{virt_launcher_pod_name} in namespace {namespace}"
)
target_container_id = None
for container in pod_info.containers:
if container.name == "compute" and container.ready and container.containerId:
target_container_id = re.sub(r".*://", "", container.containerId)
break
if not target_container_id:
raise Exception(
f"compute container in virt-launcher pod {virt_launcher_pod_name} "
f"in namespace {namespace} is not ready"
)
log_info(
f"targeting compute container {target_container_id}",
parallel,
network_chaos_pod_name,
)
pids = self.kubecli.get_lib_kubernetes().get_pod_pids(
base_pod_name=network_chaos_pod_name,
base_pod_namespace=namespace,
base_pod_container_name=container_name,
pod_name=virt_launcher_pod_name,
pod_namespace=namespace,
pod_container_id=target_container_id,
)
if not pids:
raise Exception(
f"impossible to resolve PIDs for virt-launcher pod {virt_launcher_pod_name}"
)
log_info(
f"resolved PIDs {pids} on node {node_name} for VMI {vmi_name}",
parallel,
network_chaos_pod_name,
)
# Not all PIDs are in the virt-launcher's netns — find the right one.
netns_pid = find_virt_launcher_netns_pid(
network_chaos_pod_name,
namespace,
pids,
self.kubecli.get_lib_kubernetes(),
)
if not netns_pid:
raise Exception(
f"could not find a PID in the virt-launcher netns for VMI {vmi_name}; "
"none of the compute container PIDs contain tap"
)
log_info(
f"using PID {netns_pid} for netns entry (virt-launcher netns confirmed via tap)",
parallel,
network_chaos_pod_name,
)
# Target the tap interface rather than the bridge slave (ovn-udn1-nic).
# Shaping the bridge slave also affects OVN's BFD heartbeats and can
# cause node-wide reconvergence; the tap device only connects to QEMU.
if len(scoped_config.interfaces) == 0:
iface = get_vmi_tap_interface(
network_chaos_pod_name,
namespace,
netns_pid,
self.kubecli.get_lib_kubernetes(),
)
if not iface:
log_error(
"could not detect tap interface in virt-launcher netns; "
"impossible to execute the VMI network chaos scenario",
parallel,
network_chaos_pod_name,
)
self._rollback(namespace, network_chaos_pod_name)
return
else:
iface = scoped_config.interfaces[0]
log_info(
f"targeting tap interface: {iface}",
parallel,
network_chaos_pod_name,
)
# Apply tc-based shaping (HTB + netem) inside the virt-launcher netns.
# Passing pids=[netns_pid] wraps each tc command with nsenter so it
# targets the VMI's network namespace, not the host or chaos pod netns.
common_set_limit_rules(
self.config.egress,
self.config.ingress,
[iface],
self.config.bandwidth,
self.config.latency,
self.config.loss,
parallel,
vmi_name,
self.kubecli.get_lib_kubernetes(),
network_chaos_pod_name,
namespace,
pids=[netns_pid],
)
log_info(
f"waiting {self.config.test_duration} seconds before removing tc rules",
parallel,
network_chaos_pod_name,
)
time.sleep(self.config.test_duration)
log_info("removing tc rules", parallel, network_chaos_pod_name)
self._rollback(namespace, network_chaos_pod_name, netns_pid, iface, parallel, vmi_name)
except Exception as e:
if network_chaos_pod_name:
self._rollback(
namespace, network_chaos_pod_name, netns_pid, iface, parallel, vmi_name
)
if error_queue is None:
raise e
else:
error_queue.put(str(e))
def get_config(self) -> Tuple[NetworkChaosScenarioType, BaseNetworkChaosConfig]:
return NetworkChaosScenarioType.VMI, self.config
def get_targets(self) -> list[str]:
if not self.config.namespace:
raise Exception("namespace not specified for VMI scenario, aborting")
name_regex = self.config.target if self.config.target else ".*"
label_selector = self.config.label_selector or None
# A valid literal k8s namespace is lowercase alphanumerics and hyphens.
# If config.namespace contains regex metacharacters, pass "" so get_vmis
# lists from all namespaces; re.match() below handles the filtering.
api_namespace = (
self.config.namespace
if re.fullmatch(r"[a-z0-9][a-z0-9\-]*", self.config.namespace)
else ""
)
vmis = self.kubecli.get_lib_kubernetes().get_vmis(
name_regex, api_namespace, label_selector=label_selector
)
return [
f"{vmi['metadata']['namespace']}/{vmi['metadata']['name']}"
for vmi in vmis
if re.match(name_regex, vmi.get("metadata", {}).get("name", ""))
and re.match(
self.config.namespace,
vmi.get("metadata", {}).get("namespace", ""),
)
]
@@ -36,6 +36,9 @@ from krkn.scenario_plugins.network_chaos_ng.modules.pod_network_chaos import (
from krkn.scenario_plugins.network_chaos_ng.modules.pod_network_filter import (
PodNetworkFilterModule,
)
from krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos import (
VmiNetworkChaosModule,
)
supported_modules = [
"node_network_filter",
@@ -43,6 +46,7 @@ supported_modules = [
"pod_network_chaos",
"node_network_chaos",
"node_interface_down",
"vmi_network_chaos",
]
@@ -87,5 +91,11 @@ class NetworkChaosFactory:
if len(errors) > 0:
raise Exception(f"config validation errors: [{';'.join(errors)}]")
return NodeInterfaceDownModule(scenario_config, kubecli)
if config["id"] == "vmi_network_chaos":
scenario_config = NetworkChaosConfig(**config)
errors = scenario_config.validate()
if len(errors) > 0:
raise Exception(f"config validation errors: [{';'.join(errors)}]")
return VmiNetworkChaosModule(scenario_config, kubecli)
else:
raise Exception(f"invalid network chaos id {config['id']}")
@@ -0,0 +1,17 @@
- id: vmi_network_chaos
image: "quay.io/krkn-chaos/krkn-network-chaos:latest"
wait_duration: 300
test_duration: 120
label_selector: ""
service_account: ""
taints: []
namespace: ""
instance_count: 1
execution: serial
target: ".*"
interfaces: []
ingress: true
egress: true
latency: "100ms"
loss: "10"
bandwidth: "100mbit"
+58
View File
@@ -22,12 +22,54 @@ from krkn.scenario_plugins.network_chaos_ng.modules.utils_network_chaos import (
node_qdisc_is_simple,
common_set_limit_rules,
common_delete_limit_rules,
_normalize_rate,
_normalize_delay,
_normalize_loss,
ROOT_HANDLE,
CLASS_ID,
NETEM_HANDLE,
)
class TestNormalizers(unittest.TestCase):
def test_normalize_rate_bare_integer(self):
self.assertEqual(_normalize_rate("100"), "100mbit")
def test_normalize_rate_bare_float(self):
self.assertEqual(_normalize_rate("1.5"), "1.5mbit")
def test_normalize_rate_with_unit(self):
self.assertEqual(_normalize_rate("100mbit"), "100mbit")
def test_normalize_rate_gbit(self):
self.assertEqual(_normalize_rate("1gbit"), "1gbit")
def test_normalize_rate_none(self):
self.assertEqual(_normalize_rate(None), "1gbit")
def test_normalize_delay_bare_integer(self):
self.assertEqual(_normalize_delay("50"), "50ms")
def test_normalize_delay_bare_float(self):
self.assertEqual(_normalize_delay("1.5"), "1.5ms")
def test_normalize_delay_with_unit(self):
self.assertEqual(_normalize_delay("50ms"), "50ms")
def test_normalize_delay_none(self):
self.assertEqual(_normalize_delay(None), "0ms")
def test_normalize_loss_bare(self):
self.assertEqual(_normalize_loss("10"), "10")
def test_normalize_loss_with_pct(self):
self.assertEqual(_normalize_loss("10%"), "10")
def test_normalize_loss_none(self):
self.assertEqual(_normalize_loss(None), "0")
class TestBuildTcTreeCommands(unittest.TestCase):
def test_build_tc_tree_single_interface(self):
@@ -150,6 +192,22 @@ class TestEgressShapingCommands(unittest.TestCase):
result,
)
def test_egress_shaping_with_suffixed_params(self):
"""
Test that pre-suffixed values (e.g. "100mbit", "50ms", "10%") are passed through unchanged.
"""
devices = ["eth0"]
result = get_egress_shaping_comand(devices, "100mbit", "50ms", "10%")
self.assertIn(
"tc class change dev eth0 parent 100: classid 100:1 htb rate 100mbit",
result,
)
self.assertIn(
"tc qdisc change dev eth0 parent 100:1 handle 101: netem delay 50ms loss 10%",
result,
)
def test_egress_shaping_multiple_interfaces(self):
"""
Test egress shaping for multiple interfaces
+440
View File
@@ -0,0 +1,440 @@
#!/usr/bin/env python3
"""
Test suite for VmiNetworkChaosModule
Usage:
python -m unittest tests/test_vmi_network_chaos.py -v
python -m coverage run -a -m unittest tests/test_vmi_network_chaos.py -v
"""
import queue
import unittest
from unittest.mock import MagicMock, patch
from krkn.scenario_plugins.network_chaos_ng.models import (
NetworkChaosScenarioType,
NetworkChaosConfig,
)
from krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos import (
VmiNetworkChaosModule,
)
MODULE = "krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos"
def _make_config(**overrides):
defaults = dict(
id="vmi_network_chaos",
image="quay.io/krkn-chaos/krkn-network-chaos:latest",
wait_duration=300,
test_duration=60,
label_selector="",
service_account="",
taints=[],
namespace="virt-density-udn-3",
instance_count=1,
execution="serial",
target=".*",
interfaces=[],
ingress=True,
egress=True,
latency="100ms",
loss="10",
bandwidth="100mbit",
)
defaults.update(overrides)
return NetworkChaosConfig(**defaults)
def _make_container(name, ready=True, container_id="containerd://abc123"):
c = MagicMock()
c.name = name
c.ready = ready
c.containerId = container_id
return c
class TestVmiNetworkChaosModuleInit(unittest.TestCase):
def setUp(self):
self.mock_kubecli = MagicMock()
self.config = _make_config()
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
def test_initialization(self):
self.assertEqual(self.module.config, self.config)
self.assertEqual(self.module.kubecli, self.mock_kubecli)
def test_get_config(self):
scenario_type, config = self.module.get_config()
self.assertEqual(scenario_type, NetworkChaosScenarioType.VMI)
self.assertEqual(config, self.config)
class TestVmiNetworkChaosModuleGetTargets(unittest.TestCase):
def setUp(self):
self.mock_kubecli = MagicMock()
self.mock_kubernetes = MagicMock()
self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes
self.config = _make_config(
namespace="virt-density-udn-3",
target="virt-server-.*",
)
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
def test_get_targets_success(self):
vmis = [
{"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}},
{"metadata": {"name": "virt-server-2", "namespace": "virt-density-udn-3"}},
]
self.mock_kubernetes.get_vmis.return_value = vmis
result = self.module.get_targets()
self.assertEqual(
result,
[
"virt-density-udn-3/virt-server-1",
"virt-density-udn-3/virt-server-2",
],
)
self.mock_kubernetes.get_vmis.assert_called_once_with(
"virt-server-.*", "virt-density-udn-3", label_selector=None
)
def test_get_targets_no_namespace_raises(self):
self.config.namespace = None
with self.assertRaises(Exception) as ctx:
self.module.get_targets()
self.assertIn("namespace not specified", str(ctx.exception))
def test_get_targets_no_vmis_returns_empty(self):
self.mock_kubernetes.get_vmis.return_value = []
result = self.module.get_targets()
self.assertEqual(result, [])
def test_get_targets_regex_filters_namespace(self):
vmis = [
{"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}},
{"metadata": {"name": "virt-server-2", "namespace": "other-namespace"}},
]
self.mock_kubernetes.get_vmis.return_value = vmis
result = self.module.get_targets()
self.assertIn("virt-density-udn-3/virt-server-1", result)
self.assertNotIn("other-namespace/virt-server-2", result)
def test_get_targets_passes_label_selector(self):
self.config.label_selector = "app=myapp"
self.mock_kubernetes.get_vmis.return_value = []
self.module.get_targets()
self.mock_kubernetes.get_vmis.assert_called_once_with(
"virt-server-.*", "virt-density-udn-3", label_selector="app=myapp"
)
def test_get_targets_empty_label_selector_passes_none(self):
self.config.label_selector = ""
self.mock_kubernetes.get_vmis.return_value = []
self.module.get_targets()
self.mock_kubernetes.get_vmis.assert_called_once_with(
"virt-server-.*", "virt-density-udn-3", label_selector=None
)
def test_get_targets_regex_namespace_passes_empty_string_to_api(self):
"""When namespace is a regex, get_vmis must be called with "" (all namespaces)
so the k8s API isn't asked to look up a literal namespace that doesn't exist."""
self.config.namespace = "virt-density-.*"
vmis = [
{"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}},
{"metadata": {"name": "virt-server-2", "namespace": "other-namespace"}},
]
self.mock_kubernetes.get_vmis.return_value = vmis
result = self.module.get_targets()
self.mock_kubernetes.get_vmis.assert_called_once_with(
"virt-server-.*", "", label_selector=None
)
self.assertIn("virt-density-udn-3/virt-server-1", result)
self.assertNotIn("other-namespace/virt-server-2", result)
class TestVmiNetworkChaosModuleRun(unittest.TestCase):
def setUp(self):
self.mock_kubecli = MagicMock()
self.mock_kubernetes = MagicMock()
self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes
self.config = _make_config(
namespace="virt-density-udn-.*",
target="virt-server-.*",
test_duration=60,
interfaces=[],
latency="100ms",
loss="10",
bandwidth="100mbit",
)
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
self.mock_kubernetes.get_vmi.return_value = {
"status": {"nodeName": "worker-1"}
}
self.mock_kubernetes.list_pods.return_value = [
"virt-launcher-virt-server-3-abc12"
]
compute = _make_container("compute", ready=True, container_id="containerd://deadbeef")
virt_launcher = _make_container("virt-launcher", ready=False, container_id="")
mock_pod_info = MagicMock()
mock_pod_info.containers = [virt_launcher, compute]
self.mock_kubernetes.get_pod_info.return_value = mock_pod_info
self.mock_kubernetes.get_pod_pids.return_value = ["100", "101", "102"]
# ------------------------------------------------------------------ success
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.common_set_limit_rules")
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.time.sleep")
@patch(f"{MODULE}.log_info")
def test_run_success(
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
):
self.module.run("virt-density-udn-3/virt-server-3")
mock_deploy.assert_called_once()
mock_find.assert_called_once()
mock_tap.assert_called_once()
mock_set.assert_called_once()
mock_sleep.assert_called_once_with(60)
mock_del.assert_called_once()
self.mock_kubernetes.delete_pod.assert_called_once()
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.common_set_limit_rules")
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.time.sleep")
@patch(f"{MODULE}.log_info")
def test_run_uses_resolved_namespace_not_regex(
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
):
"""Kubernetes calls must use the real namespace, not the regex pattern."""
self.module.run("virt-density-udn-3/virt-server-3")
self.mock_kubernetes.get_vmi.assert_called_once_with(
"virt-server-3", "virt-density-udn-3"
)
deploy_config = mock_deploy.call_args[0][0]
self.assertEqual(deploy_config.namespace, "virt-density-udn-3")
self.assertNotEqual(deploy_config.namespace, "virt-density-udn-.*")
# ------------------------------------------------------------------ chaos config passed correctly
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.common_set_limit_rules")
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.time.sleep")
@patch(f"{MODULE}.log_info")
def test_run_passes_latency_loss_bandwidth_to_set_limit_rules(
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
):
self.module.run("virt-density-udn-3/virt-server-3")
call_kwargs = {
k: v
for k, v in zip(
["egress", "ingress", "interfaces", "bandwidth", "latency", "loss"],
mock_set.call_args[0],
)
}
self.assertEqual(call_kwargs["latency"], "100ms")
self.assertEqual(call_kwargs["loss"], "10")
self.assertEqual(call_kwargs["bandwidth"], "100mbit")
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.common_set_limit_rules")
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.time.sleep")
@patch(f"{MODULE}.log_info")
def test_run_passes_netns_pid_as_pids_list(
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
):
"""common_set_limit_rules must receive [netns_pid], not the full pids list."""
self.module.run("virt-density-udn-3/virt-server-3")
call_kwargs = mock_set.call_args[1]
self.assertEqual(call_kwargs["pids"], ["101"])
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.common_set_limit_rules")
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.time.sleep")
@patch(f"{MODULE}.log_info")
def test_run_passes_tap_iface_as_interfaces_list(
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
):
"""common_set_limit_rules must receive [iface], not config.interfaces."""
self.module.run("virt-density-udn-3/virt-server-3")
iface_arg = mock_set.call_args[0][2]
self.assertEqual(iface_arg, ["tap0"])
# ------------------------------------------------------------------ error paths
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_vmi_not_found_raises(self, mock_log, mock_deploy):
self.mock_kubernetes.get_vmi.return_value = None
with self.assertRaises(Exception) as ctx:
self.module.run("virt-density-udn-3/virt-server-3")
self.assertIn("not found", str(ctx.exception))
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_vmi_no_node_raises(self, mock_log, mock_deploy):
self.mock_kubernetes.get_vmi.return_value = {"status": {}}
with self.assertRaises(Exception) as ctx:
self.module.run("virt-density-udn-3/virt-server-3")
self.assertIn("unable to determine node", str(ctx.exception))
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_no_virt_launcher_pod_raises(self, mock_log, mock_deploy):
self.mock_kubernetes.list_pods.return_value = []
with self.assertRaises(Exception) as ctx:
self.module.run("virt-density-udn-3/virt-server-3")
self.assertIn("no virt-launcher pod found", str(ctx.exception))
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_no_pod_info_raises(self, mock_log, mock_deploy):
self.mock_kubernetes.get_pod_info.return_value = None
with self.assertRaises(Exception) as ctx:
self.module.run("virt-density-udn-3/virt-server-3")
self.assertIn("impossible to retrieve info", str(ctx.exception))
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_no_pids_raises(self, mock_log, mock_deploy):
self.mock_kubernetes.get_pod_pids.return_value = None
with self.assertRaises(Exception) as ctx:
self.module.run("virt-density-udn-3/virt-server-3")
self.assertIn("impossible to resolve PIDs", str(ctx.exception))
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value=None)
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_no_netns_pid_raises(self, mock_log, mock_deploy, mock_find):
with self.assertRaises(Exception) as ctx:
self.module.run("virt-density-udn-3/virt-server-3")
self.assertIn("could not find a PID", str(ctx.exception))
# ------------------------------------------------------------------ error queue
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_error_queue_captures_exception(self, mock_log, mock_deploy):
self.mock_kubernetes.get_vmi.return_value = None
error_queue = queue.Queue()
self.module.run("virt-density-udn-3/virt-server-3", error_queue)
self.assertFalse(error_queue.empty())
self.assertIn("not found", error_queue.get())
class TestVmiNetworkChaosModuleRollback(unittest.TestCase):
def setUp(self):
self.mock_kubecli = MagicMock()
self.mock_kubernetes = MagicMock()
self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes
self.config = _make_config(
namespace="virt-density-udn-3",
target="virt-server-.*",
test_duration=60,
interfaces=[],
)
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
self.mock_kubernetes.get_vmi.return_value = {"status": {"nodeName": "worker-1"}}
self.mock_kubernetes.list_pods.return_value = ["virt-launcher-virt-server-3-abc12"]
compute = _make_container("compute", ready=True, container_id="containerd://deadbeef")
mock_pod_info = MagicMock()
mock_pod_info.containers = [compute]
self.mock_kubernetes.get_pod_info.return_value = mock_pod_info
self.mock_kubernetes.get_pod_pids.return_value = ["100", "101", "102"]
def test_rollback_calls_delete_limit_rules_then_delete_when_chaos_applied(self):
with patch(f"{MODULE}.common_delete_limit_rules") as mock_del:
self.module._rollback("ns", "chaos-pod", "101", "tap0")
mock_del.assert_called_once()
del_args = mock_del.call_args[0]
self.assertEqual(del_args[2], ["tap0"]) # interfaces
self.assertEqual(del_args[6], ["101"]) # pids
self.mock_kubernetes.delete_pod.assert_called_once_with("chaos-pod", "ns")
def test_rollback_skips_delete_limit_rules_when_chaos_not_applied(self):
with patch(f"{MODULE}.common_delete_limit_rules") as mock_del:
self.module._rollback("ns", "chaos-pod")
mock_del.assert_not_called()
self.mock_kubernetes.delete_pod.assert_called_once_with("chaos-pod", "ns")
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.log_info")
def test_run_rollback_deletes_pod_on_error_before_chaos(
self, mock_log, mock_deploy, mock_del
):
"""Pod deployed but setup fails before chaos: delete only, no limit rules."""
self.mock_kubernetes.get_pod_info.return_value = None
with self.assertRaises(Exception):
self.module.run("virt-density-udn-3/virt-server-3")
self.mock_kubernetes.delete_pod.assert_called_once()
mock_del.assert_not_called()
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.common_set_limit_rules")
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.time.sleep")
@patch(f"{MODULE}.log_info")
def test_run_rollback_calls_delete_limit_rules_on_error_after_chaos(
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
):
"""If interrupted after chaos is applied, delete_limit_rules and delete_pod called."""
mock_sleep.side_effect = RuntimeError("interrupted")
with self.assertRaises(RuntimeError):
self.module.run("virt-density-udn-3/virt-server-3")
mock_del.assert_called_once()
self.mock_kubernetes.delete_pod.assert_called_once()
@patch(f"{MODULE}.common_delete_limit_rules")
@patch(f"{MODULE}.common_set_limit_rules")
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
@patch(f"{MODULE}.time.sleep")
@patch(f"{MODULE}.log_info")
def test_run_rollback_passes_correct_pid_and_iface_on_error(
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
):
mock_sleep.side_effect = RuntimeError("interrupted")
with self.assertRaises(RuntimeError):
self.module.run("virt-density-udn-3/virt-server-3")
del_args = mock_del.call_args[0]
self.assertEqual(del_args[2], ["tap0"]) # interfaces
self.assertEqual(del_args[6], ["101"]) # pids
if __name__ == "__main__":
unittest.main()