mirror of
https://github.com/krkn-chaos/krkn.git
synced 2026-08-25 09:27:36 +00:00
adding start of vmi network scenario (#1260)
Assisted By: Claude Code: Assisted By: Claude Code: Signed-off-by: Paige Patton <prubenda@redhat.com>
This commit is contained in:
@@ -53,6 +53,7 @@ kraken:
|
||||
- scenarios/kube/node-network-chaos.yml
|
||||
- scenarios/kube/pod-network-chaos.yml
|
||||
- scenarios/kube/node_interface_down.yaml
|
||||
- scenarios/openshift/virt_network_chaos.yaml
|
||||
- kubevirt_vm_outage:
|
||||
- scenarios/kubevirt/kubevirt-vm-outage.yaml
|
||||
- http_load_scenarios:
|
||||
|
||||
@@ -20,6 +20,7 @@ from typing import TypeVar, Optional
|
||||
class NetworkChaosScenarioType(Enum):
|
||||
Node = 1
|
||||
Pod = 2
|
||||
VMI = 3
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
import logging
|
||||
import os
|
||||
from typing import Tuple
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import yaml
|
||||
from jinja2 import FileSystemLoader, Environment
|
||||
@@ -110,6 +110,57 @@ def get_pod_default_interface(
|
||||
return output.replace("\n", "")
|
||||
|
||||
|
||||
def find_virt_launcher_netns_pid(
|
||||
chaos_pod_name: str,
|
||||
namespace: str,
|
||||
pids: list[str],
|
||||
kubecli: KrknKubernetes,
|
||||
) -> Optional[str]:
|
||||
"""Return the first PID from `pids` whose netns contains a tap device.
|
||||
|
||||
Not all PIDs returned by get_pod_pids are inside the virt-launcher network
|
||||
namespace — some helper processes run in the host netns. Entering one of
|
||||
those would target the node's physical NIC instead of the bridge slave
|
||||
inside the virt-launcher netns.
|
||||
"""
|
||||
for pid in pids:
|
||||
try:
|
||||
result = kubecli.exec_cmd_in_pod(
|
||||
[f"nsenter --target {pid} --net -- ip link show type tun"],
|
||||
chaos_pod_name,
|
||||
namespace,
|
||||
)
|
||||
if result and "tap" in result:
|
||||
return pid
|
||||
except Exception as e:
|
||||
logging.warning(f"failed to check netns for PID {pid}: {e}")
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def get_vmi_tap_interface(
|
||||
chaos_pod_name: str,
|
||||
namespace: str,
|
||||
netns_pid: str,
|
||||
kubecli: KrknKubernetes,
|
||||
) -> Optional[str]:
|
||||
"""Return the name of the tap device inside the virt-launcher netns."""
|
||||
result = kubecli.exec_cmd_in_pod(
|
||||
[f"nsenter --target {netns_pid} --net -- ip -o link show type tun"],
|
||||
chaos_pod_name,
|
||||
namespace,
|
||||
)
|
||||
if not result:
|
||||
return None
|
||||
for line in result.splitlines():
|
||||
parts = line.split(":")
|
||||
if len(parts) >= 2:
|
||||
iface = parts[1].strip().split("@")[0].strip()
|
||||
if iface.startswith("tap"):
|
||||
return iface
|
||||
return None
|
||||
|
||||
|
||||
def setup_network_chaos_ng_scenario(
|
||||
config: BaseNetworkChaosConfig,
|
||||
node_name: str,
|
||||
|
||||
@@ -28,6 +28,37 @@ CLASS_ID = "100:1"
|
||||
NETEM_HANDLE = "101:"
|
||||
|
||||
|
||||
def _normalize_rate(value: Optional[str]) -> str:
|
||||
"""Accept "100" or "100mbit"/"1gbit"; always return a tc-valid rate string."""
|
||||
if value is None:
|
||||
return "1gbit"
|
||||
s = value.strip()
|
||||
try:
|
||||
float(s)
|
||||
return f"{s}mbit"
|
||||
except ValueError:
|
||||
return s
|
||||
|
||||
|
||||
def _normalize_delay(value: Optional[str]) -> str:
|
||||
"""Accept "50" or "50ms"; always return a tc-valid delay string."""
|
||||
if value is None:
|
||||
return "0ms"
|
||||
s = value.strip()
|
||||
try:
|
||||
float(s)
|
||||
return f"{s}ms"
|
||||
except ValueError:
|
||||
return s
|
||||
|
||||
|
||||
def _normalize_loss(value: Optional[str]) -> str:
|
||||
"""Accept "10" or "10%"; return the bare numeric string (% appended by caller)."""
|
||||
if value is None:
|
||||
return "0"
|
||||
return value.strip().rstrip("%")
|
||||
|
||||
|
||||
def run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(cmd, check=check, text=True, capture_output=True)
|
||||
|
||||
@@ -64,15 +95,15 @@ def get_egress_shaping_comand(
|
||||
) -> list[str]:
|
||||
|
||||
rate_commands = []
|
||||
rate = f"{rate_mbit}mbit" if rate_mbit is not None else "1gbit"
|
||||
d = delay_ms if delay_ms is not None else 0
|
||||
l = loss_pct if loss_pct is not None else 0
|
||||
rate = _normalize_rate(rate_mbit)
|
||||
d = _normalize_delay(delay_ms)
|
||||
l = _normalize_loss(loss_pct)
|
||||
for dev in devices:
|
||||
rate_commands.append(
|
||||
f"tc class change dev {dev} parent {ROOT_HANDLE} classid {CLASS_ID} htb rate {rate}"
|
||||
)
|
||||
rate_commands.append(
|
||||
f"tc qdisc change dev {dev} parent {CLASS_ID} handle {NETEM_HANDLE} netem delay {d}ms loss {l}%"
|
||||
f"tc qdisc change dev {dev} parent {CLASS_ID} handle {NETEM_HANDLE} netem delay {d} loss {l}%"
|
||||
)
|
||||
return rate_commands
|
||||
|
||||
@@ -108,12 +139,12 @@ def get_ingress_shaping_commands(
|
||||
)
|
||||
rate_commands.append(
|
||||
f"tc class add dev {ifb_dev} parent {ROOT_HANDLE} classid {CLASS_ID} "
|
||||
f"htb rate {rate_mbit if rate_mbit else '1gbit'} || true"
|
||||
f"htb rate {_normalize_rate(rate_mbit)} || true"
|
||||
)
|
||||
rate_commands.append(
|
||||
f"tc qdisc add dev {ifb_dev} parent {CLASS_ID} handle {NETEM_HANDLE} "
|
||||
f"netem delay {delay_ms if delay_ms else '0ms'} "
|
||||
f"loss {loss_pct if loss_pct else '0'}% || true"
|
||||
f"netem delay {_normalize_delay(delay_ms)} "
|
||||
f"loss {_normalize_loss(loss_pct)}% || true"
|
||||
)
|
||||
|
||||
return rate_commands
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
# Copyright 2025 The Krkn Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import dataclasses
|
||||
import queue
|
||||
import re
|
||||
import time
|
||||
from typing import Tuple
|
||||
|
||||
from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
|
||||
from krkn_lib.utils import get_random_string
|
||||
|
||||
from krkn.scenario_plugins.network_chaos_ng.models import (
|
||||
NetworkChaosScenarioType,
|
||||
BaseNetworkChaosConfig,
|
||||
NetworkChaosConfig,
|
||||
)
|
||||
from krkn.scenario_plugins.network_chaos_ng.modules.abstract_network_chaos_module import (
|
||||
AbstractNetworkChaosModule,
|
||||
)
|
||||
from krkn.scenario_plugins.network_chaos_ng.modules.utils import (
|
||||
log_info,
|
||||
log_error,
|
||||
deploy_network_chaos_ng_pod,
|
||||
find_virt_launcher_netns_pid,
|
||||
get_vmi_tap_interface,
|
||||
)
|
||||
from krkn.scenario_plugins.network_chaos_ng.modules.utils_network_chaos import (
|
||||
common_set_limit_rules,
|
||||
common_delete_limit_rules,
|
||||
)
|
||||
|
||||
|
||||
class VmiNetworkChaosModule(AbstractNetworkChaosModule):
|
||||
|
||||
def __init__(self, config: NetworkChaosConfig, kubecli: KrknTelemetryOpenshift):
|
||||
super().__init__(config, kubecli)
|
||||
self.config = config
|
||||
|
||||
def _rollback(
|
||||
self,
|
||||
namespace: str,
|
||||
network_chaos_pod_name: str,
|
||||
netns_pid: str = None,
|
||||
iface: str = None,
|
||||
parallel: bool = False,
|
||||
vmi_name: str = "",
|
||||
):
|
||||
if netns_pid and iface:
|
||||
common_delete_limit_rules(
|
||||
self.config.egress,
|
||||
self.config.ingress,
|
||||
[iface],
|
||||
network_chaos_pod_name,
|
||||
namespace,
|
||||
self.kubecli.get_lib_kubernetes(),
|
||||
[netns_pid],
|
||||
parallel,
|
||||
vmi_name,
|
||||
)
|
||||
self.kubecli.get_lib_kubernetes().delete_pod(
|
||||
network_chaos_pod_name, namespace
|
||||
)
|
||||
|
||||
def run(self, target: str, error_queue: queue.Queue = None):
|
||||
# target is "namespace/vmi-name" as produced by get_targets()
|
||||
parallel = False
|
||||
if error_queue:
|
||||
parallel = True
|
||||
|
||||
network_chaos_pod_name = None
|
||||
netns_pid = None
|
||||
iface = None
|
||||
namespace = ""
|
||||
vmi_name = ""
|
||||
|
||||
try:
|
||||
namespace, vmi_name = target.split("/", 1)
|
||||
# Create a scoped config with the resolved namespace so that all
|
||||
# Kubernetes calls use the actual namespace, not the regex pattern.
|
||||
scoped_config = dataclasses.replace(self.config, namespace=namespace)
|
||||
network_chaos_pod_name = f"vmi-network-chaos-{get_random_string(5)}"
|
||||
container_name = f"fedora-container-{get_random_string(5)}"
|
||||
|
||||
log_info(
|
||||
f"creating workload to inject network chaos in VMI {vmi_name} "
|
||||
f"latency:{str(self.config.latency) if self.config.latency else '0'}, "
|
||||
f"loss:{str(self.config.loss) if self.config.loss else '0'}, "
|
||||
f"bandwidth:{str(self.config.bandwidth) if self.config.bandwidth else '0'}",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
vmi = self.kubecli.get_lib_kubernetes().get_vmi(vmi_name, namespace)
|
||||
if not vmi:
|
||||
raise Exception(
|
||||
f"VMI {vmi_name} not found in namespace {namespace}"
|
||||
)
|
||||
|
||||
node_name = vmi.get("status", {}).get("nodeName")
|
||||
if not node_name:
|
||||
raise Exception(
|
||||
f"unable to determine node for VMI {vmi_name} in namespace {namespace}; "
|
||||
"VMI may not be in Running phase"
|
||||
)
|
||||
|
||||
log_info(
|
||||
f"VMI {vmi_name} is running on node {node_name}",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
# The virt-launcher pod carries the VMI's network namespace.
|
||||
virt_launcher_pods = self.kubecli.get_lib_kubernetes().list_pods(
|
||||
namespace, label_selector=f"vm.kubevirt.io/name={vmi_name}"
|
||||
)
|
||||
if not virt_launcher_pods:
|
||||
raise Exception(
|
||||
f"no virt-launcher pod found for VMI {vmi_name} in namespace {namespace}"
|
||||
)
|
||||
virt_launcher_pod_name = virt_launcher_pods[0]
|
||||
|
||||
log_info(
|
||||
f"resolved virt-launcher pod {virt_launcher_pod_name} for VMI {vmi_name}",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
# Deploy the privileged chaos pod onto the VMI's node.
|
||||
# hostPID=True (via template) allows nsenter into the virt-launcher's
|
||||
# network namespace using any of the compute container's host PIDs.
|
||||
deploy_network_chaos_ng_pod(
|
||||
scoped_config,
|
||||
node_name,
|
||||
network_chaos_pod_name,
|
||||
self.kubecli.get_lib_kubernetes(),
|
||||
container_name,
|
||||
host_network=False,
|
||||
)
|
||||
|
||||
# Prefer the 'compute' container (the QEMU process in KubeVirt).
|
||||
# 'virt-launcher' is a sidecar monitor that may not be running.
|
||||
pod_info = self.kubecli.get_lib_kubernetes().get_pod_info(
|
||||
virt_launcher_pod_name, namespace
|
||||
)
|
||||
if not pod_info:
|
||||
raise Exception(
|
||||
f"impossible to retrieve info for virt-launcher pod "
|
||||
f"{virt_launcher_pod_name} in namespace {namespace}"
|
||||
)
|
||||
|
||||
target_container_id = None
|
||||
for container in pod_info.containers:
|
||||
if container.name == "compute" and container.ready and container.containerId:
|
||||
target_container_id = re.sub(r".*://", "", container.containerId)
|
||||
break
|
||||
if not target_container_id:
|
||||
raise Exception(
|
||||
f"compute container in virt-launcher pod {virt_launcher_pod_name} "
|
||||
f"in namespace {namespace} is not ready"
|
||||
)
|
||||
|
||||
log_info(
|
||||
f"targeting compute container {target_container_id}",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
pids = self.kubecli.get_lib_kubernetes().get_pod_pids(
|
||||
base_pod_name=network_chaos_pod_name,
|
||||
base_pod_namespace=namespace,
|
||||
base_pod_container_name=container_name,
|
||||
pod_name=virt_launcher_pod_name,
|
||||
pod_namespace=namespace,
|
||||
pod_container_id=target_container_id,
|
||||
)
|
||||
if not pids:
|
||||
raise Exception(
|
||||
f"impossible to resolve PIDs for virt-launcher pod {virt_launcher_pod_name}"
|
||||
)
|
||||
|
||||
log_info(
|
||||
f"resolved PIDs {pids} on node {node_name} for VMI {vmi_name}",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
# Not all PIDs are in the virt-launcher's netns — find the right one.
|
||||
netns_pid = find_virt_launcher_netns_pid(
|
||||
network_chaos_pod_name,
|
||||
namespace,
|
||||
pids,
|
||||
self.kubecli.get_lib_kubernetes(),
|
||||
)
|
||||
if not netns_pid:
|
||||
raise Exception(
|
||||
f"could not find a PID in the virt-launcher netns for VMI {vmi_name}; "
|
||||
"none of the compute container PIDs contain tap"
|
||||
)
|
||||
|
||||
log_info(
|
||||
f"using PID {netns_pid} for netns entry (virt-launcher netns confirmed via tap)",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
# Target the tap interface rather than the bridge slave (ovn-udn1-nic).
|
||||
# Shaping the bridge slave also affects OVN's BFD heartbeats and can
|
||||
# cause node-wide reconvergence; the tap device only connects to QEMU.
|
||||
if len(scoped_config.interfaces) == 0:
|
||||
iface = get_vmi_tap_interface(
|
||||
network_chaos_pod_name,
|
||||
namespace,
|
||||
netns_pid,
|
||||
self.kubecli.get_lib_kubernetes(),
|
||||
)
|
||||
if not iface:
|
||||
log_error(
|
||||
"could not detect tap interface in virt-launcher netns; "
|
||||
"impossible to execute the VMI network chaos scenario",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
self._rollback(namespace, network_chaos_pod_name)
|
||||
return
|
||||
else:
|
||||
iface = scoped_config.interfaces[0]
|
||||
|
||||
log_info(
|
||||
f"targeting tap interface: {iface}",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
# Apply tc-based shaping (HTB + netem) inside the virt-launcher netns.
|
||||
# Passing pids=[netns_pid] wraps each tc command with nsenter so it
|
||||
# targets the VMI's network namespace, not the host or chaos pod netns.
|
||||
common_set_limit_rules(
|
||||
self.config.egress,
|
||||
self.config.ingress,
|
||||
[iface],
|
||||
self.config.bandwidth,
|
||||
self.config.latency,
|
||||
self.config.loss,
|
||||
parallel,
|
||||
vmi_name,
|
||||
self.kubecli.get_lib_kubernetes(),
|
||||
network_chaos_pod_name,
|
||||
namespace,
|
||||
pids=[netns_pid],
|
||||
)
|
||||
|
||||
log_info(
|
||||
f"waiting {self.config.test_duration} seconds before removing tc rules",
|
||||
parallel,
|
||||
network_chaos_pod_name,
|
||||
)
|
||||
|
||||
time.sleep(self.config.test_duration)
|
||||
|
||||
log_info("removing tc rules", parallel, network_chaos_pod_name)
|
||||
|
||||
self._rollback(namespace, network_chaos_pod_name, netns_pid, iface, parallel, vmi_name)
|
||||
|
||||
except Exception as e:
|
||||
if network_chaos_pod_name:
|
||||
self._rollback(
|
||||
namespace, network_chaos_pod_name, netns_pid, iface, parallel, vmi_name
|
||||
)
|
||||
if error_queue is None:
|
||||
raise e
|
||||
else:
|
||||
error_queue.put(str(e))
|
||||
|
||||
def get_config(self) -> Tuple[NetworkChaosScenarioType, BaseNetworkChaosConfig]:
|
||||
return NetworkChaosScenarioType.VMI, self.config
|
||||
|
||||
def get_targets(self) -> list[str]:
|
||||
if not self.config.namespace:
|
||||
raise Exception("namespace not specified for VMI scenario, aborting")
|
||||
name_regex = self.config.target if self.config.target else ".*"
|
||||
label_selector = self.config.label_selector or None
|
||||
|
||||
# A valid literal k8s namespace is lowercase alphanumerics and hyphens.
|
||||
# If config.namespace contains regex metacharacters, pass "" so get_vmis
|
||||
# lists from all namespaces; re.match() below handles the filtering.
|
||||
api_namespace = (
|
||||
self.config.namespace
|
||||
if re.fullmatch(r"[a-z0-9][a-z0-9\-]*", self.config.namespace)
|
||||
else ""
|
||||
)
|
||||
vmis = self.kubecli.get_lib_kubernetes().get_vmis(
|
||||
name_regex, api_namespace, label_selector=label_selector
|
||||
)
|
||||
return [
|
||||
f"{vmi['metadata']['namespace']}/{vmi['metadata']['name']}"
|
||||
for vmi in vmis
|
||||
if re.match(name_regex, vmi.get("metadata", {}).get("name", ""))
|
||||
and re.match(
|
||||
self.config.namespace,
|
||||
vmi.get("metadata", {}).get("namespace", ""),
|
||||
)
|
||||
]
|
||||
@@ -36,6 +36,9 @@ from krkn.scenario_plugins.network_chaos_ng.modules.pod_network_chaos import (
|
||||
from krkn.scenario_plugins.network_chaos_ng.modules.pod_network_filter import (
|
||||
PodNetworkFilterModule,
|
||||
)
|
||||
from krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos import (
|
||||
VmiNetworkChaosModule,
|
||||
)
|
||||
|
||||
supported_modules = [
|
||||
"node_network_filter",
|
||||
@@ -43,6 +46,7 @@ supported_modules = [
|
||||
"pod_network_chaos",
|
||||
"node_network_chaos",
|
||||
"node_interface_down",
|
||||
"vmi_network_chaos",
|
||||
]
|
||||
|
||||
|
||||
@@ -87,5 +91,11 @@ class NetworkChaosFactory:
|
||||
if len(errors) > 0:
|
||||
raise Exception(f"config validation errors: [{';'.join(errors)}]")
|
||||
return NodeInterfaceDownModule(scenario_config, kubecli)
|
||||
if config["id"] == "vmi_network_chaos":
|
||||
scenario_config = NetworkChaosConfig(**config)
|
||||
errors = scenario_config.validate()
|
||||
if len(errors) > 0:
|
||||
raise Exception(f"config validation errors: [{';'.join(errors)}]")
|
||||
return VmiNetworkChaosModule(scenario_config, kubecli)
|
||||
else:
|
||||
raise Exception(f"invalid network chaos id {config['id']}")
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
- id: vmi_network_chaos
|
||||
image: "quay.io/krkn-chaos/krkn-network-chaos:latest"
|
||||
wait_duration: 300
|
||||
test_duration: 120
|
||||
label_selector: ""
|
||||
service_account: ""
|
||||
taints: []
|
||||
namespace: ""
|
||||
instance_count: 1
|
||||
execution: serial
|
||||
target: ".*"
|
||||
interfaces: []
|
||||
ingress: true
|
||||
egress: true
|
||||
latency: "100ms"
|
||||
loss: "10"
|
||||
bandwidth: "100mbit"
|
||||
@@ -22,12 +22,54 @@ from krkn.scenario_plugins.network_chaos_ng.modules.utils_network_chaos import (
|
||||
node_qdisc_is_simple,
|
||||
common_set_limit_rules,
|
||||
common_delete_limit_rules,
|
||||
_normalize_rate,
|
||||
_normalize_delay,
|
||||
_normalize_loss,
|
||||
ROOT_HANDLE,
|
||||
CLASS_ID,
|
||||
NETEM_HANDLE,
|
||||
)
|
||||
|
||||
|
||||
class TestNormalizers(unittest.TestCase):
|
||||
|
||||
def test_normalize_rate_bare_integer(self):
|
||||
self.assertEqual(_normalize_rate("100"), "100mbit")
|
||||
|
||||
def test_normalize_rate_bare_float(self):
|
||||
self.assertEqual(_normalize_rate("1.5"), "1.5mbit")
|
||||
|
||||
def test_normalize_rate_with_unit(self):
|
||||
self.assertEqual(_normalize_rate("100mbit"), "100mbit")
|
||||
|
||||
def test_normalize_rate_gbit(self):
|
||||
self.assertEqual(_normalize_rate("1gbit"), "1gbit")
|
||||
|
||||
def test_normalize_rate_none(self):
|
||||
self.assertEqual(_normalize_rate(None), "1gbit")
|
||||
|
||||
def test_normalize_delay_bare_integer(self):
|
||||
self.assertEqual(_normalize_delay("50"), "50ms")
|
||||
|
||||
def test_normalize_delay_bare_float(self):
|
||||
self.assertEqual(_normalize_delay("1.5"), "1.5ms")
|
||||
|
||||
def test_normalize_delay_with_unit(self):
|
||||
self.assertEqual(_normalize_delay("50ms"), "50ms")
|
||||
|
||||
def test_normalize_delay_none(self):
|
||||
self.assertEqual(_normalize_delay(None), "0ms")
|
||||
|
||||
def test_normalize_loss_bare(self):
|
||||
self.assertEqual(_normalize_loss("10"), "10")
|
||||
|
||||
def test_normalize_loss_with_pct(self):
|
||||
self.assertEqual(_normalize_loss("10%"), "10")
|
||||
|
||||
def test_normalize_loss_none(self):
|
||||
self.assertEqual(_normalize_loss(None), "0")
|
||||
|
||||
|
||||
class TestBuildTcTreeCommands(unittest.TestCase):
|
||||
|
||||
def test_build_tc_tree_single_interface(self):
|
||||
@@ -150,6 +192,22 @@ class TestEgressShapingCommands(unittest.TestCase):
|
||||
result,
|
||||
)
|
||||
|
||||
def test_egress_shaping_with_suffixed_params(self):
|
||||
"""
|
||||
Test that pre-suffixed values (e.g. "100mbit", "50ms", "10%") are passed through unchanged.
|
||||
"""
|
||||
devices = ["eth0"]
|
||||
result = get_egress_shaping_comand(devices, "100mbit", "50ms", "10%")
|
||||
|
||||
self.assertIn(
|
||||
"tc class change dev eth0 parent 100: classid 100:1 htb rate 100mbit",
|
||||
result,
|
||||
)
|
||||
self.assertIn(
|
||||
"tc qdisc change dev eth0 parent 100:1 handle 101: netem delay 50ms loss 10%",
|
||||
result,
|
||||
)
|
||||
|
||||
def test_egress_shaping_multiple_interfaces(self):
|
||||
"""
|
||||
Test egress shaping for multiple interfaces
|
||||
|
||||
@@ -0,0 +1,440 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Test suite for VmiNetworkChaosModule
|
||||
|
||||
Usage:
|
||||
python -m unittest tests/test_vmi_network_chaos.py -v
|
||||
python -m coverage run -a -m unittest tests/test_vmi_network_chaos.py -v
|
||||
"""
|
||||
|
||||
import queue
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from krkn.scenario_plugins.network_chaos_ng.models import (
|
||||
NetworkChaosScenarioType,
|
||||
NetworkChaosConfig,
|
||||
)
|
||||
from krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos import (
|
||||
VmiNetworkChaosModule,
|
||||
)
|
||||
|
||||
MODULE = "krkn.scenario_plugins.network_chaos_ng.modules.vmi_network_chaos"
|
||||
|
||||
|
||||
def _make_config(**overrides):
|
||||
defaults = dict(
|
||||
id="vmi_network_chaos",
|
||||
image="quay.io/krkn-chaos/krkn-network-chaos:latest",
|
||||
wait_duration=300,
|
||||
test_duration=60,
|
||||
label_selector="",
|
||||
service_account="",
|
||||
taints=[],
|
||||
namespace="virt-density-udn-3",
|
||||
instance_count=1,
|
||||
execution="serial",
|
||||
target=".*",
|
||||
interfaces=[],
|
||||
ingress=True,
|
||||
egress=True,
|
||||
latency="100ms",
|
||||
loss="10",
|
||||
bandwidth="100mbit",
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return NetworkChaosConfig(**defaults)
|
||||
|
||||
|
||||
def _make_container(name, ready=True, container_id="containerd://abc123"):
|
||||
c = MagicMock()
|
||||
c.name = name
|
||||
c.ready = ready
|
||||
c.containerId = container_id
|
||||
return c
|
||||
|
||||
|
||||
class TestVmiNetworkChaosModuleInit(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mock_kubecli = MagicMock()
|
||||
self.config = _make_config()
|
||||
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
|
||||
|
||||
def test_initialization(self):
|
||||
self.assertEqual(self.module.config, self.config)
|
||||
self.assertEqual(self.module.kubecli, self.mock_kubecli)
|
||||
|
||||
def test_get_config(self):
|
||||
scenario_type, config = self.module.get_config()
|
||||
self.assertEqual(scenario_type, NetworkChaosScenarioType.VMI)
|
||||
self.assertEqual(config, self.config)
|
||||
|
||||
|
||||
class TestVmiNetworkChaosModuleGetTargets(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mock_kubecli = MagicMock()
|
||||
self.mock_kubernetes = MagicMock()
|
||||
self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes
|
||||
self.config = _make_config(
|
||||
namespace="virt-density-udn-3",
|
||||
target="virt-server-.*",
|
||||
)
|
||||
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
|
||||
|
||||
def test_get_targets_success(self):
|
||||
vmis = [
|
||||
{"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}},
|
||||
{"metadata": {"name": "virt-server-2", "namespace": "virt-density-udn-3"}},
|
||||
]
|
||||
self.mock_kubernetes.get_vmis.return_value = vmis
|
||||
|
||||
result = self.module.get_targets()
|
||||
|
||||
self.assertEqual(
|
||||
result,
|
||||
[
|
||||
"virt-density-udn-3/virt-server-1",
|
||||
"virt-density-udn-3/virt-server-2",
|
||||
],
|
||||
)
|
||||
self.mock_kubernetes.get_vmis.assert_called_once_with(
|
||||
"virt-server-.*", "virt-density-udn-3", label_selector=None
|
||||
)
|
||||
|
||||
def test_get_targets_no_namespace_raises(self):
|
||||
self.config.namespace = None
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
self.module.get_targets()
|
||||
self.assertIn("namespace not specified", str(ctx.exception))
|
||||
|
||||
def test_get_targets_no_vmis_returns_empty(self):
|
||||
self.mock_kubernetes.get_vmis.return_value = []
|
||||
result = self.module.get_targets()
|
||||
self.assertEqual(result, [])
|
||||
|
||||
def test_get_targets_regex_filters_namespace(self):
|
||||
vmis = [
|
||||
{"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}},
|
||||
{"metadata": {"name": "virt-server-2", "namespace": "other-namespace"}},
|
||||
]
|
||||
self.mock_kubernetes.get_vmis.return_value = vmis
|
||||
result = self.module.get_targets()
|
||||
self.assertIn("virt-density-udn-3/virt-server-1", result)
|
||||
self.assertNotIn("other-namespace/virt-server-2", result)
|
||||
|
||||
def test_get_targets_passes_label_selector(self):
|
||||
self.config.label_selector = "app=myapp"
|
||||
self.mock_kubernetes.get_vmis.return_value = []
|
||||
self.module.get_targets()
|
||||
self.mock_kubernetes.get_vmis.assert_called_once_with(
|
||||
"virt-server-.*", "virt-density-udn-3", label_selector="app=myapp"
|
||||
)
|
||||
|
||||
def test_get_targets_empty_label_selector_passes_none(self):
|
||||
self.config.label_selector = ""
|
||||
self.mock_kubernetes.get_vmis.return_value = []
|
||||
self.module.get_targets()
|
||||
self.mock_kubernetes.get_vmis.assert_called_once_with(
|
||||
"virt-server-.*", "virt-density-udn-3", label_selector=None
|
||||
)
|
||||
|
||||
def test_get_targets_regex_namespace_passes_empty_string_to_api(self):
|
||||
"""When namespace is a regex, get_vmis must be called with "" (all namespaces)
|
||||
so the k8s API isn't asked to look up a literal namespace that doesn't exist."""
|
||||
self.config.namespace = "virt-density-.*"
|
||||
vmis = [
|
||||
{"metadata": {"name": "virt-server-1", "namespace": "virt-density-udn-3"}},
|
||||
{"metadata": {"name": "virt-server-2", "namespace": "other-namespace"}},
|
||||
]
|
||||
self.mock_kubernetes.get_vmis.return_value = vmis
|
||||
result = self.module.get_targets()
|
||||
|
||||
self.mock_kubernetes.get_vmis.assert_called_once_with(
|
||||
"virt-server-.*", "", label_selector=None
|
||||
)
|
||||
self.assertIn("virt-density-udn-3/virt-server-1", result)
|
||||
self.assertNotIn("other-namespace/virt-server-2", result)
|
||||
|
||||
|
||||
class TestVmiNetworkChaosModuleRun(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mock_kubecli = MagicMock()
|
||||
self.mock_kubernetes = MagicMock()
|
||||
self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes
|
||||
self.config = _make_config(
|
||||
namespace="virt-density-udn-.*",
|
||||
target="virt-server-.*",
|
||||
test_duration=60,
|
||||
interfaces=[],
|
||||
latency="100ms",
|
||||
loss="10",
|
||||
bandwidth="100mbit",
|
||||
)
|
||||
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
|
||||
|
||||
self.mock_kubernetes.get_vmi.return_value = {
|
||||
"status": {"nodeName": "worker-1"}
|
||||
}
|
||||
self.mock_kubernetes.list_pods.return_value = [
|
||||
"virt-launcher-virt-server-3-abc12"
|
||||
]
|
||||
|
||||
compute = _make_container("compute", ready=True, container_id="containerd://deadbeef")
|
||||
virt_launcher = _make_container("virt-launcher", ready=False, container_id="")
|
||||
mock_pod_info = MagicMock()
|
||||
mock_pod_info.containers = [virt_launcher, compute]
|
||||
self.mock_kubernetes.get_pod_info.return_value = mock_pod_info
|
||||
self.mock_kubernetes.get_pod_pids.return_value = ["100", "101", "102"]
|
||||
|
||||
# ------------------------------------------------------------------ success
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.common_set_limit_rules")
|
||||
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.time.sleep")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_success(
|
||||
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
|
||||
):
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
|
||||
mock_deploy.assert_called_once()
|
||||
mock_find.assert_called_once()
|
||||
mock_tap.assert_called_once()
|
||||
mock_set.assert_called_once()
|
||||
mock_sleep.assert_called_once_with(60)
|
||||
mock_del.assert_called_once()
|
||||
self.mock_kubernetes.delete_pod.assert_called_once()
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.common_set_limit_rules")
|
||||
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.time.sleep")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_uses_resolved_namespace_not_regex(
|
||||
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
|
||||
):
|
||||
"""Kubernetes calls must use the real namespace, not the regex pattern."""
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
|
||||
self.mock_kubernetes.get_vmi.assert_called_once_with(
|
||||
"virt-server-3", "virt-density-udn-3"
|
||||
)
|
||||
deploy_config = mock_deploy.call_args[0][0]
|
||||
self.assertEqual(deploy_config.namespace, "virt-density-udn-3")
|
||||
self.assertNotEqual(deploy_config.namespace, "virt-density-udn-.*")
|
||||
|
||||
# ------------------------------------------------------------------ chaos config passed correctly
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.common_set_limit_rules")
|
||||
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.time.sleep")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_passes_latency_loss_bandwidth_to_set_limit_rules(
|
||||
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
|
||||
):
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
|
||||
call_kwargs = {
|
||||
k: v
|
||||
for k, v in zip(
|
||||
["egress", "ingress", "interfaces", "bandwidth", "latency", "loss"],
|
||||
mock_set.call_args[0],
|
||||
)
|
||||
}
|
||||
self.assertEqual(call_kwargs["latency"], "100ms")
|
||||
self.assertEqual(call_kwargs["loss"], "10")
|
||||
self.assertEqual(call_kwargs["bandwidth"], "100mbit")
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.common_set_limit_rules")
|
||||
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.time.sleep")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_passes_netns_pid_as_pids_list(
|
||||
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
|
||||
):
|
||||
"""common_set_limit_rules must receive [netns_pid], not the full pids list."""
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
|
||||
call_kwargs = mock_set.call_args[1]
|
||||
self.assertEqual(call_kwargs["pids"], ["101"])
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.common_set_limit_rules")
|
||||
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.time.sleep")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_passes_tap_iface_as_interfaces_list(
|
||||
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
|
||||
):
|
||||
"""common_set_limit_rules must receive [iface], not config.interfaces."""
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
|
||||
iface_arg = mock_set.call_args[0][2]
|
||||
self.assertEqual(iface_arg, ["tap0"])
|
||||
|
||||
# ------------------------------------------------------------------ error paths
|
||||
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_vmi_not_found_raises(self, mock_log, mock_deploy):
|
||||
self.mock_kubernetes.get_vmi.return_value = None
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
self.assertIn("not found", str(ctx.exception))
|
||||
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_vmi_no_node_raises(self, mock_log, mock_deploy):
|
||||
self.mock_kubernetes.get_vmi.return_value = {"status": {}}
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
self.assertIn("unable to determine node", str(ctx.exception))
|
||||
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_no_virt_launcher_pod_raises(self, mock_log, mock_deploy):
|
||||
self.mock_kubernetes.list_pods.return_value = []
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
self.assertIn("no virt-launcher pod found", str(ctx.exception))
|
||||
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_no_pod_info_raises(self, mock_log, mock_deploy):
|
||||
self.mock_kubernetes.get_pod_info.return_value = None
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
self.assertIn("impossible to retrieve info", str(ctx.exception))
|
||||
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_no_pids_raises(self, mock_log, mock_deploy):
|
||||
self.mock_kubernetes.get_pod_pids.return_value = None
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
self.assertIn("impossible to resolve PIDs", str(ctx.exception))
|
||||
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value=None)
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_no_netns_pid_raises(self, mock_log, mock_deploy, mock_find):
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
self.assertIn("could not find a PID", str(ctx.exception))
|
||||
|
||||
# ------------------------------------------------------------------ error queue
|
||||
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_error_queue_captures_exception(self, mock_log, mock_deploy):
|
||||
self.mock_kubernetes.get_vmi.return_value = None
|
||||
error_queue = queue.Queue()
|
||||
self.module.run("virt-density-udn-3/virt-server-3", error_queue)
|
||||
self.assertFalse(error_queue.empty())
|
||||
self.assertIn("not found", error_queue.get())
|
||||
|
||||
|
||||
class TestVmiNetworkChaosModuleRollback(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mock_kubecli = MagicMock()
|
||||
self.mock_kubernetes = MagicMock()
|
||||
self.mock_kubecli.get_lib_kubernetes.return_value = self.mock_kubernetes
|
||||
self.config = _make_config(
|
||||
namespace="virt-density-udn-3",
|
||||
target="virt-server-.*",
|
||||
test_duration=60,
|
||||
interfaces=[],
|
||||
)
|
||||
self.module = VmiNetworkChaosModule(self.config, self.mock_kubecli)
|
||||
|
||||
self.mock_kubernetes.get_vmi.return_value = {"status": {"nodeName": "worker-1"}}
|
||||
self.mock_kubernetes.list_pods.return_value = ["virt-launcher-virt-server-3-abc12"]
|
||||
compute = _make_container("compute", ready=True, container_id="containerd://deadbeef")
|
||||
mock_pod_info = MagicMock()
|
||||
mock_pod_info.containers = [compute]
|
||||
self.mock_kubernetes.get_pod_info.return_value = mock_pod_info
|
||||
self.mock_kubernetes.get_pod_pids.return_value = ["100", "101", "102"]
|
||||
|
||||
def test_rollback_calls_delete_limit_rules_then_delete_when_chaos_applied(self):
|
||||
with patch(f"{MODULE}.common_delete_limit_rules") as mock_del:
|
||||
self.module._rollback("ns", "chaos-pod", "101", "tap0")
|
||||
mock_del.assert_called_once()
|
||||
del_args = mock_del.call_args[0]
|
||||
self.assertEqual(del_args[2], ["tap0"]) # interfaces
|
||||
self.assertEqual(del_args[6], ["101"]) # pids
|
||||
self.mock_kubernetes.delete_pod.assert_called_once_with("chaos-pod", "ns")
|
||||
|
||||
def test_rollback_skips_delete_limit_rules_when_chaos_not_applied(self):
|
||||
with patch(f"{MODULE}.common_delete_limit_rules") as mock_del:
|
||||
self.module._rollback("ns", "chaos-pod")
|
||||
mock_del.assert_not_called()
|
||||
self.mock_kubernetes.delete_pod.assert_called_once_with("chaos-pod", "ns")
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_rollback_deletes_pod_on_error_before_chaos(
|
||||
self, mock_log, mock_deploy, mock_del
|
||||
):
|
||||
"""Pod deployed but setup fails before chaos: delete only, no limit rules."""
|
||||
self.mock_kubernetes.get_pod_info.return_value = None
|
||||
with self.assertRaises(Exception):
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
self.mock_kubernetes.delete_pod.assert_called_once()
|
||||
mock_del.assert_not_called()
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.common_set_limit_rules")
|
||||
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.time.sleep")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_rollback_calls_delete_limit_rules_on_error_after_chaos(
|
||||
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
|
||||
):
|
||||
"""If interrupted after chaos is applied, delete_limit_rules and delete_pod called."""
|
||||
mock_sleep.side_effect = RuntimeError("interrupted")
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
mock_del.assert_called_once()
|
||||
self.mock_kubernetes.delete_pod.assert_called_once()
|
||||
|
||||
@patch(f"{MODULE}.common_delete_limit_rules")
|
||||
@patch(f"{MODULE}.common_set_limit_rules")
|
||||
@patch(f"{MODULE}.get_vmi_tap_interface", return_value="tap0")
|
||||
@patch(f"{MODULE}.find_virt_launcher_netns_pid", return_value="101")
|
||||
@patch(f"{MODULE}.deploy_network_chaos_ng_pod")
|
||||
@patch(f"{MODULE}.time.sleep")
|
||||
@patch(f"{MODULE}.log_info")
|
||||
def test_run_rollback_passes_correct_pid_and_iface_on_error(
|
||||
self, mock_log, mock_sleep, mock_deploy, mock_find, mock_tap, mock_set, mock_del
|
||||
):
|
||||
mock_sleep.side_effect = RuntimeError("interrupted")
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.module.run("virt-density-udn-3/virt-server-3")
|
||||
del_args = mock_del.call_args[0]
|
||||
self.assertEqual(del_args[2], ["tap0"]) # interfaces
|
||||
self.assertEqual(del_args[6], ["101"]) # pids
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user