From 5f3fe38d5a62c6da21af58d30a815a923e324281 Mon Sep 17 00:00:00 2001 From: Darshan Jain Date: Fri, 14 Aug 2026 22:16:18 +0530 Subject: [PATCH] fix: serialize node network chaos tests to prevent xdist race condition (#1569) * fix: serialize node network chaos tests to prevent xdist race condition Node network chaos tests share mutable tc rules on the same KinD worker node. Under pytest-xdist parallel execution, concurrent tests interfere with each other's tc state causing intermittent marker assertion failures. Pin all TestNodeNetworkChaos tests to a single xdist worker via xdist_group marker and switch dist mode to loadgroup so the marker is respected. Other test classes remain freely distributed. Signed-off-by: ddjain Co-authored-by: Cursor * fix: move --dist loadgroup from pytest.ini to invocation sites Keeping --dist loadgroup in pytest.ini addopts makes pytest fail at argument parsing when xdist is not installed, bypassing the preflight fixture's graceful skip message. Move the flag to the three places that already pass -n (Makefile test/test-fast, CI workflow) and update the README parallel-run example accordingly. Signed-off-by: ddjain Co-authored-by: Cursor --------- Signed-off-by: ddjain Co-authored-by: Cursor --- .github/workflows/tests_v2.yml | 2 +- CI/tests_v2/Makefile | 4 ++-- CI/tests_v2/README.md | 2 +- CI/tests_v2/pytest.ini | 1 + .../scenarios/node_network_chaos/test_node_network_chaos.py | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_v2.yml b/.github/workflows/tests_v2.yml index 56096967..0391c22c 100644 --- a/.github/workflows/tests_v2.yml +++ b/.github/workflows/tests_v2.yml @@ -53,7 +53,7 @@ jobs: - name: Run tests_v2 run: | KRKN_TEST_COVERAGE=1 python -m pytest CI/tests_v2/ -v --timeout=300 --reruns=1 --reruns-delay=5 \ - --html=CI/tests_v2/report.html -n auto --junitxml=CI/tests_v2/results.xml + --html=CI/tests_v2/report.html -n auto --dist loadgroup --junitxml=CI/tests_v2/results.xml - name: Upload tests_v2 artifacts if: always() diff --git a/CI/tests_v2/Makefile b/CI/tests_v2/Makefile index 5b4aae22..92e582d6 100644 --- a/CI/tests_v2/Makefile +++ b/CI/tests_v2/Makefile @@ -75,10 +75,10 @@ preflight: test: preflight cd $(REPO_ROOT) && KRKN_TEST_COVERAGE=1 $(PYTHON) -m pytest $(TESTS_DIR)/ -v --timeout=300 --reruns=2 --reruns-delay=10 \ - --html=$(TESTS_DIR)/report.html -n auto --junitxml=$(TESTS_DIR)/results.xml + --html=$(TESTS_DIR)/report.html -n auto --dist loadgroup --junitxml=$(TESTS_DIR)/results.xml test-fast: preflight - cd $(REPO_ROOT) && $(PYTHON) -m pytest $(TESTS_DIR)/ -v -p no:rerunfailures -n auto --timeout=120 + cd $(REPO_ROOT) && $(PYTHON) -m pytest $(TESTS_DIR)/ -v -p no:rerunfailures -n auto --dist loadgroup --timeout=120 test-debug: preflight cd $(REPO_ROOT) && $(PYTHON) -m pytest $(TESTS_DIR)/ -v -s -p no:rerunfailures --timeout=300 \ diff --git a/CI/tests_v2/README.md b/CI/tests_v2/README.md index 997d5b8a..a2d4d7df 100644 --- a/CI/tests_v2/README.md +++ b/CI/tests_v2/README.md @@ -89,7 +89,7 @@ pytest CI/tests_v2/ -v --timeout=300 --reruns=2 --reruns-delay=10 --html=CI/test ### Run in parallel (faster suite) ```bash -pytest CI/tests_v2/ -v -n 4 --timeout=300 +pytest CI/tests_v2/ -v -n 4 --dist loadgroup --timeout=300 ``` Ephemeral namespaces make tests parallel-safe; use `-n` with the number of workers (e.g. 4). diff --git a/CI/tests_v2/pytest.ini b/CI/tests_v2/pytest.ini index 0a1cbdd2..f4b797fa 100644 --- a/CI/tests_v2/pytest.ini +++ b/CI/tests_v2/pytest.ini @@ -19,4 +19,5 @@ markers = namespace_deletion: marks a test as a namespace deletion (service_disruption) scenario test no_workload: skip workload deployment for this test (e.g. negative tests) order: set test order (pytest-order) + xdist_group: assign tests to a named xdist group so they run on the same worker (no parallel tc conflicts) junit_family = xunit2 diff --git a/CI/tests_v2/scenarios/node_network_chaos/test_node_network_chaos.py b/CI/tests_v2/scenarios/node_network_chaos/test_node_network_chaos.py index 3e00cc56..734b151c 100644 --- a/CI/tests_v2/scenarios/node_network_chaos/test_node_network_chaos.py +++ b/CI/tests_v2/scenarios/node_network_chaos/test_node_network_chaos.py @@ -40,6 +40,7 @@ TEST_DURATION = 30 @pytest.mark.functional @pytest.mark.node_network_chaos +@pytest.mark.xdist_group("node_network_chaos") class TestNodeNetworkChaos(BaseScenarioTest): """Node network chaos: packet loss, latency, bandwidth, direction, targeting, safety, cleanup."""