test: migrate container_scenarios functional test to tests_v2 (#1407)

* feat: migrate container scenarios tests to pytest v2
Migrate CI/tests/test_container.sh to CI/tests_v2/scenarios/container_scenarios/.
Adds dry_run support to ContainerScenarioPlugin and moves legacy test to CI/legacy/.
Closes #1399

Signed-off-by: qsxDree <kurling.town@gmail.com>

* test: strengthen container label selector and dry-run behavior

Deploy a decoy workload to verify label_selector filtering, and skip pod
monitoring when dry_run is enabled so dry runs return without waiting for
expected_recovery_time.

Signed-off-by: qsxDree <kurling.town@gmail.com>

* drop dry_run and keep v1 container test

Signed-off-by: qsxDree <kurling.town@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

* fix container scenario test assertions

Signed-off-by: qsxDree <kurling.town@gmail.com>

---------

Signed-off-by: qsxDree <kurling.town@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Darshan Jain <darjain@redhat.com>

CI/tests_v2/lib/utils.py

@@ -25,6 +25,7 @@ SCENARIO_EXECUTION_MARKERS = {
     "pod_error_scenarios": r"Deleting pod |waiting up to .* seconds for pod recovery",
     "application_outage": r"Creating the network policy|Deleting the network policy",
     "storage_throttle": r"Setting io\.max|Verified blkio settings|Privileged pod deployed",
+    "container_scenarios": r"Killing container .+ in pod",
     "namespace_deletion": r"Delete objects in selected namespace|Deleted all objects in namespace",
 }
 

CI/tests_v2/pytest.ini

@@ -13,6 +13,7 @@ markers =
     storage_throttle: marks a test as a storage throttle scenario test
     cpu_hog: marks a test as a CPU hog scenario test
     memory_hog: marks a test as a memory hog scenario test
+    container_scenarios: marks a test as a container_scenarios scenario test
     node_scenarios: marks a test as a node chaos scenario test (node reboot/stop-start)
     namespace_deletion: marks a test as a namespace deletion (service_disruption) scenario test
     no_workload: skip workload deployment for this test (e.g. negative tests)

CI/tests_v2/scenarios/container_scenarios/resource.yaml

@@ -0,0 +1,21 @@
+# Target workload for container_scenarios scenario tests.
+# Namespace is patched at deploy time by the test framework.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: container-target
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      scenario: container
+  template:
+    metadata:
+      labels:
+        scenario: container
+    spec:
+      containers:
+      - name: fedtools
+        image: nginx:alpine
+        ports:
+        - containerPort: 80

CI/tests_v2/scenarios/container_scenarios/resource_decoy.yaml

@@ -0,0 +1,21 @@
+# Decoy workload: same container name as the target but a different label selector.
+# Used by test_container_label_selector_targeting to prove Krkn respects label_selector.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: container-decoy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      scenario: decoy
+  template:
+    metadata:
+      labels:
+        scenario: decoy
+    spec:
+      containers:
+      - name: fedtools
+        image: nginx:alpine
+        ports:
+        - containerPort: 80

CI/tests_v2/scenarios/container_scenarios/scenario_base.yaml

@@ -0,0 +1,10 @@
+# Base scenario for container_scenarios (used by build_config with scenario_type: container_scenarios).
+scenarios:
+  - name: "container disruption"
+    namespace: "default"
+    label_selector: "scenario=container"
+    container_name: "fedtools"
+    action: 1
+    count: 1
+    expected_recovery_time: 120
+    exclude_label: ""

CI/tests_v2/scenarios/container_scenarios/test_container_scenarios.py

@@ -0,0 +1,145 @@
+"""
+Functional test for container disruption scenario.
+Migrated from CI/tests/test_container.sh.
+Each test runs in its own ephemeral namespace with workload deployed automatically.
+"""
+
+import pytest
+
+from lib.base import BaseScenarioTest, READINESS_TIMEOUT
+from lib.utils import (
+    assert_all_pods_running_and_ready,
+    assert_kraken_failure,
+    assert_kraken_success,
+    assert_pod_count_unchanged,
+    assert_scenario_executed,
+    get_pods_list,
+    pod_uids,
+    restart_counts,
+)
+
+
+@pytest.mark.functional
+@pytest.mark.container_scenarios
+class TestContainerScenarios(BaseScenarioTest):
+    """Container disruption scenario: kill containers and verify recovery."""
+
+    WORKLOAD_MANIFEST = "CI/tests_v2/scenarios/container_scenarios/resource.yaml"
+    WORKLOAD_IS_PATH = True
+    LABEL_SELECTOR = "scenario=container"
+    DECOY_LABEL_SELECTOR = "scenario=decoy"
+    DECOY_MANIFEST = "CI/tests_v2/scenarios/container_scenarios/resource_decoy.yaml"
+    SCENARIO_NAME = "container_scenarios"
+    SCENARIO_TYPE = "container_scenarios"
+    NAMESPACE_KEY_PATH = ["scenarios", 0, "namespace"]
+    NAMESPACE_IS_REGEX = False
+    OVERRIDES_KEY_PATH = ["scenarios", 0]
+
+    @pytest.mark.order(1)
+    def test_container_kill_and_recovery(self, wait_for_pods_running):
+        """Happy path: target container is killed and the workload recovers."""
+        ns = self.ns
+        before = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR)
+        before_uids = pod_uids(before)
+        before_restarts = restart_counts(before)
+
+        result = self.run_scenario(self.tmp_path, ns, overrides={
+            "container_name": "fedtools",
+            "expected_recovery_time": 30,
+        })
+        assert_kraken_success(result, context=f"namespace={ns}", tmp_path=self.tmp_path)
+        assert_scenario_executed(
+            result, self.SCENARIO_NAME, context=f"namespace={ns}", tmp_path=self.tmp_path
+        )
+
+        after = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR)
+        after_uids = pod_uids(after)
+        after_restarts = restart_counts(after)
+        uids_changed = set(after_uids) != set(before_uids)
+        restarts_increased = after_restarts > before_restarts
+        assert uids_changed or restarts_increased, (
+            f"Container chaos had no effect in namespace={ns}: pod UIDs unchanged and "
+            f"restart count did not increase. Before UIDs: {before_uids}, "
+            f"restarts: {before_restarts}. After UIDs: {after_uids}, restarts: {after_restarts}."
+        )
+
+        wait_for_pods_running(ns, self.LABEL_SELECTOR, timeout=READINESS_TIMEOUT)
+        after = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR)
+        assert_pod_count_unchanged(before, after, namespace=ns)
+        assert_all_pods_running_and_ready(after, namespace=ns)
+
+    @pytest.mark.order(2)
+    def test_container_label_selector_targeting(self, wait_for_pods_running, deploy_workload):
+        """Label selector must target only matching pods when a decoy workload shares the namespace."""
+        ns = self.ns
+        deploy_workload(self.DECOY_MANIFEST, self.DECOY_LABEL_SELECTOR)
+
+        before_target = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR)
+        before_decoy = get_pods_list(self.k8s_core, ns, self.DECOY_LABEL_SELECTOR)
+        before_target_uids = pod_uids(before_target)
+        before_target_restarts = restart_counts(before_target)
+        before_decoy_restarts = restart_counts(before_decoy)
+
+        result = self.run_scenario(self.tmp_path, ns, overrides={
+            "container_name": "fedtools",
+            "label_selector": self.LABEL_SELECTOR,
+            "expected_recovery_time": 30,
+        })
+        assert_kraken_success(
+            result, context=f"label_selector namespace={ns}", tmp_path=self.tmp_path
+        )
+        assert_scenario_executed(
+            result, self.SCENARIO_NAME,
+            context=f"label_selector namespace={ns}", tmp_path=self.tmp_path,
+        )
+
+        after_target = get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR)
+        after_decoy = get_pods_list(self.k8s_core, ns, self.DECOY_LABEL_SELECTOR)
+        target_uids_changed = set(pod_uids(after_target)) != set(before_target_uids)
+        target_restarts_increased = restart_counts(after_target) > before_target_restarts
+        assert target_uids_changed or target_restarts_increased, (
+            f"Label selector {self.LABEL_SELECTOR!r} did not disrupt the target workload "
+            f"in namespace={ns}."
+        )
+
+        after_decoy_restarts = restart_counts(after_decoy)
+        assert after_decoy_restarts == before_decoy_restarts, (
+            f"Label selector {self.LABEL_SELECTOR!r} disrupted decoy pods "
+            f"({self.DECOY_LABEL_SELECTOR!r}): before restarts={before_decoy_restarts}, "
+            f"after restarts={after_decoy_restarts} (namespace={ns})"
+        )
+
+        wait_for_pods_running(ns, self.LABEL_SELECTOR, timeout=READINESS_TIMEOUT)
+        wait_for_pods_running(ns, self.DECOY_LABEL_SELECTOR, timeout=READINESS_TIMEOUT)
+        assert_all_pods_running_and_ready(
+            get_pods_list(self.k8s_core, ns, self.LABEL_SELECTOR), namespace=ns
+        )
+        assert_all_pods_running_and_ready(
+            get_pods_list(self.k8s_core, ns, self.DECOY_LABEL_SELECTOR), namespace=ns
+        )
+
+    @pytest.mark.order(3)
+    def test_invalid_container_name_fails(self):
+        """Negative: invalid container name must fail when kill count exceeds matches."""
+        ns = self.ns
+        result = self.run_scenario(self.tmp_path, ns, overrides={
+            "container_name": "nonexistent-container",
+            "count": 2,
+        })
+        assert_kraken_failure(
+            result, context=f"invalid_container namespace={ns}", tmp_path=self.tmp_path
+        )
+
+    @pytest.mark.order(4)
+    def test_invalid_label_selector_fails(self, wait_for_pods_running):
+        """Negative: selector must fail when pods exist but none match the label."""
+        ns = self.ns
+        wait_for_pods_running(ns, self.LABEL_SELECTOR, timeout=READINESS_TIMEOUT)
+
+        result = self.run_scenario(self.tmp_path, ns, overrides={
+            "container_name": "fedtools",
+            "label_selector": "nonexistent=label",
+        })
+        assert_kraken_failure(
+            result, context=f"invalid_selector namespace={ns}", tmp_path=self.tmp_path
+        )