Update tests.yml

Signed-off-by: Paige Patton <prubenda@redhat.com>

.coveragerc

@@ -0,0 +1,4 @@
+[run]
+omit =
+    tests/*
+    krkn/tests/**

.github/CODEOWNERS

@@ -0,0 +1 @@
+*   @paigerube14 @tsebastiani @chaitanyaenr

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,43 @@
+---
+name: Bug report
+about: Create a report an issue
+title: "[BUG]"
+labels: bug
+---
+
+# Bug Description
+
+## **Describe the bug**
+
+A clear and concise description of what the bug is.
+
+## **To Reproduce**
+
+Any specific steps used to reproduce the behavior
+
+### Scenario File
+Scenario file(s) that were specified in your config file (can be starred (*) with confidential information )
+```yaml
+<config>
+
+```
+
+### Config File 
+Config file you used when error was seen (the default used is config/config.yaml)
+
+```yaml
+<config>
+
+```
+
+## **Expected behavior**
+
+A clear and concise description of what you expected to happen.
+
+## **Krkn Output**
+
+Krkn output to help show your problem 
+
+## **Additional context**
+
+Add any other context about the problem 

.github/ISSUE_TEMPLATE/feature.md

@@ -0,0 +1,16 @@
+---
+name: New Feature Request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to see added/changed. Ex. new parameter in [xxx] scenario, new scenario that does [xxx]
+
+**Additional context**
+Add any other context about the feature request here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,10 +1,46 @@
-## Description  
-<!-- Provide a brief description of the changes made in this PR. -->  
+# Type of change
 
-## Documentation  
+- [ ] Refactor
+- [ ] New feature
+- [ ] Bug fix
+- [ ] Optimization
+
+# Description  
+<-- Provide a brief description of the changes made in this PR. -->  
+
+## Related Tickets & Documents
+If no related issue, please create one and start the converasation on wants of 
+
+- Related Issue #: 
+- Closes #: 
+
+# Documentation  
 - [ ] **Is documentation needed for this update?**
 
 If checked, a documentation PR must be created and merged in the [website repository](https://github.com/krkn-chaos/website/).
 
 ## Related Documentation PR (if applicable)  
-<!-- Add the link to the corresponding documentation PR in the website repository -->  
+<-- Add the link to the corresponding documentation PR in the website repository -->  
+
+# Checklist before requesting a review
+See [testing your changes](https://krkn-chaos.dev/docs/developers-guide/testing-changes/) and run on any Kubernetes or OpenShift cluster to validate your changes
+- [ ] I have performed a self-review of my code by running krkn and specific scenario 
+- [ ] If it is a core feature, I have added thorough unit tests with above 80% coverage
+
+*REQUIRED*:
+Description of combination of tests performed and output of run
+
+```bash
+python run_kraken.py
+...
+<---insert test results output--->
+```
+
+OR
+
+
+```bash
+python -m coverage run -a -m unittest discover -s tests -v
+...
+<---insert test results output--->
+```

.github/workflows/tests.yml

@@ -16,14 +16,19 @@ jobs:
         uses: redhat-chaos/actions/kind@main
       - name: Deploy prometheus & Port Forwarding
         uses: redhat-chaos/actions/prometheus@main
-      
       - name: Deploy Elasticsearch
         with:
-          ELASTIC_URL: ${{ vars.ELASTIC_URL }}
-          ELASTIC_PORT: ${{ vars.ELASTIC_PORT }}
-          ELASTIC_USER: ${{ vars.ELASTIC_USER }}
-          ELASTIC_PASSWORD: ${{ vars.ELASTIC_PASSWORD }}
+          ELASTIC_PORT: ${{ env.ELASTIC_PORT }}
+          RUN_ID: ${{ github.run_id }}
         uses: redhat-chaos/actions/elastic@main
+      - name: Download elastic password
+        uses: actions/download-artifact@v4
+        with:
+          name: elastic_password_${{ github.run_id }}
+      - name: Set elastic password on env
+        run: |
+          ELASTIC_PASSWORD=$(cat elastic_password.txt)
+          echo "ELASTIC_PASSWORD=$ELASTIC_PASSWORD" >> "$GITHUB_ENV"
       - name: Install Python
         uses: actions/setup-python@v4
         with:
@@ -37,10 +42,22 @@ jobs:
 
       - name: Deploy test workloads
         run: |
-          es_pod_name=$(kubectl get pods -l "app.kubernetes.io/instance=elasticsearch" -o name)
+          es_pod_name=$(kubectl get pods -l "app=elasticsearch-master" -o name)
+          echo "POD_NAME: $es_pod_name"
           kubectl --namespace default port-forward $es_pod_name 9200 &
           prom_name=$(kubectl get pods -n monitoring -l "app.kubernetes.io/name=prometheus" -o name)
           kubectl --namespace monitoring port-forward $prom_name 9090 &
+
+          # Wait for Elasticsearch to be ready
+          echo "Waiting for Elasticsearch to be ready..."
+          for i in {1..30}; do
+            if curl -k -s -u elastic:$ELASTIC_PASSWORD https://localhost:9200/_cluster/health > /dev/null 2>&1; then
+              echo "Elasticsearch is ready!"
+              break
+            fi
+            echo "Attempt $i: Elasticsearch not ready yet, waiting..."
+            sleep 2
+          done
           kubectl apply -f CI/templates/outage_pod.yaml
           kubectl wait --for=condition=ready pod -l scenario=outage --timeout=300s
           kubectl apply -f CI/templates/container_scenario_pod.yaml
@@ -50,37 +67,39 @@ jobs:
           kubectl wait --for=condition=ready pod -l scenario=time-skew --timeout=300s
           kubectl apply -f CI/templates/service_hijacking.yaml
           kubectl wait --for=condition=ready pod -l "app.kubernetes.io/name=proxy" --timeout=300s
+          kubectl apply -f CI/legacy/scenarios/volume_scenario.yaml
+          kubectl wait --for=condition=ready pod kraken-test-pod -n kraken --timeout=300s
       - name: Get Kind nodes
         run: |
           kubectl get nodes --show-labels=true
       # Pull request only steps
       - name: Run unit tests
-        if: github.event_name == 'pull_request'
         run: python -m coverage run -a -m unittest discover -s tests -v
 
-      - name: Setup Pull Request Functional Tests
-        if: |
-          github.event_name == 'pull_request'
+      - name: Setup Functional Tests
         run: |
-            yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
-            yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
             yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
             yq -i '.elastic.elastic_port=9200' CI/config/common_test_config.yaml
             yq -i '.elastic.elastic_url="https://localhost"' CI/config/common_test_config.yaml
-            yq -i '.elastic.enable_elastic=True' CI/config/common_test_config.yaml
+            yq -i '.elastic.enable_elastic=False' CI/config/common_test_config.yaml
+            yq -i '.elastic.password="${{env.ELASTIC_PASSWORD}}"' CI/config/common_test_config.yaml
             yq -i '.performance_monitoring.prometheus_url="http://localhost:9090"' CI/config/common_test_config.yaml
-            echo "test_service_hijacking" > ./CI/tests/functional_tests
             echo "test_app_outages" >> ./CI/tests/functional_tests
-            echo "test_container"      >> ./CI/tests/functional_tests
-            echo "test_pod" >> ./CI/tests/functional_tests
-            echo "test_namespace"      >> ./CI/tests/functional_tests
-            echo "test_net_chaos"      >> ./CI/tests/functional_tests
-            echo "test_time"           >> ./CI/tests/functional_tests
+            echo "test_container" >> ./CI/tests/functional_tests
             echo "test_cpu_hog" >> ./CI/tests/functional_tests
-            echo "test_memory_hog" >> ./CI/tests/functional_tests
+            echo "test_customapp_pod" >> ./CI/tests/functional_tests
             echo "test_io_hog" >> ./CI/tests/functional_tests
+            echo "test_memory_hog" >> ./CI/tests/functional_tests
+            echo "test_namespace" >> ./CI/tests/functional_tests
+            echo "test_net_chaos" >> ./CI/tests/functional_tests
+            echo "test_node" >> ./CI/tests/functional_tests
+            echo "test_pod" >> ./CI/tests/functional_tests
+            echo "test_pod_error" >> ./CI/tests/functional_tests
+            echo "test_service_hijacking" > ./CI/tests/functional_tests
             echo "test_pod_network_filter" >> ./CI/tests/functional_tests
-
+            echo "test_pod_server" >> ./CI/tests/functional_tests
+            echo "test_time" >> ./CI/tests/functional_tests
+            # echo "test_pvc" >> ./CI/tests/functional_tests
 
       # Push on main only steps + all other functional to collect coverage
       # for the badge
@@ -94,28 +113,9 @@ jobs:
       - name: Setup Post Merge Request Functional Tests
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |
-          yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml
-          yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml
-          yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml
-          yq -i '.elastic.enable_elastic=True' CI/config/common_test_config.yaml
-          yq -i '.elastic.elastic_port=9200' CI/config/common_test_config.yaml
-          yq -i '.elastic.elastic_url="https://localhost"' CI/config/common_test_config.yaml
-          yq -i '.performance_monitoring.prometheus_url="http://localhost:9090"' CI/config/common_test_config.yaml
           yq -i '.telemetry.username="${{secrets.TELEMETRY_USERNAME}}"' CI/config/common_test_config.yaml
           yq -i '.telemetry.password="${{secrets.TELEMETRY_PASSWORD}}"' CI/config/common_test_config.yaml
-          echo "test_telemetry" > ./CI/tests/functional_tests
-          echo "test_service_hijacking" >> ./CI/tests/functional_tests
-          echo "test_app_outages" >> ./CI/tests/functional_tests
-          echo "test_container"      >> ./CI/tests/functional_tests
-          echo "test_pod" >> ./CI/tests/functional_tests
-          echo "test_namespace"      >> ./CI/tests/functional_tests
-          echo "test_net_chaos"      >> ./CI/tests/functional_tests
-          echo "test_time"           >> ./CI/tests/functional_tests
-          echo "test_cpu_hog" >> ./CI/tests/functional_tests
-          echo "test_memory_hog" >> ./CI/tests/functional_tests
-          echo "test_io_hog" >> ./CI/tests/functional_tests
-          echo "test_pod_network_filter" >> ./CI/tests/functional_tests
-
+          echo "test_telemetry" >> ./CI/tests/functional_tests
       # Final common steps
       - name: Run Functional tests
         env:
@@ -125,38 +125,38 @@ jobs:
           cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY
           echo >> $GITHUB_STEP_SUMMARY
       - name: Upload CI logs
-        if: ${{ success() || failure() }}
+        if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           name: ci-logs
           path: CI/out
           if-no-files-found: error
       - name: Collect coverage report
-        if: ${{ success() || failure() }}
+        if: ${{ always() }}
         run: |
           python -m coverage html
           python -m coverage json
       - name: Publish coverage report to job summary
-        if: ${{ success() || failure() }}
+        if: ${{ always() }}
         run: |
           pip install html2text
           html2text --ignore-images --ignore-links -b 0 htmlcov/index.html >> $GITHUB_STEP_SUMMARY
       - name: Upload coverage data
-        if: ${{ success() || failure() }}
+        if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           name: coverage
           path: htmlcov
           if-no-files-found: error
       - name: Upload json coverage
-        if: ${{ success() || failure() }}
+        if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           name: coverage.json
           path: coverage.json
           if-no-files-found: error
       - name: Check CI results
-        if: ${{ success() || failure() }}
+        if: ${{ always() }}
         run: "! grep Fail CI/results.markdown"
 
   badge:

ADOPTERS.md

@@ -6,3 +6,4 @@ This is a list of organizations that have publicly acknowledged usage of Krkn an
 |:-|:-|:-|:-|
 | MarketAxess | 2024 | https://www.marketaxess.com/ | Kraken enables us to achieve our goal of increasing the reliability of our cloud products on Kubernetes. The tool allows us to automatically run various chaos scenarios, identify resilience and performance bottlenecks, and seamlessly restore the system to its original state once scenarios finish. These chaos scenarios include pod disruptions, node (EC2) outages, simulating availability zone (AZ) outages, and filling up storage spaces like EBS and EFS. The community is highly responsive to requests and works on expanding the tool's capabilities. MarketAxess actively contributes to the project, adding features such as the ability to leverage existing network ACLs and proposing several feature improvements to enhance test coverage. |
 | Red Hat Openshift | 2020 | https://www.redhat.com/ | Kraken is a highly reliable chaos testing tool used to ensure the quality and resiliency of Red Hat Openshift. The engineering team runs all the test scenarios under Kraken on different cloud platforms on both self-managed and cloud services environments prior to the release of a new version of the product. The team also contributes to the Kraken project consistently which helps the test scenarios to keep up with the new features introduced to the product. Inclusion of this test coverage has contributed to gaining the trust of new and existing customers of the product.   |
+| IBM | 2023 | https://www.ibm.com/ | While working on AI for Chaos Testing at IBM Research, we closely collaborated with the Kraken (Krkn) team to advance intelligent chaos engineering. Our contributions included developing AI-enabled chaos injection strategies and integrating reinforcement learning (RL)-based fault search techniques into the Krkn tool, enabling it to identify and explore system vulnerabilities more efficiently. Kraken stands out as one of the most user-friendly and effective tools for chaos engineering, and the Kraken team’s deep technical involvement played a crucial role in the success of this collaboration—helping bridge cutting-edge AI research with practical, real-world system reliability testing.   |

CI/config/common_test_config.yaml

@@ -2,6 +2,10 @@ kraken:
     distribution: kubernetes                                # Distribution can be kubernetes or openshift.
     kubeconfig_path: ~/.kube/config                        # Path to kubeconfig.
     exit_on_failure: False                                 # Exit when a post action scenario fails.
+    publish_kraken_status: True                            # Can be accessed at http://0.0.0.0:8081
+    signal_state: RUN                                      # Will wait for the RUN signal when set to PAUSE before running the scenarios, refer docs/signal.md for more details
+    signal_address: 0.0.0.0                                # Signal listening address
+    port: 8081                                             # Signal port
     auto_rollback: True                                    # Enable auto rollback for scenarios.
     rollback_versions_directory: /tmp/kraken-rollback      # Directory to store rollback version files.
     chaos_scenarios:                                       # List of policies/chaos scenarios to load.
@@ -32,7 +36,7 @@ telemetry:
     api_url: https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test #telemetry service endpoint
     username: $TELEMETRY_USERNAME                                      # telemetry service username
     password: $TELEMETRY_PASSWORD                                      # telemetry service password
-    prometheus_namespace: 'prometheus-k8s'                                # prometheus namespace
+    prometheus_namespace: 'monitoring'                                # prometheus namespace
     prometheus_pod_name: 'prometheus-kind-prometheus-kube-prome-prometheus-0'                                 # prometheus pod_name
     prometheus_container_name: 'prometheus'
     prometheus_backup: True                                 # enables/disables prometheus data collection

CI/legacy/scenarios/volume_scenario.yaml

@@ -45,15 +45,45 @@ metadata:
   name: kraken-test-pod
   namespace: kraken
 spec:
+  securityContext:
+    fsGroup: 1001
+  # initContainer to fix permissions on the mounted volume
+  initContainers:
+    - name: fix-permissions
+      image: 'quay.io/centos7/httpd-24-centos7:centos7'
+      command:
+        - sh
+        - -c
+        - |
+          echo "Setting up permissions for /home/kraken..."
+          # Create the directory if it doesn't exist
+          mkdir -p /home/kraken
+          # Set ownership to user 1001 and group 1001
+          chown -R 1001:1001 /home/kraken
+          # Set permissions to allow read/write
+          chmod -R 755 /home/kraken
+          rm -rf /home/kraken/*
+          echo "Permissions fixed. Current state:"
+          ls -la /home/kraken
+      volumeMounts:
+        - mountPath: "/home/kraken"
+          name: kraken-test-pv
+      securityContext:
+        runAsUser: 0  # Run as root to fix permissions
   volumes:
     - name: kraken-test-pv
       persistentVolumeClaim:
         claimName: kraken-test-pvc
   containers:
     - name: kraken-test-container
-      image: 'quay.io/centos7/httpd-24-centos7:latest'
-      volumeMounts:
-        - mountPath: "/home/krake-dir/"
-          name: kraken-test-pv
+      image: 'quay.io/centos7/httpd-24-centos7:centos7'
       securityContext:
-        privileged: true
+        runAsUser: 1001
+        runAsNonRoot: true
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop:
+            - ALL
+      volumeMounts:
+        - mountPath: "/home/kraken"
+          name: kraken-test-pv

CI/tests/test_app_outages.sh

@@ -19,6 +19,7 @@ function functional_test_app_outage {
   kubectl get pods 
   envsubst < CI/config/common_test_config.yaml > CI/config/app_outage.yaml
   cat $scenario_file
+  cat CI/config/app_outage.yaml
   python3 -m coverage run -a run_kraken.py -c CI/config/app_outage.yaml
   echo "App outage scenario test: Success"
 }

CI/tests/test_container.sh

@@ -16,8 +16,10 @@ function functional_test_container_crash {
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/container_config.yaml
 
-  python3 -m coverage run -a run_kraken.py -c CI/config/container_config.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/container_config.yaml -d True
   echo "Container scenario test: Success"
+
+  kubectl get pods -n kube-system -l component=etcd
 }
 
 functional_test_container_crash

CI/tests/test_customapp_pod.sh

@@ -0,0 +1,18 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+function functional_test_customapp_pod_node_selector {
+  export scenario_type="pod_disruption_scenarios"
+  export scenario_file="scenarios/openshift/customapp_pod.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/customapp_pod_config.yaml
+
+  python3 -m coverage run -a run_kraken.py -c CI/config/customapp_pod_config.yaml -d True
+  echo "Pod disruption with node_label_selector test: Success"
+}
+
+functional_test_customapp_pod_node_selector

CI/tests/test_node.sh

@@ -0,0 +1,18 @@
+uset -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+function functional_test_node_stop_start {
+  export scenario_type="node_scenarios"
+  export scenario_file="scenarios/kind/node_scenarios_example.yml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/node_config.yaml
+  cat CI/config/node_config.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/node_config.yaml
+  echo "Node Stop/Start scenario test: Success"
+}
+
+functional_test_node_stop_start

CI/tests/test_pod.sh

@@ -13,6 +13,8 @@ function functional_test_pod_crash {
 
   python3 -m coverage run -a run_kraken.py -c CI/config/pod_config.yaml
   echo "Pod disruption scenario test: Success"
+  date
+  kubectl get pods -n kube-system -l component=etcd -o yaml
 }
 
 functional_test_pod_crash

CI/tests/test_pod_error.sh

@@ -0,0 +1,28 @@
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+function functional_test_pod_error {
+  export scenario_type="pod_disruption_scenarios"
+  export scenario_file="scenarios/kind/pod_etcd.yml"
+  export post_config=""
+  yq -i '.[0].config.kill=5' scenarios/kind/pod_etcd.yml
+  envsubst < CI/config/common_test_config.yaml > CI/config/pod_config.yaml
+  cat CI/config/pod_config.yaml
+
+  cat scenarios/kind/pod_etcd.yml
+  python3 -m coverage run -a run_kraken.py -c CI/config/pod_config.yaml
+  
+  ret=$?
+  echo "\n\nret $ret"
+  if [[ $ret -ge 1 ]]; then
+      echo "Pod disruption error scenario test: Success"
+  else 
+    echo "Pod disruption error scenario test: Failure"
+    exit 1
+  fi
+}
+
+functional_test_pod_error

CI/tests/test_pod_server.sh

@@ -0,0 +1,35 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+function functional_test_pod_server {
+  export scenario_type="pod_disruption_scenarios"
+  export scenario_file="scenarios/kind/pod_etcd.yml"
+  export post_config=""
+
+  envsubst < CI/config/common_test_config.yaml > CI/config/pod_config.yaml
+  yq -i '.[0].config.kill=1' scenarios/kind/pod_etcd.yml
+  
+  yq -i '.tunings.daemon_mode=True' CI/config/pod_config.yaml
+  cat CI/config/pod_config.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/pod_config.yaml & 
+  sleep 15
+  curl -X POST http:/0.0.0.0:8081/STOP
+
+  wait
+
+  yq -i '.kraken.signal_state="PAUSE"' CI/config/pod_config.yaml
+  yq -i '.tunings.daemon_mode=False' CI/config/pod_config.yaml
+  cat CI/config/pod_config.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/pod_config.yaml & 
+  sleep 5
+  curl -X POST http:/0.0.0.0:8081/RUN
+  wait
+
+  echo "Pod disruption with server scenario test: Success"
+}
+
+functional_test_pod_server

CI/tests/test_pvc.sh

@@ -0,0 +1,18 @@
+set -xeEo pipefail
+
+source CI/tests/common.sh
+
+trap error ERR
+trap finish EXIT
+
+function functional_test_pvc_fill {
+  export scenario_type="pvc_scenarios"
+  export scenario_file="scenarios/kind/pvc_scenario.yaml"
+  export post_config=""
+  envsubst < CI/config/common_test_config.yaml > CI/config/pvc_config.yaml
+  cat CI/config/pvc_config.yaml
+  python3 -m coverage run -a run_kraken.py -c CI/config/pvc_config.yaml --debug True
+  echo "PVC Fill scenario test: Success"
+}
+
+functional_test_pvc_fill

CI/tests/test_telemetry.sh

@@ -18,9 +18,8 @@ function functional_test_telemetry {
   yq -i '.performance_monitoring.prometheus_url="http://localhost:9090"' CI/config/common_test_config.yaml
   yq -i '.telemetry.run_tag=env(RUN_TAG)' CI/config/common_test_config.yaml
 
-  export scenario_type="hog_scenarios"
-
-  export scenario_file="scenarios/kube/cpu-hog.yml"
+  export scenario_type="pod_disruption_scenarios"
+  export scenario_file="scenarios/kind/pod_etcd.yml"
 
   export post_config=""
   envsubst < CI/config/common_test_config.yaml > CI/config/telemetry.yaml

README.md

@@ -22,14 +22,8 @@ Kraken injects deliberate failures into Kubernetes clusters to check if it is re
 Instructions on how to setup, configure and run Kraken can be found in the [documentation](https://krkn-chaos.dev/docs/).
 
 
-### Blogs and other useful resources
-- Blog post on introduction to Kraken: https://www.openshift.com/blog/introduction-to-kraken-a-chaos-tool-for-openshift/kubernetes
-- Discussion and demo on how Kraken can be leveraged to ensure OpenShift is reliable, performant and scalable: https://www.youtube.com/watch?v=s1PvupI5sD0&ab_channel=OpenShift
-- Blog post emphasizing the importance of making Chaos part of Performance and Scale runs to mimic the production environments: https://www.openshift.com/blog/making-chaos-part-of-kubernetes/openshift-performance-and-scalability-tests
-- Blog post on findings from Chaos test runs: https://cloud.redhat.com/blog/openshift/kubernetes-chaos-stories
-- Discussion with CNCF TAG App Delivery on Krkn workflow, features and addition to CNCF sandbox: [Github](https://github.com/cncf/sandbox/issues/44), [Tracker](https://github.com/cncf/tag-app-delivery/issues/465), [recording](https://www.youtube.com/watch?v=nXQkBFK_MWc&t=722s)
-- Blog post on supercharging chaos testing using AI integration in Krkn: https://www.redhat.com/en/blog/supercharging-chaos-testing-using-ai
-- Blog post announcing Krkn joining CNCF Sandbox: https://www.redhat.com/en/blog/krknchaos-joining-cncf-sandbox
+### Blogs, podcasts and interviews
+Additional resources, including blog posts, podcasts, and community interviews, can be found on the [website](https://krkn-chaos.dev/blog)
 
 
 ### Roadmap

config/config.yaml

@@ -8,55 +8,55 @@ kraken:
     signal_address: 0.0.0.0                                # Signal listening address
     port: 8081                                             # Signal port
     chaos_scenarios:
-        # List of policies/chaos scenarios to load
-        - hog_scenarios:
-            - scenarios/kube/cpu-hog.yml
-            - scenarios/kube/memory-hog.yml
-            - scenarios/kube/io-hog.yml
-        - application_outages_scenarios:
-            - scenarios/openshift/app_outage.yaml
-        - container_scenarios:                             # List of chaos pod scenarios to load
-            - scenarios/openshift/container_etcd.yml
-        - pod_network_scenarios:
-              - scenarios/openshift/network_chaos_ingress.yml
-              - scenarios/openshift/pod_network_outage.yml
-        - pod_disruption_scenarios:
-            - scenarios/openshift/etcd.yml
-            - scenarios/openshift/regex_openshift_pod_kill.yml
-            - scenarios/openshift/prom_kill.yml
-            - scenarios/openshift/openshift-apiserver.yml
-            - scenarios/openshift/openshift-kube-apiserver.yml
-        - node_scenarios:                                  # List of chaos node scenarios to load
-            - scenarios/openshift/aws_node_scenarios.yml
-            - scenarios/openshift/vmware_node_scenarios.yml
-            - scenarios/openshift/ibmcloud_node_scenarios.yml
-        - time_scenarios:                                  # List of chaos time scenarios to load
-            - scenarios/openshift/time_scenarios_example.yml
-        - cluster_shut_down_scenarios:
-            - scenarios/openshift/cluster_shut_down_scenario.yml
-        - service_disruption_scenarios:
-             - scenarios/openshift/regex_namespace.yaml
-             - scenarios/openshift/ingress_namespace.yaml
-        - zone_outages_scenarios:
-            - scenarios/openshift/zone_outage.yaml
-        - pvc_scenarios:
-            - scenarios/openshift/pvc_scenario.yaml
-        - network_chaos_scenarios:
-            - scenarios/openshift/network_chaos.yaml
-        - service_hijacking_scenarios:
-              - scenarios/kube/service_hijacking.yaml
-        - syn_flood_scenarios:
-              - scenarios/kube/syn_flood.yaml
-        -  network_chaos_ng_scenarios:
+       # List of policies/chaos scenarios to load
+       - hog_scenarios:
+           - scenarios/kube/cpu-hog.yml
+           - scenarios/kube/memory-hog.yml
+           - scenarios/kube/io-hog.yml
+       - application_outages_scenarios:
+           - scenarios/openshift/app_outage.yaml
+       - container_scenarios:                             # List of chaos pod scenarios to load
+           - scenarios/openshift/container_etcd.yml
+       - pod_network_scenarios:
+             - scenarios/openshift/network_chaos_ingress.yml
+             - scenarios/openshift/pod_network_outage.yml
+       - pod_disruption_scenarios:
+           - scenarios/openshift/etcd.yml
+           - scenarios/openshift/regex_openshift_pod_kill.yml
+           - scenarios/openshift/prom_kill.yml
+           - scenarios/openshift/openshift-apiserver.yml
+           - scenarios/openshift/openshift-kube-apiserver.yml
+       - node_scenarios:                                  # List of chaos node scenarios to load
+           - scenarios/openshift/aws_node_scenarios.yml
+           - scenarios/openshift/vmware_node_scenarios.yml
+           - scenarios/openshift/ibmcloud_node_scenarios.yml
+       - time_scenarios:                                  # List of chaos time scenarios to load
+           - scenarios/openshift/time_scenarios_example.yml
+       - cluster_shut_down_scenarios:
+           - scenarios/openshift/cluster_shut_down_scenario.yml
+       - service_disruption_scenarios:
+            - scenarios/openshift/regex_namespace.yaml
+            - scenarios/openshift/ingress_namespace.yaml
+       - zone_outages_scenarios:
+           - scenarios/openshift/zone_outage.yaml
+       - pvc_scenarios:
+           - scenarios/openshift/pvc_scenario.yaml
+       - network_chaos_scenarios:
+           - scenarios/openshift/network_chaos.yaml
+       - service_hijacking_scenarios:
+             - scenarios/kube/service_hijacking.yaml
+       - syn_flood_scenarios:
+             - scenarios/kube/syn_flood.yaml
+       - network_chaos_ng_scenarios:
                - scenarios/kube/pod-network-filter.yml
                - scenarios/kube/node-network-filter.yml
-        -  kubevirt_vm_outage:
-               - scenarios/kubevirt/kubevirt-vm-outage.yaml
+       -  kubevirt_vm_outage:
+              - scenarios/kubevirt/kubevirt-vm-outage.yaml
 
 cerberus:
     cerberus_enabled: False                                # Enable it when cerberus is previously installed
     cerberus_url:                                          # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal
-    check_applicaton_routes: False                         # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
+    check_application_routes: False                         # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
 
 performance_monitoring:
     prometheus_url: ''                                    # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.
@@ -79,7 +79,7 @@ elastic:
     telemetry_index: "krkn-telemetry"
 
 tunings:
-    wait_duration: 60                                      # Duration to wait between each chaos scenario
+    wait_duration: 1                                      # Duration to wait between each chaos scenario
     iterations: 1                                          # Number of times to execute the scenarios
     daemon_mode: False                                     # Iterations are set to infinity which means that the kraken will cause chaos forever
 telemetry:
@@ -125,4 +125,7 @@ kubevirt_checks:                                            # Utilizing virt che
     namespace:                                              # Namespace where to find VMI's
     name:                                                   # Regex Name style of VMI's to watch, optional, will watch all VMI names in the namespace if left blank
     only_failures: False                                    # Boolean of whether to show all VMI's failures and successful ssh connection (False), or only failure status' (True) 
-    disconnected: False                                     # Boolean of how to try to connect to the VMIs; if True will use the ip_address to try ssh from within a node, if false will use the name and uses virtctl to try to connect; Default is False
+    disconnected: False                                     # Boolean of how to try to connect to the VMIs; if True will use the ip_address to try ssh from within a node, if false will use the name and uses virtctl to try to connect; Default is False
+    ssh_node: ""                                            # If set, will be a backup way to ssh to a node. Will want to set to a node that isn't targeted in chaos
+    node_names: ""
+    exit_on_failure:                                        # If value is True and VMI's are failing post chaos returns failure, values can be True/False

config/config_kind.yaml

@@ -13,7 +13,7 @@ kraken:
 cerberus:
     cerberus_enabled: False                                # Enable it when cerberus is previously installed
     cerberus_url:                                          # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal
-    check_applicaton_routes: False                         # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
+    check_application_routes: False                         # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
 
 performance_monitoring:
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.

config/config_kubernetes.yaml

@@ -14,7 +14,7 @@ kraken:
 cerberus:
     cerberus_enabled: False                                # Enable it when cerberus is previously installed
     cerberus_url:                                          # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal
-    check_applicaton_routes: False                         # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
+    check_application_routes: False                         # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
 
 performance_monitoring:
     prometheus_url:                                       # The prometheus url/route is automatically obtained in case of OpenShift, please set it when the distribution is Kubernetes.

config/config_performance.yaml

@@ -35,7 +35,7 @@ kraken:
 cerberus:
     cerberus_enabled: True                                # Enable it when cerberus is previously installed
     cerberus_url: http://0.0.0.0:8080                     # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal
-    check_applicaton_routes: False                        # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
+    check_application_routes: False                        # When enabled will look for application unavailability using the routes specified in the cerberus config and fails the run
 
 performance_monitoring:
     deploy_dashboards: True                               # Install a mutable grafana and load the performance dashboards. Enable this only when running on OpenShift

containers/Dockerfile.template

@@ -1,22 +1,35 @@
 # oc build
-FROM golang:1.23.1 AS oc-build
+FROM golang:1.24.9 AS oc-build
 RUN apt-get update && apt-get install -y --no-install-recommends libkrb5-dev
 WORKDIR /tmp
+# oc build
 RUN git clone --branch release-4.18 https://github.com/openshift/oc.git
 WORKDIR /tmp/oc
-RUN go mod edit -go 1.23.1 &&\
-    go get github.com/moby/buildkit@v0.12.5 &&\
-    go get github.com/containerd/containerd@v1.7.11&&\
-    go get github.com/docker/docker@v25.0.6&&\
-    go get github.com/opencontainers/runc@v1.1.14&&\
-    go get github.com/go-git/go-git/v5@v5.13.0&&\
-    go get golang.org/x/net@v0.38.0&&\
-    go get github.com/containerd/containerd@v1.7.27&&\
-    go get golang.org/x/oauth2@v0.27.0&&\
-    go get golang.org/x/crypto@v0.35.0&&\
+RUN go mod edit -go 1.24.9 &&\
+    go mod edit -require github.com/moby/buildkit@v0.12.5 &&\
+    go mod edit -require github.com/containerd/containerd@v1.7.29&&\
+    go mod edit -require github.com/docker/docker@v27.5.1+incompatible&&\
+    go mod edit -require github.com/opencontainers/runc@v1.2.8&&\
+    go mod edit -require github.com/go-git/go-git/v5@v5.13.0&&\
+    go mod edit -require github.com/opencontainers/selinux@v1.13.0&&\
+    go mod edit -require github.com/ulikunitz/xz@v0.5.15&&\
+    go mod edit -require golang.org/x/net@v0.38.0&&\
+    go mod edit -require github.com/containerd/containerd@v1.7.27&&\
+    go mod edit -require golang.org/x/oauth2@v0.27.0&&\
+    go mod edit -require golang.org/x/crypto@v0.35.0&&\
+    go mod edit -replace github.com/containerd/containerd@v1.7.27=github.com/containerd/containerd@v1.7.29&&\
     go mod tidy && go mod vendor
+
 RUN make GO_REQUIRED_MIN_VERSION:= oc
 
+# virtctl build
+WORKDIR /tmp
+RUN git clone https://github.com/kubevirt/kubevirt.git
+WORKDIR /tmp/kubevirt
+RUN go mod edit -go 1.24.9 &&\
+    go work use &&\
+    go build -o virtctl ./cmd/virtctl/
+
 FROM fedora:40
 ARG PR_NUMBER
 ARG TAG
@@ -31,18 +44,17 @@ RUN dnf update && dnf install -y --setopt=install_weak_deps=False \
     git python39 jq yq gettext wget which ipmitool openssh-server &&\
     dnf clean all
 
-# Virtctl 
-RUN export VERSION=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt) && \
-    wget https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/virtctl-${VERSION}-linux-amd64 && \
-    chmod +x virtctl-${VERSION}-linux-amd64 && sudo mv virtctl-${VERSION}-linux-amd64 /usr/local/bin/virtctl
-
 # copy oc client binary from oc-build image
 COPY --from=oc-build /tmp/oc/oc /usr/bin/oc
+COPY --from=oc-build /tmp/kubevirt/virtctl /usr/bin/virtctl
 
 # krkn build
 RUN git clone https://github.com/krkn-chaos/krkn.git /home/krkn/kraken && \
     mkdir -p /home/krkn/.kube
 
+RUN mkdir -p /home/krkn/.ssh && \
+    chmod 700 /home/krkn/.ssh
+
 WORKDIR /home/krkn/kraken
 
 # default behaviour will be to build main
@@ -53,6 +65,11 @@ RUN if [ -n "$TAG" ]; then git checkout "$TAG";fi
 
 RUN python3.9 -m ensurepip --upgrade --default-pip
 RUN python3.9 -m pip install --upgrade pip setuptools==78.1.1
+
+# removes the the vulnerable versions of setuptools and pip
+RUN rm -rf "$(pip cache dir)"
+RUN rm -rf /tmp/*
+RUN rm -rf /usr/local/lib/python3.9/ensurepip/_bundled
 RUN pip3.9 install -r requirements.txt
 RUN pip3.9 install jsonschema
 
@@ -60,8 +77,14 @@ LABEL krknctl.title.global="Krkn Base Image"
 LABEL krknctl.description.global="This is the krkn base image."
 LABEL krknctl.input_fields.global='$KRKNCTL_INPUT'
 
+# SSH setup script
+RUN chmod +x /home/krkn/kraken/containers/setup-ssh.sh
+
+# Main entrypoint script
+RUN chmod +x /home/krkn/kraken/containers/entrypoint.sh
 
 RUN chown -R krkn:krkn /home/krkn && chmod 755 /home/krkn
 USER krkn
-ENTRYPOINT ["python3.9", "run_kraken.py"]
+
+ENTRYPOINT ["/bin/bash", "/home/krkn/kraken/containers/entrypoint.sh"]
 CMD ["--config=config/config.yaml"]

containers/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Run SSH setup
+./containers/setup-ssh.sh
+# Change to kraken directory
+
+# Execute the main command
+exec python3.9 run_kraken.py "$@"

containers/krknctl-input.json

@@ -31,6 +31,24 @@
     "separator": ",",
     "required": "false"
   },
+  {
+    "name": "ssh-public-key",
+    "short_description": "Krkn ssh public key path",
+    "description": "Sets the path where krkn will search for ssh public key (in container)",
+    "variable": "KRKN_SSH_PUBLIC",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
+  {
+    "name": "ssh-private-key",
+    "short_description": "Krkn ssh private key path",
+    "description": "Sets the path where krkn will search for ssh private key (in container)",
+    "variable": "KRKN_SSH_PRIVATE",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
   {
     "name": "krkn-kubeconfig",
     "short_description": "Krkn kubeconfig path",
@@ -67,6 +85,24 @@
     "default": "False",
     "required": "false"
   },
+  {
+    "name": "prometheus-url",
+    "short_description": "Prometheus url",
+    "description": "Prometheus url for when running on kuberenetes",
+    "variable": "PROMETHEUS_URL",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
+  {
+    "name": "prometheus-token",
+    "short_description": "Prometheus bearer token",
+    "description": "Prometheus bearer token for prometheus url authentication",
+    "variable": "PROMETHEUS_TOKEN",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
   {
     "name": "uuid",
     "short_description": "Sets krkn run uuid",
@@ -474,6 +510,35 @@
     "default": "False",
     "required": "false"
   },
+  {
+    "name": "kubevirt-ssh-node",
+    "short_description": "KubeVirt node to ssh from",
+    "description": "KubeVirt node to ssh from, should be available whole chaos run",
+    "variable": "KUBE_VIRT_SSH_NODE",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
+  {
+    "name": "kubevirt-exit-on-failure",
+    "short_description": "KubeVirt fail if failed vms at end of run",
+    "description": "KubeVirt fails run if vms still have false status",
+    "variable": "KUBE_VIRT_EXIT_ON_FAIL",
+    "type": "enum",
+    "allowed_values": "True,False,true,false",
+    "separator": ",",
+    "default": "False",
+    "required": "false"
+  },
+  {
+    "name": "kubevirt-node-node",
+    "short_description": "KubeVirt node to filter vms on",
+    "description": "Only track VMs in KubeVirt on given node name",
+    "variable": "KUBE_VIRT_NODE_NAME",
+    "type": "string",
+    "default": "",
+    "required": "false"
+  },
   {
     "name": "krkn-debug",
     "short_description": "Krkn debug mode",

containers/setup-ssh.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Setup SSH key if mounted
+# Support multiple mount locations
+MOUNTED_PRIVATE_KEY_ALT="/secrets/id_rsa"
+MOUNTED_PRIVATE_KEY="/home/krkn/.ssh/id_rsa"
+MOUNTED_PUBLIC_KEY="/home/krkn/.ssh/id_rsa.pub"
+WORKING_KEY="/home/krkn/.ssh/id_rsa.key"
+
+# Determine which source to use
+SOURCE_KEY=""
+if [ -f "$MOUNTED_PRIVATE_KEY_ALT" ]; then
+    SOURCE_KEY="$MOUNTED_PRIVATE_KEY_ALT"
+    echo "Found SSH key at alternative location: $SOURCE_KEY"
+elif [ -f "$MOUNTED_PRIVATE_KEY" ]; then
+    SOURCE_KEY="$MOUNTED_PRIVATE_KEY"
+    echo "Found SSH key at default location: $SOURCE_KEY"
+fi
+
+# Setup SSH private key and create config for outbound connections
+if [ -n "$SOURCE_KEY" ]; then
+    echo "Setting up SSH private key from: $SOURCE_KEY"
+
+    # Check current permissions and ownership
+    ls -la "$SOURCE_KEY"
+
+    # Since the mounted key might be owned by root and we run as krkn user,
+    # we cannot modify it directly. Copy to a new location we can control.
+    echo "Copying SSH key to working location: $WORKING_KEY"
+
+    # Try to copy - if readable by anyone, this will work
+    if cp "$SOURCE_KEY" "$WORKING_KEY" 2>/dev/null || cat "$SOURCE_KEY" > "$WORKING_KEY" 2>/dev/null; then
+        chmod 600 "$WORKING_KEY"
+        echo "SSH key copied successfully"
+        ls -la "$WORKING_KEY"
+
+        # Verify the key is readable
+        if ssh-keygen -y -f "$WORKING_KEY" > /dev/null 2>&1; then
+            echo "SSH private key verified successfully"
+        else
+            echo "Warning: SSH key verification failed, but continuing anyway"
+        fi
+
+        # Create SSH config to use the working key
+        cat > /home/krkn/.ssh/config <<EOF
+Host *
+    IdentityFile $WORKING_KEY
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+EOF
+        chmod 600 /home/krkn/.ssh/config
+        echo "SSH config created with default identity: $WORKING_KEY"
+    else
+        echo "ERROR: Cannot read SSH key at $SOURCE_KEY"
+        echo "Key is owned by: $(stat -c '%U:%G' "$SOURCE_KEY" 2>/dev/null || stat -f '%Su:%Sg' "$SOURCE_KEY" 2>/dev/null)"
+        echo ""
+        echo "Solutions:"
+        echo "1. Mount with world-readable permissions (less secure): chmod 644 /path/to/key"
+        echo "2. Mount to /secrets/id_rsa instead of /home/krkn/.ssh/id_rsa"
+        echo "3. Change ownership on host: chown \$(id -u):\$(id -g) /path/to/key"
+        exit 1
+    fi
+fi
+
+# Setup SSH public key if mounted (for inbound server access)
+if [ -f "$MOUNTED_PUBLIC_KEY" ]; then
+    echo "SSH public key already present at $MOUNTED_PUBLIC_KEY"
+    # Try to fix permissions (will fail silently if file is mounted read-only or owned by another user)
+    chmod 600 "$MOUNTED_PUBLIC_KEY" 2>/dev/null
+    if [ ! -f "/home/krkn/.ssh/authorized_keys" ]; then
+        cp "$MOUNTED_PUBLIC_KEY" /home/krkn/.ssh/authorized_keys
+        chmod 600 /home/krkn/.ssh/authorized_keys
+    fi
+fi

krkn/cerberus/setup.py

@@ -14,7 +14,7 @@ def get_status(config, start_time, end_time):
     if config["cerberus"]["cerberus_enabled"]:
         cerberus_url = config["cerberus"]["cerberus_url"]
         check_application_routes = \
-            config["cerberus"]["check_applicaton_routes"]
+            config["cerberus"]["check_application_routes"]
         if not cerberus_url:
             logging.error(
                 "url where Cerberus publishes True/False signal "

krkn/invoke/command.py

@@ -15,7 +15,7 @@ def invoke(command, timeout=None):
 
 
 # Invokes a given command and returns the stdout
-def invoke_no_exit(command, timeout=None):
+def invoke_no_exit(command, timeout=15):
     output = ""
     try:
         output = subprocess.check_output(command, shell=True, universal_newlines=True, timeout=timeout, stderr=subprocess.DEVNULL)

krkn/prometheus/client.py

@@ -75,10 +75,12 @@ def alerts(
 def critical_alerts(
     prom_cli: KrknPrometheus,
     summary: ChaosRunAlertSummary,
+    elastic: KrknElastic,
     run_id,
     scenario,
     start_time,
     end_time,
+    elastic_alerts_index
 ):
     summary.scenario = scenario
     summary.run_id = run_id
@@ -113,7 +115,6 @@ def critical_alerts(
             summary.chaos_alerts.append(alert)
 
     post_critical_alerts = prom_cli.process_query(query)
-
     for alert in post_critical_alerts:
         if "metric" in alert:
             alertname = (
@@ -136,6 +137,21 @@ def critical_alerts(
             )
             alert = ChaosRunAlert(alertname, alertstate, namespace, severity)
             summary.post_chaos_alerts.append(alert)
+            if elastic:
+                elastic_alert = ElasticAlert(
+                    run_uuid=run_id,
+                    severity=severity,
+                    alert=alertname,
+                    created_at=end_time,
+                    namespace=namespace,
+                    alertstate=alertstate,
+                    phase="post_chaos"
+                )
+                result = elastic.push_alert(elastic_alert, elastic_alerts_index)
+                if result == -1:
+                    logging.error("failed to save alert on ElasticSearch")
+                pass
+
 
     during_critical_alerts_count = len(during_critical_alerts)
     post_critical_alerts_count = len(post_critical_alerts)
@@ -149,8 +165,8 @@ def critical_alerts(
 
     if not firing_alerts:
         logging.info("No critical alerts are firing!!")
-
-
+    
+   
 def metrics(
     prom_cli: KrknPrometheus,
     elastic: KrknElastic,
@@ -252,6 +268,14 @@ def metrics(
                         metric[k] = v
                         metric['timestamp'] = str(datetime.datetime.now())
                     metrics_list.append(metric.copy())
+        if telemetry_json['virt_checks']:
+            for virt_check in telemetry_json["virt_checks"]:
+                    metric_name = "virt_check_recovery"
+                    metric = {"metricName": metric_name}
+                    for k,v in virt_check.items():
+                        metric[k] = v
+                        metric['timestamp'] = str(datetime.datetime.now())
+                    metrics_list.append(metric.copy())
 
         save_metrics = False
         if elastic is not None and elastic_metrics_index is not None:

krkn/rollback/command.py

@@ -3,7 +3,7 @@ import logging
 from typing import Optional, TYPE_CHECKING
 
 from krkn.rollback.config import RollbackConfig
-from krkn.rollback.handler import execute_rollback_version_files, cleanup_rollback_version_files
+from krkn.rollback.handler import execute_rollback_version_files
 
 
 
@@ -96,24 +96,16 @@ def execute_rollback(telemetry_ocp: "KrknTelemetryOpenshift", run_uuid: Optional
     :return: Exit code (0 for success, 1 for error)
     """
     logging.info("Executing rollback version files")
-    
-    if not run_uuid:
-        logging.error("run_uuid is required for execute-rollback command")
-        return 1
-    
-    if not scenario_type:
-        logging.warning("scenario_type is not specified, executing all scenarios in rollback directory")
-    
+    logging.info(f"Executing rollback for run_uuid={run_uuid  or '*'}, scenario_type={scenario_type or '*'}")
+
     try:
         # Execute rollback version files
-        logging.info(f"Executing rollback for run_uuid={run_uuid}, scenario_type={scenario_type or '*'}")
-        execute_rollback_version_files(telemetry_ocp, run_uuid, scenario_type)
-        
-        # If execution was successful, cleanup the version files
-        logging.info("Rollback execution completed successfully, cleaning up version files")
-        cleanup_rollback_version_files(run_uuid, scenario_type)
-        
-        logging.info("Rollback execution and cleanup completed successfully")
+        execute_rollback_version_files(
+            telemetry_ocp,
+            run_uuid,
+            scenario_type,
+            ignore_auto_rollback_config=True
+        )
         return 0
         
     except Exception as e:

krkn/rollback/config.py

@@ -108,7 +108,76 @@ class RollbackConfig(metaclass=SingletonMeta):
         return f"{cls().versions_directory}/{rollback_context}"
     
     @classmethod
-    def search_rollback_version_files(cls, run_uuid: str, scenario_type: str | None = None) -> list[str]:
+    def is_rollback_version_file_format(cls, file_name: str, expected_scenario_type: str | None = None) -> bool:
+        """
+        Validate the format of a rollback version file name.
+
+        Expected format: <scenario_type>_<timestamp>_<hash_suffix>.py
+        where:
+            - scenario_type: string (can include underscores)
+            - timestamp: integer (nanoseconds since epoch)
+            - hash_suffix: alphanumeric string (length 8)
+            - .py: file extension
+
+        :param file_name: The name of the file to validate.
+        :param expected_scenario_type: The expected scenario type (if any) to validate against.
+        :return: True if the file name matches the expected format, False otherwise.
+        """
+        if not file_name.endswith(".py"):
+            return False
+
+        parts = file_name.split("_")
+        if len(parts) < 3:
+            return False
+
+        scenario_type = "_".join(parts[:-2])
+        timestamp_str = parts[-2]
+        hash_suffix_with_ext = parts[-1]
+        hash_suffix = hash_suffix_with_ext[:-3]
+
+        if expected_scenario_type and scenario_type != expected_scenario_type:
+            return False
+
+        if not timestamp_str.isdigit():
+            return False
+
+        if len(hash_suffix) != 8 or not hash_suffix.isalnum():
+            return False
+
+        return True
+    
+    @classmethod
+    def is_rollback_context_directory_format(cls, directory_name: str, expected_run_uuid: str | None = None) -> bool:
+        """
+        Validate the format of a rollback context directory name.
+
+        Expected format: <timestamp>-<run_uuid>
+        where:
+            - timestamp: integer (nanoseconds since epoch)
+            - run_uuid: alphanumeric string
+
+        :param directory_name: The name of the directory to validate.
+        :param expected_run_uuid: The expected run UUID (if any) to validate against.
+        :return: True if the directory name matches the expected format, False otherwise.
+        """
+        parts = directory_name.split("-", 1)
+        if len(parts) != 2:
+            return False
+
+        timestamp_str, run_uuid = parts
+
+        # Validate timestamp is numeric
+        if not timestamp_str.isdigit():
+            return False
+
+        # Validate run_uuid
+        if expected_run_uuid and expected_run_uuid != run_uuid:
+            return False
+
+        return True
+
+    @classmethod
+    def search_rollback_version_files(cls, run_uuid: str | None = None, scenario_type: str | None = None) -> list[str]:
         """
         Search for rollback version files based on run_uuid and scenario_type.
 
@@ -123,34 +192,35 @@ class RollbackConfig(metaclass=SingletonMeta):
         if not os.path.exists(cls().versions_directory):
             return []
 
-        rollback_context_directories = [
-            dirname for dirname in os.listdir(cls().versions_directory) if run_uuid in dirname
-        ]
+        rollback_context_directories = []
+        for dir in os.listdir(cls().versions_directory):
+            if cls.is_rollback_context_directory_format(dir, run_uuid):
+                rollback_context_directories.append(dir)
+            else:
+                logger.warning(f"Directory {dir} does not match expected pattern of <timestamp>-<run_uuid>")
+
         if not rollback_context_directories:
             logger.warning(f"No rollback context directories found for run UUID {run_uuid}")
             return []
 
-        if len(rollback_context_directories) > 1:
-            logger.warning(
-                f"Expected one directory for run UUID {run_uuid}, found: {rollback_context_directories}"
-            )
-
-        rollback_context_directory = rollback_context_directories[0]
 
         version_files = []
-        scenario_rollback_versions_directory = os.path.join(
-            cls().versions_directory, rollback_context_directory
-        )
-        for file in os.listdir(scenario_rollback_versions_directory):
-            # assert all files start with scenario_type and end with .py
-            if file.endswith(".py") and (scenario_type is None or file.startswith(scenario_type)):
-                version_files.append(
-                    os.path.join(scenario_rollback_versions_directory, file)
-                )
-            else:
-                logger.warning(
-                    f"File {file} does not match expected pattern for scenario type {scenario_type}"
-                )
+        for rollback_context_dir in rollback_context_directories:
+            rollback_context_dir = os.path.join(cls().versions_directory, rollback_context_dir)
+
+            for file in os.listdir(rollback_context_dir):
+                # Skip known non-rollback files/directories
+                if file == "__pycache__" or file.endswith(".executed"):
+                    continue
+
+                if cls.is_rollback_version_file_format(file, scenario_type):
+                    version_files.append(
+                        os.path.join(rollback_context_dir, file)
+                    )
+                else:
+                    logger.warning(
+                        f"File {file} does not match expected pattern of <{scenario_type or '*'}>_<timestamp>_<hash_suffix>.py"
+                    )
         return version_files
 
 @dataclass(frozen=True)

krkn/rollback/handler.py

@@ -117,23 +117,32 @@ def _parse_rollback_module(version_file_path: str) -> tuple[RollbackCallable, Ro
     return rollback_callable, rollback_content
 
 
-def execute_rollback_version_files(telemetry_ocp: "KrknTelemetryOpenshift", run_uuid: str, scenario_type: str | None = None):
+def execute_rollback_version_files(
+    telemetry_ocp: "KrknTelemetryOpenshift",
+    run_uuid: str | None = None,
+    scenario_type: str | None = None,
+    ignore_auto_rollback_config: bool = False
+):
     """
     Execute rollback version files for the given run_uuid and scenario_type.
     This function is called when a signal is received to perform rollback operations.
     
     :param run_uuid: Unique identifier for the run.
     :param scenario_type: Type of the scenario being rolled back.
+    :param ignore_auto_rollback_config: Flag to ignore auto rollback configuration. Will be set to True for manual execute-rollback calls.
     """
-    
+    if not ignore_auto_rollback_config and RollbackConfig().auto is False:
+            logger.warning(f"Auto rollback is disabled, skipping execution for run_uuid={run_uuid or '*'}, scenario_type={scenario_type or '*'}")
+            return
+
     # Get the rollback versions directory
     version_files = RollbackConfig.search_rollback_version_files(run_uuid, scenario_type)
     if not version_files:
-        logger.warning(f"Skip execution for run_uuid={run_uuid}, scenario_type={scenario_type or '*'}")
+        logger.warning(f"Skip execution for run_uuid={run_uuid or '*'}, scenario_type={scenario_type or '*'}")
         return
 
     # Execute all version files in the directory
-    logger.info(f"Executing rollback version files for run_uuid={run_uuid}, scenario_type={scenario_type or '*'}")
+    logger.info(f"Executing rollback version files for run_uuid={run_uuid or '*'}, scenario_type={scenario_type or '*'}")
     for version_file in version_files:
         try:
             logger.info(f"Executing rollback version file: {version_file}")
@@ -144,28 +153,37 @@ def execute_rollback_version_files(telemetry_ocp: "KrknTelemetryOpenshift", run_
             logger.info('Executing rollback callable...')
             rollback_callable(rollback_content, telemetry_ocp)
             logger.info('Rollback completed.')
-            
-            logger.info(f"Executed {version_file} successfully.")
+            success = True
         except Exception as e:
+            success = False
             logger.error(f"Failed to execute rollback version file {version_file}: {e}")
             raise
 
+        # Rename the version file with .executed suffix if successful
+        if success:
+            try:
+                executed_file = f"{version_file}.executed"
+                os.rename(version_file, executed_file)
+                logger.info(f"Renamed {version_file} to {executed_file} successfully.")
+            except Exception as e:
+                logger.error(f"Failed to rename rollback version file {version_file}: {e}")
+                raise
 
 def cleanup_rollback_version_files(run_uuid: str, scenario_type: str):
     """
     Cleanup rollback version files for the given run_uuid and scenario_type.
-    This function is called to remove the rollback version files after execution.
+    This function is called to remove the rollback version files after successful scenario execution in run_scenarios.
     
     :param run_uuid: Unique identifier for the run.
     :param scenario_type: Type of the scenario being rolled back.
     """
-    
+
     # Get the rollback versions directory
     version_files = RollbackConfig.search_rollback_version_files(run_uuid, scenario_type)
     if not version_files:
         logger.warning(f"Skip cleanup for run_uuid={run_uuid}, scenario_type={scenario_type or '*'}")
         return
-    
+
     # Remove all version files in the directory
     logger.info(f"Cleaning up rollback version files for run_uuid={run_uuid}, scenario_type={scenario_type}")
     for version_file in version_files:
@@ -176,7 +194,6 @@ def cleanup_rollback_version_files(run_uuid: str, scenario_type: str):
             logger.error(f"Failed to remove rollback version file {version_file}: {e}")
             raise
 
-
 class RollbackHandler:
     def __init__(
         self,

krkn/scenario_plugins/abstract_scenario_plugin.py

@@ -115,14 +115,15 @@ class AbstractScenarioPlugin(ABC):
                     )
                     return_value = 1
 
-            # execute rollback files based on the return value
-            if return_value != 0:
+            if return_value == 0:
+                cleanup_rollback_version_files(
+                    run_uuid, scenario_telemetry.scenario_type
+                )
+            else:
+                # execute rollback files based on the return value
                 execute_rollback_version_files(
                     telemetry, run_uuid, scenario_telemetry.scenario_type
                 )
-            cleanup_rollback_version_files(
-                run_uuid, scenario_telemetry.scenario_type
-            )
             scenario_telemetry.exit_status = return_value
             scenario_telemetry.end_timestamp = time.time()
             utils.collect_and_put_ocp_logs(

krkn/scenario_plugins/application_outage/application_outage_scenario_plugin.py

@@ -34,6 +34,21 @@ class ApplicationOutageScenarioPlugin(AbstractScenarioPlugin):
                 )
                 namespace = get_yaml_item_value(scenario_config, "namespace", "")
                 duration = get_yaml_item_value(scenario_config, "duration", 60)
+                exclude_label = get_yaml_item_value(
+                    scenario_config, "exclude_label", None
+                )
+                match_expressions = self._build_exclude_expressions(exclude_label)
+                if match_expressions:
+                    # Log the format being used for better clarity
+                    format_type = "dict" if isinstance(exclude_label, dict) else "string"
+                    logging.info(
+                        "Excluding pods with labels (%s format): %s",
+                        format_type,
+                        ", ".join(
+                            f"{expr['key']} NOT IN {expr['values']}"
+                            for expr in match_expressions
+                        ),
+                    )
 
                 start_time = int(time.time())
                 policy_name = f"krkn-deny-{get_random_string(5)}"
@@ -43,18 +58,30 @@ class ApplicationOutageScenarioPlugin(AbstractScenarioPlugin):
         apiVersion: networking.k8s.io/v1
         kind: NetworkPolicy
         metadata:
-          name: """
-                    + policy_name
-                    + """
+          name: {{ policy_name }}
         spec:
           podSelector:
             matchLabels: {{ pod_selector }}
+{% if match_expressions %}
+            matchExpressions:
+{% for expression in match_expressions %}
+              - key: {{ expression["key"] }}
+                operator: NotIn
+                values:
+{% for value in expression["values"] %}
+                  - {{ value }}
+{% endfor %}
+{% endfor %}
+{% endif %}
           policyTypes: {{ traffic_type }}
         """
                 )
                 t = Template(network_policy_template)
                 rendered_spec = t.render(
-                    pod_selector=pod_selector, traffic_type=traffic_type
+                    pod_selector=pod_selector,
+                    traffic_type=traffic_type,
+                    match_expressions=match_expressions,
+                    policy_name=policy_name,
                 )
                 yaml_spec = yaml.safe_load(rendered_spec)
                 # Block the traffic by creating network policy
@@ -122,3 +149,63 @@ class ApplicationOutageScenarioPlugin(AbstractScenarioPlugin):
 
     def get_scenario_types(self) -> list[str]:
         return ["application_outages_scenarios"]
+
+    @staticmethod
+    def _build_exclude_expressions(exclude_label) -> list[dict]:
+        """
+        Build match expressions for NetworkPolicy from exclude_label.
+        
+        Supports multiple formats:
+        - Dict format (preferred, similar to pod_selector): {key1: value1, key2: [value2, value3]}
+          Example: {tier: "gold", env: ["prod", "staging"]}
+        - String format: "key1=value1,key2=value2" or "key1=value1|value2"
+          Example: "tier=gold,env=prod" or "tier=gold|platinum"
+        - List format (list of strings): ["key1=value1", "key2=value2"]
+          Example: ["tier=gold", "env=prod"]
+          Note: List elements must be strings in "key=value" format.
+        
+        :param exclude_label: Can be dict, string, list of strings, or None
+        :return: List of match expression dictionaries
+        """
+        expressions: list[dict] = []
+
+        if not exclude_label:
+            return expressions
+
+        def _append_expr(key: str, values):
+            if not key or values is None:
+                return
+            if not isinstance(values, list):
+                values = [values]
+            cleaned_values = [str(v).strip() for v in values if str(v).strip()]
+            if cleaned_values:
+                expressions.append({"key": key.strip(), "values": cleaned_values})
+
+        if isinstance(exclude_label, dict):
+            for k, v in exclude_label.items():
+                _append_expr(str(k), v)
+            return expressions
+
+        if isinstance(exclude_label, list):
+            selectors = exclude_label
+        else:
+            selectors = [sel.strip() for sel in str(exclude_label).split(",")]
+
+        for selector in selectors:
+            if not selector:
+                continue
+            if "=" not in selector:
+                logging.warning(
+                    "exclude_label entry '%s' is invalid, expected key=value format",
+                    selector,
+                )
+                continue
+            key, value = selector.split("=", 1)
+            value_items = (
+                [item.strip() for item in value.split("|") if item.strip()]
+                if value
+                else []
+            )
+            _append_expr(key, value_items or value)
+
+        return expressions

krkn/scenario_plugins/container/container_scenario_plugin.py

@@ -1,6 +1,7 @@
 import logging
 import random
 import time
+import traceback
 from asyncio import Future
 import yaml
 from krkn_lib.k8s import KrknKubernetes
@@ -37,9 +38,12 @@ class ContainerScenarioPlugin(AbstractScenarioPlugin):
                     snapshot = future_snapshot.result()
                     result = snapshot.get_pods_status()
                     scenario_telemetry.affected_pods = result
-
-        except (RuntimeError, Exception):
-            logging.error("ContainerScenarioPlugin exiting due to Exception %s")
+                    if len(result.unrecovered) > 0:
+                        logging.info("ContainerScenarioPlugin failed with unrecovered containers")
+                        return 1
+        except (RuntimeError, Exception) as e:
+            logging.error("Stack trace:\n%s", traceback.format_exc())
+            logging.error("ContainerScenarioPlugin exiting due to Exception %s" % e)
             return 1
         else:
             return 0
@@ -48,7 +52,6 @@ class ContainerScenarioPlugin(AbstractScenarioPlugin):
         return ["container_scenarios"]
 
     def start_monitoring(self, kill_scenario: dict, lib_telemetry: KrknTelemetryOpenshift) -> Future:
-        
         namespace_pattern = f"^{kill_scenario['namespace']}$"
         label_selector = kill_scenario["label_selector"]
         recovery_time = kill_scenario["expected_recovery_time"]
@@ -68,6 +71,7 @@ class ContainerScenarioPlugin(AbstractScenarioPlugin):
         container_name = get_yaml_item_value(cont_scenario, "container_name", "")
         kill_action = get_yaml_item_value(cont_scenario, "action", 1)
         kill_count = get_yaml_item_value(cont_scenario, "count", 1)
+        exclude_label = get_yaml_item_value(cont_scenario, "exclude_label", "")
         if not isinstance(kill_action, int):
             logging.error(
                 "Please make sure the action parameter defined in the "
@@ -89,7 +93,19 @@ class ContainerScenarioPlugin(AbstractScenarioPlugin):
                 pods = kubecli.get_all_pods(label_selector)
             else:
                 # Only returns pod names
-                pods = kubecli.list_pods(namespace, label_selector)
+                # Use list_pods with exclude_label parameter to exclude pods
+                if exclude_label:
+                    logging.info(
+                        "Using exclude_label '%s' to exclude pods from container scenario %s in namespace %s",
+                        exclude_label,
+                        scenario_name,
+                        namespace,
+                    )
+                pods = kubecli.list_pods(
+                    namespace=namespace,
+                    label_selector=label_selector,
+                    exclude_label=exclude_label if exclude_label else None
+                )
         else:
             if namespace == "*":
                 logging.error(
@@ -100,6 +116,7 @@ class ContainerScenarioPlugin(AbstractScenarioPlugin):
                 # sys.exit(1)
                 raise RuntimeError()
             pods = pod_names
+
         # get container and pod name
         container_pod_list = []
         for pod in pods:
@@ -216,4 +233,5 @@ class ContainerScenarioPlugin(AbstractScenarioPlugin):
             timer += 5
             logging.info("Waiting 5 seconds for containers to become ready")
             time.sleep(5)
+
         return killed_container_list

krkn/scenario_plugins/kubevirt_vm_outage/kubevirt_vm_outage_scenario_plugin.py

@@ -25,6 +25,7 @@ class KubevirtVmOutageScenarioPlugin(AbstractScenarioPlugin):
         super().__init__(scenario_type)
         self.k8s_client = None
         self.original_vmi = None
+        self.vmis_list = []
         
     # Scenario type is handled directly in execute_scenario
     def get_scenario_types(self) -> list[str]:
@@ -54,7 +55,8 @@ class KubevirtVmOutageScenarioPlugin(AbstractScenarioPlugin):
                     pods_status.merge(single_pods_status)
             
             scenario_telemetry.affected_pods = pods_status
-                        
+            if len(scenario_telemetry.affected_pods.unrecovered) > 0: 
+                return 1
             return 0
         except Exception as e:
             logging.error(f"KubeVirt VM Outage scenario failed: {e}")
@@ -106,20 +108,20 @@ class KubevirtVmOutageScenarioPlugin(AbstractScenarioPlugin):
         :return: The VMI object if found, None otherwise
         """
         try:
-            vmis = self.custom_object_client.list_namespaced_custom_object(
-                group="kubevirt.io",
-                version="v1",
-                namespace=namespace,
-                plural="virtualmachineinstances",
-            )
+            namespaces = self.k8s_client.list_namespaces_by_regex(namespace)
+            for namespace in namespaces:
+                vmis = self.custom_object_client.list_namespaced_custom_object(
+                    group="kubevirt.io",
+                    version="v1",
+                    namespace=namespace,
+                    plural="virtualmachineinstances",
+                )
 
-            vmi_list = []
-            for vmi in vmis.get("items"):
-                vmi_name = vmi.get("metadata",{}).get("name")
-                match = re.match(regex_name, vmi_name)
-                if match:
-                    vmi_list.append(vmi)
-            return vmi_list
+                for vmi in vmis.get("items"):
+                    vmi_name = vmi.get("metadata",{}).get("name")
+                    match = re.match(regex_name, vmi_name)
+                    if match:
+                        self.vmis_list.append(vmi)
         except ApiException as e:
             if e.status == 404:
                 logging.warning(f"VMI {regex_name} not found in namespace {namespace}")
@@ -149,44 +151,49 @@ class KubevirtVmOutageScenarioPlugin(AbstractScenarioPlugin):
             disable_auto_restart = params.get("disable_auto_restart", False)
             
             if not vm_name:
-                raise Exception("vm_name parameter is required")
-            vmis_list = self.get_vmis(vm_name,namespace)
-            if len(vmis_list) == 0:
-                raise Exception(f"No matching VMs with name {vm_name} in namespace {namespace}")
-            rand_int = random.randint(0, len(vmis_list) - 1)
-            vmi = vmis_list[rand_int]
+                logging.error("vm_name parameter is required")
+                return 1
+            self.pods_status = PodsStatus()
+            self.get_vmis(vm_name,namespace)
+            for _ in range(kill_count):
                 
-            logging.info(f"Starting KubeVirt VM outage scenario for VM: {vm_name} in namespace: {namespace}")
-            vmi_name = vmi.get("metadata").get("name")
-            if not self.validate_environment(vmi_name, namespace):
-                return self.pods_status
-                
-            vmi = self.get_vmi(vmi_name, namespace)
-            self.affected_pod = AffectedPod(
-                pod_name=vmi_name,
-                namespace=namespace,
-            )
-            if not vmi:
-                logging.error(f"VMI {vm_name} not found in namespace {namespace}")
-                return self.pods_status
-                
-            self.original_vmi = vmi
-            logging.info(f"Captured initial state of VMI: {vm_name}")
-            result = self.delete_vmi(vmi_name, namespace, disable_auto_restart)
-            if result != 0:
-                return self.pods_status
+                rand_int = random.randint(0, len(self.vmis_list) - 1)
+                vmi = self.vmis_list[rand_int]
+                    
+                logging.info(f"Starting KubeVirt VM outage scenario for VM: {vm_name} in namespace: {namespace}")
+                vmi_name = vmi.get("metadata").get("name")
+                vmi_namespace = vmi.get("metadata").get("namespace")
+                if not self.validate_environment(vmi_name, vmi_namespace):
+                    return 1
+                    
+                vmi = self.get_vmi(vmi_name, vmi_namespace)
+                self.affected_pod = AffectedPod(
+                    pod_name=vmi_name,
+                    namespace=vmi_namespace,
+                )
+                if not vmi:
+                    logging.error(f"VMI {vm_name} not found in namespace {namespace}")
+                    return 1
+                    
+                self.original_vmi = vmi
+                logging.info(f"Captured initial state of VMI: {vm_name}")
+                result = self.delete_vmi(vmi_name, vmi_namespace, disable_auto_restart)
+                if result != 0:
+                    self.pods_status.unrecovered.append(self.affected_pod)
+                    continue
 
-            result = self.wait_for_running(vmi_name,namespace, timeout)
-            if result != 0:
-                return self.pods_status
-            
-            self.affected_pod.total_recovery_time = (
-                self.affected_pod.pod_readiness_time
-                + self.affected_pod.pod_rescheduling_time
-            )
+                result = self.wait_for_running(vmi_name,vmi_namespace, timeout)
+                if result != 0:
+                    self.pods_status.unrecovered.append(self.affected_pod)
+                    continue
+                
+                self.affected_pod.total_recovery_time = (
+                    self.affected_pod.pod_readiness_time
+                    + self.affected_pod.pod_rescheduling_time
+                )
 
-            self.pods_status.recovered.append(self.affected_pod)
-            logging.info(f"Successfully completed KubeVirt VM outage scenario for VM: {vm_name}")
+                self.pods_status.recovered.append(self.affected_pod)
+                logging.info(f"Successfully completed KubeVirt VM outage scenario for VM: {vm_name}")
             
             return self.pods_status
             
@@ -316,13 +323,13 @@ class KubevirtVmOutageScenarioPlugin(AbstractScenarioPlugin):
                 time.sleep(1)
                 
             logging.error(f"Timed out waiting for VMI {vm_name} to be deleted")
-            self.pods_status.unrecovered = self.affected_pod
+            self.pods_status.unrecovered.append(self.affected_pod)
             return 1
             
         except Exception as e:
             logging.error(f"Error deleting VMI {vm_name}: {e}")
             log_exception(e)
-            self.pods_status.unrecovered = self.affected_pod
+            self.pods_status.unrecovered.append(self.affected_pod)
             return 1
 
     def wait_for_running(self, vm_name: str, namespace: str, timeout: int = 120) -> int: 

krkn/scenario_plugins/native/network/cerberus.py

@@ -27,7 +27,7 @@ def get_status(config, start_time, end_time):
     application_routes_status = True
     if config["cerberus"]["cerberus_enabled"]:
         cerberus_url = config["cerberus"]["cerberus_url"]
-        check_application_routes = config["cerberus"]["check_applicaton_routes"]
+        check_application_routes = config["cerberus"]["check_application_routes"]
         if not cerberus_url:
             logging.error("url where Cerberus publishes True/False signal is not provided.")
             sys.exit(1)

krkn/scenario_plugins/native/pod_network_outage/cerberus.py

@@ -27,7 +27,7 @@ def get_status(config, start_time, end_time):
     application_routes_status = True
     if config["cerberus"]["cerberus_enabled"]:
         cerberus_url = config["cerberus"]["cerberus_url"]
-        check_application_routes = config["cerberus"]["check_applicaton_routes"]
+        check_application_routes = config["cerberus"]["check_application_routes"]
         if not cerberus_url:
             logging.error(
                 "url where Cerberus publishes True/False signal is not provided.")

krkn/scenario_plugins/native/pod_network_outage/kubernetes_functions.py

@@ -23,8 +23,7 @@ def create_job(batch_cli, body, namespace="default"):
     """
 
     try:
-        api_response = batch_cli.create_namespaced_job(
-            body=body, namespace=namespace)
+        api_response = batch_cli.create_namespaced_job(body=body, namespace=namespace)
         return api_response
     except ApiException as api:
         logging.warning(
@@ -71,7 +70,8 @@ def create_pod(cli, body, namespace, timeout=120):
         end_time = time.time() + timeout
         while True:
             pod_stat = cli.read_namespaced_pod(
-                name=body["metadata"]["name"], namespace=namespace)
+                name=body["metadata"]["name"], namespace=namespace
+            )
             if pod_stat.status.phase == "Running":
                 break
             if time.time() > end_time:
@@ -121,16 +121,18 @@ def exec_cmd_in_pod(cli, command, pod_name, namespace, container=None):
     return ret
 
 
-def list_pods(cli, namespace, label_selector=None):
+def list_pods(cli, namespace, label_selector=None, exclude_label=None):
     """
-    Function used to list pods in a given namespace and having a certain label
+    Function used to list pods in a given namespace and having a certain label and excluding pods with exclude_label
+    and excluding pods with exclude_label
     """
 
     pods = []
     try:
         if label_selector:
             ret = cli.list_namespaced_pod(
-                namespace, pretty=True, label_selector=label_selector)
+                namespace, pretty=True, label_selector=label_selector
+            )
         else:
             ret = cli.list_namespaced_pod(namespace, pretty=True)
     except ApiException as e:
@@ -140,7 +142,16 @@ def list_pods(cli, namespace, label_selector=None):
             % e
         )
         raise e
+
     for pod in ret.items:
+        # Skip pods with the exclude label if specified
+        if exclude_label and pod.metadata.labels:
+            exclude_key, exclude_value = exclude_label.split("=", 1)
+            if (
+                exclude_key in pod.metadata.labels
+                and pod.metadata.labels[exclude_key] == exclude_value
+            ):
+                continue
         pods.append(pod.metadata.name)
 
     return pods
@@ -152,8 +163,7 @@ def get_job_status(batch_cli, name, namespace="default"):
     """
 
     try:
-        return batch_cli.read_namespaced_job_status(
-            name=name, namespace=namespace)
+        return batch_cli.read_namespaced_job_status(name=name, namespace=namespace)
     except Exception as e:
         logging.error(
             "Exception when calling \
@@ -169,7 +179,10 @@ def get_pod_log(cli, name, namespace="default"):
     """
 
     return cli.read_namespaced_pod_log(
-        name=name, namespace=namespace, _return_http_data_only=True, _preload_content=False
+        name=name,
+        namespace=namespace,
+        _return_http_data_only=True,
+        _preload_content=False,
     )
 
 
@@ -191,7 +204,8 @@ def delete_job(batch_cli, name, namespace="default"):
             name=name,
             namespace=namespace,
             body=client.V1DeleteOptions(
-                propagation_policy="Foreground", grace_period_seconds=0),
+                propagation_policy="Foreground", grace_period_seconds=0
+            ),
         )
         logging.debug("Job deleted. status='%s'" % str(api_response.status))
         return api_response
@@ -247,11 +261,8 @@ def get_node(node_name, label_selector, instance_kill_count, cli):
         )
     nodes = list_ready_nodes(cli, label_selector)
     if not nodes:
-        raise Exception(
-            "Ready nodes with the provided label selector do not exist")
-    logging.info(
-        "Ready nodes with the label selector %s: %s" % (label_selector, nodes)
-    )
+        raise Exception("Ready nodes with the provided label selector do not exist")
+    logging.info("Ready nodes with the label selector %s: %s" % (label_selector, nodes))
     number_of_nodes = len(nodes)
     if instance_kill_count == number_of_nodes:
         return nodes

krkn/scenario_plugins/native/pod_network_outage/pod_network_outage_plugin.py

@@ -19,7 +19,11 @@ from . import cerberus
 
 
 def get_test_pods(
-    pod_name: str, pod_label: str, namespace: str, kubecli: KrknKubernetes
+    pod_name: str,
+    pod_label: str,
+    namespace: str,
+    kubecli: KrknKubernetes,
+    exclude_label: str = None,
 ) -> typing.List[str]:
     """
     Function that returns a list of pods to apply network policy
@@ -38,11 +42,16 @@ def get_test_pods(
         kubecli (KrknKubernetes)
             - Object to interact with Kubernetes Python client
 
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         pod names (string) in the namespace
     """
     pods_list = []
-    pods_list = kubecli.list_pods(label_selector=pod_label, namespace=namespace)
+    pods_list = kubecli.list_pods(
+        label_selector=pod_label, namespace=namespace, exclude_label=exclude_label
+    )
     if pod_name and pod_name not in pods_list:
         raise Exception("pod name not found in namespace ")
     elif pod_name and pod_name in pods_list:
@@ -226,6 +235,10 @@ def apply_outage_policy(
 
         image (string)
             - Image of network chaos tool
+            
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         The name of the job created that executes the commands on a node
         for ingress chaos scenario
@@ -324,6 +337,9 @@ def apply_ingress_policy(
         test_execution (String)
             - The order in which the filters are applied
 
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         The name of the job created that executes the traffic shaping
         filter
@@ -407,6 +423,9 @@ def apply_net_policy(
         test_execution (String)
             - The order in which the filters are applied
 
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         The name of the job created that executes the traffic shaping
         filter
@@ -466,6 +485,9 @@ def get_ingress_cmd(
         duration (str):
             - Duration for which the traffic control is to be done
 
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         str: ingress filter
     """
@@ -517,6 +539,9 @@ def get_egress_cmd(
         duration (str):
             - Duration for which the traffic control is to be done
 
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         str: egress filter
     """
@@ -652,6 +677,10 @@ def list_bridges(node: str, pod_template, kubecli: KrknKubernetes, image: str) -
 
         image (string)
             - Image of network chaos tool
+            
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         List of bridges on the node.
     """
@@ -829,6 +858,9 @@ def check_bridge_interface(
         kubecli (KrknKubernetes)
             - Object to interact with Kubernetes Python client
 
+        exclude_label (string)
+            - pods matching this label will be excluded from the outage
+
     Returns:
         Returns True if the bridge is found in the  node.
     """
@@ -922,6 +954,15 @@ class InputParams:
         },
     )
 
+    exclude_label: typing.Optional[str] = field(
+        default=None,
+        metadata={
+            "name": "Exclude label",
+            "description": "Kubernetes label selector for pods to exclude from the chaos. "
+            "Pods matching this label will be excluded even if they match the label_selector",
+        },
+    )
+
     kraken_config: typing.Dict[str, typing.Any] = field(
         default=None,
         metadata={
@@ -1055,7 +1096,11 @@ def pod_outage(
 
         br_name = get_bridge_name(api_ext, custom_obj)
         pods_list = get_test_pods(
-            test_pod_name, test_label_selector, test_namespace, kubecli
+            test_pod_name,
+            test_label_selector,
+            test_namespace,
+            kubecli,
+            params.exclude_label,
         )
 
         while not len(pods_list) <= params.instance_count:
@@ -1176,6 +1221,15 @@ class EgressParams:
         },
     )
 
+    exclude_label: typing.Optional[str] = field(
+        default=None,
+        metadata={
+            "name": "Exclude label",
+            "description": "Kubernetes label selector for pods to exclude from the chaos. "
+            "Pods matching this label will be excluded even if they match the label_selector",
+        },
+    )
+
     kraken_config: typing.Dict[str, typing.Any] = field(
         default=None,
         metadata={
@@ -1314,7 +1368,11 @@ def pod_egress_shaping(
 
         br_name = get_bridge_name(api_ext, custom_obj)
         pods_list = get_test_pods(
-            test_pod_name, test_label_selector, test_namespace, kubecli
+            test_pod_name,
+            test_label_selector,
+            test_namespace,
+            kubecli,
+            params.exclude_label,
         )
 
         while not len(pods_list) <= params.instance_count:
@@ -1450,6 +1508,15 @@ class IngressParams:
         },
     )
 
+    exclude_label: typing.Optional[str] = field(
+        default=None,
+        metadata={
+            "name": "Exclude label",
+            "description": "Kubernetes label selector for pods to exclude from the chaos. "
+            "Pods matching this label will be excluded even if they match the label_selector",
+        },
+    )
+
     kraken_config: typing.Dict[str, typing.Any] = field(
         default=None,
         metadata={
@@ -1589,7 +1656,11 @@ def pod_ingress_shaping(
 
         br_name = get_bridge_name(api_ext, custom_obj)
         pods_list = get_test_pods(
-            test_pod_name, test_label_selector, test_namespace, kubecli
+            test_pod_name,
+            test_label_selector,
+            test_namespace,
+            kubecli,
+            params.exclude_label,
         )
 
         while not len(pods_list) <= params.instance_count:

krkn/scenario_plugins/network_chaos_ng/modules/pod_network_filter.py

@@ -1,3 +1,4 @@
+import logging
 import queue
 import time
 
@@ -12,10 +13,9 @@ from krkn.scenario_plugins.network_chaos_ng.models import (
 from krkn.scenario_plugins.network_chaos_ng.modules.abstract_network_chaos_module import (
     AbstractNetworkChaosModule,
 )
-from krkn.scenario_plugins.network_chaos_ng.modules.utils import log_info
+from krkn.scenario_plugins.network_chaos_ng.modules.utils import log_info, log_error
 from krkn.scenario_plugins.network_chaos_ng.modules.utils_network_filter import (
     deploy_network_filter_pod,
-    get_default_interface,
     generate_namespaced_rules,
     apply_network_rules,
     clean_network_rules_namespaced,
@@ -56,23 +56,28 @@ class PodNetworkFilterModule(AbstractNetworkChaosModule):
                 pod_name,
                 self.kubecli.get_lib_kubernetes(),
                 container_name,
+                host_network=False,
             )
 
             if len(self.config.interfaces) == 0:
-                interfaces = [
-                    get_default_interface(
-                        pod_name,
-                        self.config.namespace,
-                        self.kubecli.get_lib_kubernetes(),
+                interfaces = (
+                    self.kubecli.get_lib_kubernetes().list_pod_network_interfaces(
+                        target, self.config.namespace
                     )
-                ]
+                )
 
+                if len(interfaces) == 0:
+                    log_error(
+                        "no network interface found in pod, impossible to execute the network filter scenario",
+                        parallel,
+                        pod_name,
+                    )
+                    return
                 log_info(
-                    f"detected default interface {interfaces[0]}",
+                    f"detected network interfaces: {','.join(interfaces)}",
                     parallel,
                     pod_name,
                 )
-
             else:
                 interfaces = self.config.interfaces
 

krkn/scenario_plugins/network_chaos_ng/modules/utils.py

@@ -11,7 +11,7 @@ def log_info(message: str, parallel: bool = False, node_name: str = ""):
         logging.info(message)
 
 
-def log_error(self, message: str, parallel: bool = False, node_name: str = ""):
+def log_error(message: str, parallel: bool = False, node_name: str = ""):
     """
     log helper method for ERROR severity to be used in the scenarios
     """
@@ -21,7 +21,7 @@ def log_error(self, message: str, parallel: bool = False, node_name: str = ""):
         logging.error(message)
 
 
-def log_warning(self, message: str, parallel: bool = False, node_name: str = ""):
+def log_warning(message: str, parallel: bool = False, node_name: str = ""):
     """
     log helper method for WARNING severity to be used in the scenarios
     """

krkn/scenario_plugins/network_chaos_ng/modules/utils_network_filter.py

@@ -54,6 +54,7 @@ def deploy_network_filter_pod(
     pod_name: str,
     kubecli: KrknKubernetes,
     container_name: str = "fedora",
+    host_network: bool = True,
 ):
     file_loader = FileSystemLoader(os.path.abspath(os.path.dirname(__file__)))
     env = Environment(loader=file_loader, autoescape=True)
@@ -78,17 +79,16 @@ def deploy_network_filter_pod(
             toleration["value"] = value
         tolerations.append(toleration)
 
-
     pod_body = yaml.safe_load(
         pod_template.render(
             pod_name=pod_name,
             namespace=config.namespace,
-            host_network=True,
+            host_network=host_network,
             target=target_node,
             container_name=container_name,
             workload_image=config.image,
             taints=tolerations,
-            service_account=config.service_account
+            service_account=config.service_account,
         )
     )
 

krkn/scenario_plugins/node_actions/abstract_node_scenarios.py

@@ -18,20 +18,20 @@ class abstract_node_scenarios:
         self.node_action_kube_check = node_action_kube_check
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         pass
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         pass
 
     # Node scenario to stop and then start the node
-    def node_stop_start_scenario(self, instance_kill_count, node, timeout, duration):
+    def node_stop_start_scenario(self, instance_kill_count, node, timeout, duration, poll_interval):
         logging.info("Starting node_stop_start_scenario injection")
-        self.node_stop_scenario(instance_kill_count, node, timeout)
+        self.node_stop_scenario(instance_kill_count, node, timeout, poll_interval)
         logging.info("Waiting for %s seconds before starting the node" % (duration))
         time.sleep(duration)
-        self.node_start_scenario(instance_kill_count, node, timeout)
+        self.node_start_scenario(instance_kill_count, node, timeout, poll_interval)
         self.affected_nodes_status.merge_affected_nodes()
         logging.info("node_stop_start_scenario has been successfully injected!")
 
@@ -56,11 +56,11 @@ class abstract_node_scenarios:
             logging.error("node_disk_detach_attach_scenario failed!")
 
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         pass
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         pass
 
     # Node scenario to stop the kubelet

krkn/scenario_plugins/node_actions/alibaba_node_scenarios.py

@@ -234,7 +234,7 @@ class alibaba_node_scenarios(abstract_node_scenarios):
         
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -260,7 +260,7 @@ class alibaba_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -286,7 +286,7 @@ class alibaba_node_scenarios(abstract_node_scenarios):
 
     # Might need to stop and then release the instance
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -316,7 +316,7 @@ class alibaba_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:

krkn/scenario_plugins/node_actions/aws_node_scenarios.py

@@ -77,10 +77,21 @@ class AWS:
     # until a successful state is reached. An error is returned after 40 failed checks
     # Setting timeout for consistency with other cloud functions
     # Wait until the node instance is running
-    def wait_until_running(self, instance_id, timeout=600, affected_node=None):
+    def wait_until_running(self, instance_id, timeout=600, affected_node=None, poll_interval=15):
         try:
             start_time = time.time()
-            self.boto_instance.wait_until_running(InstanceIds=[instance_id])
+            if timeout > 0:
+                max_attempts = max(1, int(timeout / poll_interval))
+            else:
+                max_attempts = 40
+
+            self.boto_instance.wait_until_running(
+                InstanceIds=[instance_id],
+                WaiterConfig={
+                    'Delay': poll_interval,
+                    'MaxAttempts': max_attempts
+                }
+            )
             end_time = time.time()
             if affected_node:
                 affected_node.set_affected_node_status("running", end_time - start_time)
@@ -93,10 +104,21 @@ class AWS:
             return False
 
     # Wait until the node instance is stopped
-    def wait_until_stopped(self, instance_id, timeout=600, affected_node= None):
+    def wait_until_stopped(self, instance_id, timeout=600, affected_node= None, poll_interval=15):
         try:
             start_time = time.time()
-            self.boto_instance.wait_until_stopped(InstanceIds=[instance_id])
+            if timeout > 0:
+                max_attempts = max(1, int(timeout / poll_interval))
+            else:
+                max_attempts = 40
+
+            self.boto_instance.wait_until_stopped(
+                InstanceIds=[instance_id],
+                WaiterConfig={
+                    'Delay': poll_interval,
+                    'MaxAttempts': max_attempts
+                }
+            )
             end_time = time.time()
             if affected_node:
                 affected_node.set_affected_node_status("stopped", end_time - start_time)
@@ -109,10 +131,21 @@ class AWS:
             return False
 
     # Wait until the node instance is terminated
-    def wait_until_terminated(self, instance_id, timeout=600, affected_node= None):
+    def wait_until_terminated(self, instance_id, timeout=600, affected_node= None, poll_interval=15):
         try:
             start_time = time.time()
-            self.boto_instance.wait_until_terminated(InstanceIds=[instance_id])
+            if timeout > 0:
+                max_attempts = max(1, int(timeout / poll_interval))
+            else:
+                max_attempts = 40
+
+            self.boto_instance.wait_until_terminated(
+                InstanceIds=[instance_id],
+                WaiterConfig={
+                    'Delay': poll_interval,
+                    'MaxAttempts': max_attempts
+                }
+            )
             end_time = time.time()
             if affected_node:
                 affected_node.set_affected_node_status("terminated", end_time - start_time)
@@ -267,7 +300,7 @@ class aws_node_scenarios(abstract_node_scenarios):
         self.node_action_kube_check = node_action_kube_check
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -278,7 +311,7 @@ class aws_node_scenarios(abstract_node_scenarios):
                     "Starting the node %s with instance ID: %s " % (node, instance_id)
                 )
                 self.aws.start_instances(instance_id)
-                self.aws.wait_until_running(instance_id, affected_node=affected_node)
+                self.aws.wait_until_running(instance_id, timeout=timeout, affected_node=affected_node, poll_interval=poll_interval)
                 if self.node_action_kube_check: 
                     nodeaction.wait_for_ready_status(node, timeout, self.kubecli, affected_node)
                 logging.info(
@@ -296,7 +329,7 @@ class aws_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -307,7 +340,7 @@ class aws_node_scenarios(abstract_node_scenarios):
                     "Stopping the node %s with instance ID: %s " % (node, instance_id)
                 )
                 self.aws.stop_instances(instance_id)
-                self.aws.wait_until_stopped(instance_id, affected_node=affected_node)
+                self.aws.wait_until_stopped(instance_id, timeout=timeout, affected_node=affected_node, poll_interval=poll_interval)
                 logging.info(
                     "Node with instance ID: %s is in stopped state" % (instance_id)
                 )
@@ -324,7 +357,7 @@ class aws_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -336,7 +369,7 @@ class aws_node_scenarios(abstract_node_scenarios):
                     % (node, instance_id)
                 )
                 self.aws.terminate_instances(instance_id)
-                self.aws.wait_until_terminated(instance_id, affected_node=affected_node)
+                self.aws.wait_until_terminated(instance_id, timeout=timeout, affected_node=affected_node, poll_interval=poll_interval)
                 for _ in range(timeout):
                     if node not in self.kubecli.list_nodes():
                         break
@@ -358,7 +391,7 @@ class aws_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:

krkn/scenario_plugins/node_actions/az_node_scenarios.py

@@ -18,8 +18,6 @@ class Azure:
         logging.info("azure " + str(self))
         # Acquire a credential object using CLI-based authentication.
         credentials = DefaultAzureCredential()
-        # az_account = runcommand.invoke("az account list -o yaml")
-        # az_account_yaml = yaml.safe_load(az_account, Loader=yaml.FullLoader)
         logger = logging.getLogger("azure")
         logger.setLevel(logging.WARNING)
         subscription_id = os.getenv("AZURE_SUBSCRIPTION_ID")
@@ -218,7 +216,7 @@ class azure_node_scenarios(abstract_node_scenarios):
         
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -246,7 +244,7 @@ class azure_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -273,7 +271,7 @@ class azure_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -308,7 +306,7 @@ class azure_node_scenarios(abstract_node_scenarios):
 
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:

krkn/scenario_plugins/node_actions/bm_node_scenarios.py

@@ -153,7 +153,7 @@ class bm_node_scenarios(abstract_node_scenarios):
         self.node_action_kube_check = node_action_kube_check
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -182,7 +182,7 @@ class bm_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -210,11 +210,11 @@ class bm_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         logging.info("Node termination scenario is not supported on baremetal")
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:

krkn/scenario_plugins/node_actions/common_node_functions.py

@@ -16,20 +16,22 @@ def get_node_by_name(node_name_list, kubecli: KrknKubernetes):
             )
             return
     return node_name_list
-        
+
 
 # Pick a random node with specified label selector
 def get_node(label_selector, instance_kill_count, kubecli: KrknKubernetes):
 
-    label_selector_list  = label_selector.split(",")
+    label_selector_list = label_selector.split(",")
     nodes = []
-    for label_selector in label_selector_list: 
+    for label_selector in label_selector_list:
         nodes.extend(kubecli.list_killable_nodes(label_selector))
     if not nodes:
         raise Exception("Ready nodes with the provided label selector do not exist")
-    logging.info("Ready nodes with the label selector %s: %s" % (label_selector_list, nodes))
+    logging.info(
+        "Ready nodes with the label selector %s: %s" % (label_selector_list, nodes)
+    )
     number_of_nodes = len(nodes)
-    if instance_kill_count == number_of_nodes:
+    if instance_kill_count == number_of_nodes or instance_kill_count == 0:
         return nodes
     nodes_to_return = []
     for i in range(instance_kill_count):
@@ -38,23 +40,30 @@ def get_node(label_selector, instance_kill_count, kubecli: KrknKubernetes):
         nodes.remove(node_to_add)
     return nodes_to_return
 
+
 # krkn_lib
 # Wait until the node status becomes Ready
-def wait_for_ready_status(node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None):
-    affected_node =  kubecli.watch_node_status(node, "True", timeout, affected_node)
+def wait_for_ready_status(
+    node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None
+):
+    affected_node = kubecli.watch_node_status(node, "True", timeout, affected_node)
     return affected_node
-   
+
 
 # krkn_lib
 # Wait until the node status becomes Not Ready
-def wait_for_not_ready_status(node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None):
+def wait_for_not_ready_status(
+    node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None
+):
     affected_node = kubecli.watch_node_status(node, "False", timeout, affected_node)
     return affected_node
-    
+
 
 # krkn_lib
 # Wait until the node status becomes Unknown
-def wait_for_unknown_status(node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None):
+def wait_for_unknown_status(
+    node, timeout, kubecli: KrknKubernetes, affected_node: AffectedNode = None
+):
     affected_node = kubecli.watch_node_status(node, "Unknown", timeout, affected_node)
     return affected_node
 

krkn/scenario_plugins/node_actions/docker_node_scenarios.py

@@ -2,49 +2,176 @@ import krkn.scenario_plugins.node_actions.common_node_functions as nodeaction
 from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
     abstract_node_scenarios,
 )
+import os
+import platform
 import logging
 import docker
 from krkn_lib.k8s import KrknKubernetes
 from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
 
 class Docker:
+    """
+    Container runtime client wrapper supporting both Docker and Podman.
+
+    This class automatically detects and connects to either Docker or Podman
+    container runtimes using the Docker-compatible API. It tries multiple
+    connection methods in order of preference:
+
+    1. Docker Unix socket (unix:///var/run/docker.sock)
+    2. Platform-specific Podman sockets:
+       - macOS: ~/.local/share/containers/podman/machine/podman.sock
+       - Linux rootful: unix:///run/podman/podman.sock
+       - Linux rootless: unix:///run/user/<uid>/podman/podman.sock
+    3. Environment variables (DOCKER_HOST or CONTAINER_HOST)
+
+    The runtime type (docker/podman) is auto-detected and logged for debugging.
+    Supports Kind clusters running on Podman.
+
+    Assisted By: Claude Code
+    """
     def __init__(self):
-        self.client = docker.from_env()
+        self.client = None
+        self.runtime = 'unknown'
+        
+
+        # Try multiple connection methods in order of preference
+        # Supports both Docker and Podman
+        connection_methods = [
+            ('unix:///var/run/docker.sock', 'Docker Unix socket'),
+        ]
+
+        # Add platform-specific Podman sockets
+        if platform.system() == 'Darwin':  # macOS
+            # On macOS, Podman uses podman-machine with socket typically at:
+            # ~/.local/share/containers/podman/machine/podman.sock
+            # This is often symlinked to /var/run/docker.sock
+            podman_machine_sock = os.path.expanduser('~/.local/share/containers/podman/machine/podman.sock')
+            if os.path.exists(podman_machine_sock):
+                connection_methods.append((f'unix://{podman_machine_sock}', 'Podman machine socket (macOS)'))
+        else:  # Linux
+            connection_methods.extend([
+                ('unix:///run/podman/podman.sock', 'Podman Unix socket (rootful)'),
+                ('unix:///run/user/{uid}/podman/podman.sock', 'Podman Unix socket (rootless)'),
+            ])
+
+        # Always try from_env as last resort
+        connection_methods.append(('from_env', 'Environment variables (DOCKER_HOST/CONTAINER_HOST)'))
+
+        for method, description in connection_methods:
+            try:
+                # Handle rootless Podman socket path with {uid} placeholder
+                if '{uid}' in method:
+                    uid = os.getuid()
+                    method = method.format(uid=uid)
+                    logging.info(f'Attempting to connect using {description}: {method}')
+
+                if method == 'from_env':
+                    logging.info(f'Attempting to connect using {description}')
+                    self.client = docker.from_env()
+                else:
+                    logging.info(f'Attempting to connect using {description}: {method}')
+                    self.client = docker.DockerClient(base_url=method)
+
+                # Test the connection
+                self.client.ping()
+
+                # Detect runtime type
+                try:
+                    version_info = self.client.version()
+                    version_str = version_info.get('Version', '')
+                    if 'podman' in version_str.lower():
+                        self.runtime = 'podman'
+                    else:
+                        self.runtime = 'docker'
+                    logging.debug(f'Runtime version info: {version_str}')
+                except Exception as version_err:
+                    logging.warning(f'Could not detect runtime version: {version_err}')
+                    self.runtime = 'unknown'
+
+                logging.info(f'Successfully connected to {self.runtime} using {description}')
+
+                # Log available containers for debugging
+                try:
+                    containers = self.client.containers.list(all=True)
+                    logging.info(f'Found {len(containers)} total containers')
+                    for container in containers[:5]:  # Log first 5
+                        logging.debug(f'  Container: {container.name} ({container.status})')
+                except Exception as list_err:
+                    logging.warning(f'Could not list containers: {list_err}')
+
+                break
+
+            except Exception as e:
+                logging.warning(f'Failed to connect using {description}: {e}')
+                continue
+
+        if self.client is None:
+            error_msg = 'Failed to initialize container runtime client (Docker/Podman) with any connection method'
+            logging.error(error_msg)
+            logging.error('Attempted connection methods:')
+            for method, desc in connection_methods:
+                logging.error(f'  - {desc}: {method}')
+            raise RuntimeError(error_msg)
+
+        logging.info(f'Container runtime client initialized successfully: {self.runtime}')
 
     def get_container_id(self, node_name):
+        """Get the container ID for a given node name."""
         container = self.client.containers.get(node_name)
+        logging.info(f'Found {self.runtime} container for node {node_name}: {container.id}')
         return container.id
 
     # Start the node instance
     def start_instances(self, node_name):
+        """Start a container instance (works with both Docker and Podman)."""
+        logging.info(f'Starting {self.runtime} container for node: {node_name}')
         container = self.client.containers.get(node_name)
         container.start()
+        logging.info(f'Container {container.id} started successfully')
 
     # Stop the node instance
     def stop_instances(self, node_name):
+        """Stop a container instance (works with both Docker and Podman)."""
+        logging.info(f'Stopping {self.runtime} container for node: {node_name}')
         container = self.client.containers.get(node_name)
         container.stop()
+        logging.info(f'Container {container.id} stopped successfully')
 
     # Reboot the node instance
     def reboot_instances(self, node_name):
+        """Restart a container instance (works with both Docker and Podman)."""
+        logging.info(f'Restarting {self.runtime} container for node: {node_name}')
         container = self.client.containers.get(node_name)
         container.restart()
+        logging.info(f'Container {container.id} restarted successfully')
 
     # Terminate the node instance
     def terminate_instances(self, node_name):
+        """Stop and remove a container instance (works with both Docker and Podman)."""
+        logging.info(f'Terminating {self.runtime} container for node: {node_name}')
         container = self.client.containers.get(node_name)
         container.stop()
         container.remove()
+        logging.info(f'Container {container.id} terminated and removed successfully')
 
 
 class docker_node_scenarios(abstract_node_scenarios):
+    """
+    Node chaos scenarios for containerized Kubernetes nodes.
+
+    Supports both Docker and Podman container runtimes. This class provides
+    methods to inject chaos into Kubernetes nodes running as containers
+    (e.g., Kind clusters, Podman-based clusters).
+    """
     def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus):
+        logging.info('Initializing docker_node_scenarios (supports Docker and Podman)')
         super().__init__(kubecli, node_action_kube_check, affected_nodes_status)
         self.docker = Docker()
         self.node_action_kube_check = node_action_kube_check
+        logging.info(f'Node scenarios initialized successfully using {self.docker.runtime} runtime')
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -71,7 +198,7 @@ class docker_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -97,7 +224,7 @@ class docker_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             try:
                 logging.info("Starting node_termination_scenario injection")
@@ -120,7 +247,7 @@ class docker_node_scenarios(abstract_node_scenarios):
                 raise e
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:

krkn/scenario_plugins/node_actions/gcp_node_scenarios.py

@@ -227,7 +227,7 @@ class gcp_node_scenarios(abstract_node_scenarios):
         self.node_action_kube_check = node_action_kube_check
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -257,7 +257,7 @@ class gcp_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -286,7 +286,7 @@ class gcp_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -321,7 +321,7 @@ class gcp_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:

krkn/scenario_plugins/node_actions/general_cloud_node_scenarios.py

@@ -18,28 +18,28 @@ class general_node_scenarios(abstract_node_scenarios):
         self.node_action_kube_check = node_action_kube_check
 
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         logging.info(
             "Node start is not set up yet for this cloud type, "
             "no action is going to be taken"
         )
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         logging.info(
             "Node stop is not set up yet for this cloud type,"
             " no action is going to be taken"
         )
 
     # Node scenario to terminate the node
-    def node_termination_scenario(self, instance_kill_count, node, timeout):
+    def node_termination_scenario(self, instance_kill_count, node, timeout, poll_interval):
         logging.info(
             "Node termination is not set up yet for this cloud type, "
             "no action is going to be taken"
         )
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         logging.info(
             "Node reboot is not set up yet for this cloud type,"
             " no action is going to be taken"

krkn/scenario_plugins/node_actions/ibmcloud_node_scenarios.py

@@ -284,7 +284,7 @@ class ibm_node_scenarios(abstract_node_scenarios):
         
         self.node_action_kube_check = node_action_kube_check
 
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         try:
             instance_id = self.ibmcloud.get_instance_id( node)
             affected_node = AffectedNode(node, node_id=instance_id)
@@ -317,7 +317,7 @@ class ibm_node_scenarios(abstract_node_scenarios):
         self.affected_nodes_status.affected_nodes.append(affected_node)
 
 
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         try:
             instance_id = self.ibmcloud.get_instance_id(node)
             for _ in range(instance_kill_count):
@@ -327,46 +327,59 @@ class ibm_node_scenarios(abstract_node_scenarios):
                 vm_stopped = self.ibmcloud.stop_instances(instance_id)
                 if vm_stopped:
                     self.ibmcloud.wait_until_stopped(instance_id, timeout, affected_node)
-                logging.info(
-                    "Node with instance ID: %s is in stopped state" % node
-                )
-                logging.info(
-                    "node_stop_scenario has been successfully injected!"
-                )
+                    logging.info(
+                        "Node with instance ID: %s is in stopped state" % node
+                    )
+                    logging.info(
+                        "node_stop_scenario has been successfully injected!"
+                    )
+                else:
+                    logging.error(
+                        "Failed to stop node instance %s. Stop command failed." % instance_id
+                    )
+                    raise Exception("Stop command failed for instance %s" % instance_id)
+                self.affected_nodes_status.affected_nodes.append(affected_node)
         except Exception as e:
-            logging.error("Failed to stop node instance. Test Failed")
+            logging.error("Failed to stop node instance. Test Failed: %s" % str(e))
             logging.error("node_stop_scenario injection failed!")
 
 
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         try:
             instance_id = self.ibmcloud.get_instance_id(node)
             for _ in range(instance_kill_count):
                 affected_node = AffectedNode(node, node_id=instance_id)
                 logging.info("Starting node_reboot_scenario injection")
                 logging.info("Rebooting the node %s " % (node))
-                self.ibmcloud.reboot_instances(instance_id)
-                self.ibmcloud.wait_until_rebooted(instance_id, timeout, affected_node)
-                if self.node_action_kube_check:
-                    nodeaction.wait_for_unknown_status(
-                        node, timeout, affected_node
+                vm_rebooted = self.ibmcloud.reboot_instances(instance_id)
+                if vm_rebooted:
+                    self.ibmcloud.wait_until_rebooted(instance_id, timeout, affected_node)
+                    if self.node_action_kube_check:
+                        nodeaction.wait_for_unknown_status(
+                            node, timeout, self.kubecli, affected_node
+                        )
+                        nodeaction.wait_for_ready_status(
+                            node, timeout, self.kubecli, affected_node
+                        )
+                    logging.info(
+                        "Node with instance ID: %s has rebooted successfully" % node
                     )
-                    nodeaction.wait_for_ready_status(
-                        node, timeout, affected_node
+                    logging.info(
+                        "node_reboot_scenario has been successfully injected!"
                     )
-                logging.info(
-                    "Node with instance ID: %s has rebooted successfully" % node
-                )
-                logging.info(
-                    "node_reboot_scenario has been successfully injected!"
-                )
+                else:
+                    logging.error(
+                        "Failed to reboot node instance %s. Reboot command failed." % instance_id
+                    )
+                    raise Exception("Reboot command failed for instance %s" % instance_id)
+                self.affected_nodes_status.affected_nodes.append(affected_node)
 
         except Exception as e:
-            logging.error("Failed to reboot node instance. Test Failed")
+            logging.error("Failed to reboot node instance. Test Failed: %s" % str(e))
             logging.error("node_reboot_scenario injection failed!")
 
 
-    def node_terminate_scenario(self, instance_kill_count, node, timeout):
+    def node_terminate_scenario(self, instance_kill_count, node, timeout, poll_interval):
         try:
             instance_id = self.ibmcloud.get_instance_id(node)
             for _ in range(instance_kill_count):
@@ -383,7 +396,8 @@ class ibm_node_scenarios(abstract_node_scenarios):
                 logging.info(
                     "node_terminate_scenario has been successfully injected!"
                 )
+                self.affected_nodes_status.affected_nodes.append(affected_node)
         except Exception as e:
-            logging.error("Failed to terminate node instance. Test Failed")
+            logging.error("Failed to terminate node instance. Test Failed: %s" % str(e))
             logging.error("node_terminate_scenario injection failed!")
 

krkn/scenario_plugins/node_actions/ibmcloud_power_node_scenarios.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python
+import time
+from os import environ
+from dataclasses import dataclass
+import logging
+
+from krkn_lib.k8s import KrknKubernetes
+import krkn.scenario_plugins.node_actions.common_node_functions as nodeaction
+from krkn.scenario_plugins.node_actions.abstract_node_scenarios import (
+    abstract_node_scenarios,
+)
+import requests
+import sys
+import json
+
+
+#  -o, --operation string   Operation to be done in a PVM server instance. 
+# Valid values are: hard-reboot, immediate-shutdown, soft-reboot, reset-state, start, stop.
+
+from krkn_lib.models.k8s import AffectedNodeStatus, AffectedNode
+
+
+class IbmCloudPower:
+    def __init__(self):
+        """
+        Initialize the ibm cloud client by using the the env variables:
+            'IBMC_APIKEY' 'IBMC_URL'
+        """
+        self.api_key = environ.get("IBMC_APIKEY")
+        self.service_url = environ.get("IBMC_POWER_URL")
+        self.CRN = environ.get("IBMC_POWER_CRN")
+        self.cloud_instance_id = self.CRN.split(":")[-3]
+        print(self.cloud_instance_id)
+        self.headers = None
+        self.token = None
+        if not self.api_key:
+            raise Exception("Environmental variable 'IBMC_APIKEY' is not set")
+        if not self.service_url:
+            raise Exception("Environmental variable 'IBMC_POWER_URL' is not set")
+        if not self.CRN:
+            raise Exception("Environmental variable 'IBMC_POWER_CRN' is not set")
+        try:
+            self.authenticate()
+            
+        except Exception as e:
+            logging.error("error authenticating" + str(e))
+    
+    def authenticate(self):
+        url = "https://iam.cloud.ibm.com/identity/token"
+        iam_auth_headers = {
+            "content-type": "application/x-www-form-urlencoded",
+            "accept": "application/json",
+        }
+        data = {
+            "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
+            "apikey": self.api_key,
+        }
+
+        response = self._make_request("POST", url, data=data, headers=iam_auth_headers)
+        if response.status_code == 200:
+            self.token = response.json()
+            self.headers = {
+                "Authorization": f"Bearer {self.token['access_token']}",
+                "Content-Type": "application/json",
+                "CRN": self.CRN,
+            }
+        else:
+            logging.error(f"Authentication Error: {response.status_code}")
+            return None, None
+            
+
+    def _make_request(self,method, url, data=None, headers=None):
+        try:
+            response = requests.request(method, url, data=data, headers=headers)
+            response.raise_for_status()
+            return response
+        except Exception as e:
+            raise Exception(f"API Error: {e}")
+
+    # Get the instance ID of the node
+    def get_instance_id(self, node_name):
+
+        url = f"{self.service_url}/pcloud/v1/cloud-instances/{self.cloud_instance_id}/pvm-instances/"
+        response = self._make_request("GET", url, headers=self.headers)
+        for node in response.json()["pvmInstances"]:
+            if node_name == node["serverName"]:
+                return node["pvmInstanceID"]
+        logging.error("Couldn't find node with name " + str(node_name) + ", you could try another region")
+        sys.exit(1)
+
+    def delete_instance(self, instance_id):
+        """
+        Deletes the Instance whose name is given by 'instance_id'
+        """
+        try:
+            url = f"{self.service_url}/pcloud/v1/cloud-instances/{self.cloud_instance_id}/pvm-instances/{instance_id}/action"
+            self._make_request("POST", url, headers=self.headers, data=json.dumps({"action": "immediate-shutdown"}))
+            logging.info("Deleted Instance -- '{}'".format(instance_id))
+        except Exception as e:
+            logging.info("Instance '{}' could not be deleted. ".format(instance_id))
+            return False
+
+    def reboot_instances(self, instance_id, soft=False):
+        """
+        Reboots the Instance whose name is given by 'instance_id'. Returns True if successful, or
+        returns False if the Instance is not powered on
+        """
+
+        try:
+            if soft:
+                action = "soft-reboot"
+            else:
+                action = "hard-reboot"
+            url = f"{self.service_url}/pcloud/v1/cloud-instances/{self.cloud_instance_id}/pvm-instances/{instance_id}/action"
+            self._make_request("POST", url, headers=self.headers, data=json.dumps({"action": action}))
+            logging.info("Reset Instance -- '{}'".format(instance_id))
+            return True
+        except Exception as e:
+            logging.info("Instance '{}' could not be rebooted".format(instance_id))
+            return False
+
+    def stop_instances(self, instance_id):
+        """
+        Stops the Instance whose name is given by 'instance_id'. Returns True if successful, or
+        returns False if the Instance is already stopped
+        """
+
+        try:
+            url = f"{self.service_url}/pcloud/v1/cloud-instances/{self.cloud_instance_id}/pvm-instances/{instance_id}/action"
+            self._make_request("POST", url, headers=self.headers, data=json.dumps({"action": "stop"}))
+            logging.info("Stopped Instance -- '{}'".format(instance_id))
+            return True
+        except Exception as e:
+            logging.info("Instance '{}' could not be stopped".format(instance_id))
+            logging.info("error" + str(e))
+            return False
+
+    def start_instances(self, instance_id):
+        """
+        Stops the Instance whose name is given by 'instance_id'. Returns True if successful, or
+        returns False if the Instance is already running
+        """
+
+        try:
+            url = f"{self.service_url}/pcloud/v1/cloud-instances/{self.cloud_instance_id}/pvm-instances/{instance_id}/action"
+            self._make_request("POST", url, headers=self.headers, data=json.dumps({"action": "start"}))
+            logging.info("Started Instance -- '{}'".format(instance_id))
+            return True
+        except Exception as e:
+            logging.info("Instance '{}' could not start running".format(instance_id))
+            return False
+
+    def list_instances(self):
+        """
+        Returns a list of Instances present in the datacenter
+        """
+        instance_names = []
+        try:
+            url = f"{self.service_url}/pcloud/v1/cloud-instances/{self.cloud_instance_id}/pvm-instances/"
+            response = self._make_request("GET", url, headers=self.headers)
+            for pvm in response.json()["pvmInstances"]:
+                instance_names.append({"serverName": pvm.serverName, "pvmInstanceID": pvm.pvmInstanceID})
+        except Exception as e:
+            logging.error("Error listing out instances: " + str(e))
+            sys.exit(1)
+        return instance_names
+
+    def find_id_in_list(self, name, vpc_list):
+        for vpc in vpc_list:
+            if vpc["vpc_name"] == name:
+                return vpc["vpc_id"]
+
+    def get_instance_status(self, instance_id):
+        """
+        Returns the status of the Instance whose name is given by 'instance_id'
+        """
+
+        try:
+            url = f"{self.service_url}/pcloud/v1/cloud-instances/{self.cloud_instance_id}/pvm-instances/{instance_id}"
+            response = self._make_request("GET", url, headers=self.headers)
+            state = response.json()["status"]
+            return state
+        except Exception as e:
+            logging.error(
+                "Failed to get node instance status %s. Encountered following "
+                "exception: %s." % (instance_id, e)
+            )
+            return None
+
+    def wait_until_deleted(self, instance_id, timeout, affected_node=None):
+        """
+        Waits until the instance is deleted or until the timeout. Returns True if
+        the instance is successfully deleted, else returns False
+        """
+        start_time = time.time()
+        time_counter = 0
+        vpc = self.get_instance_status(instance_id)
+        while vpc is not None:
+            vpc = self.get_instance_status(instance_id)
+            logging.info(
+                "Instance %s is still being deleted, sleeping for 5 seconds"
+                % instance_id
+            )
+            time.sleep(5)
+            time_counter += 5
+            if time_counter >= timeout:
+                logging.info(
+                    "Instance %s is still not deleted in allotted time" % instance_id
+                )
+                return False
+        end_time = time.time()
+        if affected_node:
+            affected_node.set_affected_node_status("terminated", end_time - start_time)
+        return True
+
+    def wait_until_running(self, instance_id, timeout, affected_node=None):
+        """
+        Waits until the Instance switches to running state or until the timeout.
+        Returns True if the Instance switches to running, else returns False
+        """
+        start_time = time.time()
+        time_counter = 0
+        status = self.get_instance_status(instance_id)
+        while status != "ACTIVE":
+            status = self.get_instance_status(instance_id)
+            logging.info(
+                "Instance %s is still not running, sleeping for 5 seconds" % instance_id
+            )
+            time.sleep(5)
+            time_counter += 5
+            if time_counter >= timeout:
+                logging.info(
+                    "Instance %s is still not ready in allotted time" % instance_id
+                )
+                return False
+        end_time = time.time()
+        if affected_node:
+            affected_node.set_affected_node_status("running", end_time - start_time)
+        return True
+
+    def wait_until_stopped(self, instance_id, timeout, affected_node):
+        """
+        Waits until the Instance switches to stopped state or until the timeout.
+        Returns True if the Instance switches to stopped, else returns False
+        """
+        start_time = time.time()
+        time_counter = 0
+        status = self.get_instance_status(instance_id)
+        while status != "STOPPED":
+            status = self.get_instance_status(instance_id)
+            logging.info(
+                "Instance %s is still not stopped, sleeping for 5 seconds" % instance_id
+            )
+            time.sleep(5)
+            time_counter += 5
+            if time_counter >= timeout:
+                logging.info(
+                    "Instance %s is still not stopped in allotted time" % instance_id
+                )
+                return False
+        end_time = time.time()
+        print('affected_node' + str(affected_node))
+        if affected_node:
+            affected_node.set_affected_node_status("stopped", end_time - start_time)
+        return True
+
+
+    def wait_until_rebooted(self, instance_id, timeout, affected_node):
+        """
+        Waits until the Instance switches to restarting state and then running state or until the timeout.
+        Returns True if the Instance switches back to running, else returns False
+        """
+
+        time_counter = 0
+        status = self.get_instance_status(instance_id)
+        while status == "HARD_REBOOT" or status == "SOFT_REBOOT":
+            status = self.get_instance_status(instance_id)
+            logging.info(
+                "Instance %s is still restarting, sleeping for 5 seconds" % instance_id
+            )
+            time.sleep(5)
+            time_counter += 5
+            if time_counter >= timeout:
+                logging.info(
+                    "Instance %s is still restarting after allotted time" % instance_id
+                )
+                return False
+        self.wait_until_running(instance_id, timeout, affected_node)
+        print('affected_node' + str(affected_node))
+        return True
+
+
+@dataclass
+class ibmcloud_power_node_scenarios(abstract_node_scenarios):
+    def __init__(self, kubecli: KrknKubernetes, node_action_kube_check: bool, affected_nodes_status: AffectedNodeStatus, disable_ssl_verification: bool):
+        super().__init__(kubecli, node_action_kube_check, affected_nodes_status)
+        self.ibmcloud_power = IbmCloudPower()
+        
+        self.node_action_kube_check = node_action_kube_check
+
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
+        try:
+            instance_id = self.ibmcloud_power.get_instance_id( node)
+            affected_node = AffectedNode(node, node_id=instance_id)
+            for _ in range(instance_kill_count):
+                logging.info("Starting node_start_scenario injection")
+                logging.info("Starting the node %s " % (node))
+                
+                if instance_id:
+                    vm_started = self.ibmcloud_power.start_instances(instance_id)
+                    if vm_started:
+                        self.ibmcloud_power.wait_until_running(instance_id, timeout, affected_node)
+                        if self.node_action_kube_check: 
+                            nodeaction.wait_for_ready_status(
+                                node, timeout, self.kubecli, affected_node
+                            )
+                    logging.info(
+                        "Node with instance ID: %s is in running state" % node
+                    )
+                    logging.info(
+                        "node_start_scenario has been successfully injected!"
+                    )
+                else:
+                    logging.error(
+                        "Failed to find node that matched instances on ibm cloud in region"
+                    )
+
+        except Exception as e:
+            logging.error("Failed to start node instance. Test Failed")
+            logging.error("node_start_scenario injection failed!")
+        self.affected_nodes_status.affected_nodes.append(affected_node)
+
+
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
+        try:
+            instance_id = self.ibmcloud_power.get_instance_id(node)
+            for _ in range(instance_kill_count):
+                affected_node = AffectedNode(node, instance_id)
+                logging.info("Starting node_stop_scenario injection")
+                logging.info("Stopping the node %s " % (node))
+                vm_stopped = self.ibmcloud_power.stop_instances(instance_id)
+                if vm_stopped:
+                    self.ibmcloud_power.wait_until_stopped(instance_id, timeout, affected_node)
+                logging.info(
+                    "Node with instance ID: %s is in stopped state" % node
+                )
+                logging.info(
+                    "node_stop_scenario has been successfully injected!"
+                )
+        except Exception as e:
+            logging.error("Failed to stop node instance. Test Failed")
+            logging.error("node_stop_scenario injection failed!")
+
+
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
+        try:
+            instance_id = self.ibmcloud_power.get_instance_id(node)
+            for _ in range(instance_kill_count):
+                affected_node = AffectedNode(node, node_id=instance_id)
+                logging.info("Starting node_reboot_scenario injection")
+                logging.info("Rebooting the node %s " % (node))
+                self.ibmcloud_power.reboot_instances(instance_id, soft_reboot)
+                self.ibmcloud_power.wait_until_rebooted(instance_id, timeout, affected_node)
+                if self.node_action_kube_check:
+                    nodeaction.wait_for_unknown_status(
+                        node, timeout, affected_node
+                    )
+                    nodeaction.wait_for_ready_status(
+                        node, timeout, affected_node
+                    )
+                logging.info(
+                    "Node with instance ID: %s has rebooted successfully" % node
+                )
+                logging.info(
+                    "node_reboot_scenario has been successfully injected!"
+                )
+
+        except Exception as e:
+            logging.error("Failed to reboot node instance. Test Failed")
+            logging.error("node_reboot_scenario injection failed!")
+
+
+    def node_terminate_scenario(self, instance_kill_count, node, timeout, poll_interval):
+        try:
+            instance_id = self.ibmcloud_power.get_instance_id(node)
+            for _ in range(instance_kill_count):
+                affected_node = AffectedNode(node, node_id=instance_id)
+                logging.info(
+                    "Starting node_termination_scenario injection by first stopping the node"
+                )
+                logging.info("Deleting the node with instance ID: %s " % (node))
+                self.ibmcloud_power.delete_instance(instance_id)
+                self.ibmcloud_power.wait_until_deleted(node, timeout, affected_node)
+                logging.info(
+                    "Node with instance ID: %s has been released" % node
+                )
+                logging.info(
+                    "node_terminate_scenario has been successfully injected!"
+                )
+        except Exception as e:
+            logging.error("Failed to terminate node instance. Test Failed")
+            logging.error("node_terminate_scenario injection failed!")
+

krkn/scenario_plugins/node_actions/node_actions_scenario_plugin.py

@@ -22,8 +22,16 @@ from krkn.scenario_plugins.node_actions.gcp_node_scenarios import gcp_node_scena
 from krkn.scenario_plugins.node_actions.general_cloud_node_scenarios import (
     general_node_scenarios,
 )
-from krkn.scenario_plugins.node_actions.vmware_node_scenarios import vmware_node_scenarios
-from krkn.scenario_plugins.node_actions.ibmcloud_node_scenarios import ibm_node_scenarios
+from krkn.scenario_plugins.node_actions.vmware_node_scenarios import (
+    vmware_node_scenarios,
+)
+from krkn.scenario_plugins.node_actions.ibmcloud_node_scenarios import (
+    ibm_node_scenarios,
+)
+
+from krkn.scenario_plugins.node_actions.ibmcloud_power_node_scenarios import (
+     ibmcloud_power_node_scenarios,
+)
 node_general = False
 
 
@@ -63,29 +71,39 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
     def get_node_scenario_object(self, node_scenario, kubecli: KrknKubernetes):
         affected_nodes_status = AffectedNodeStatus()
 
-        node_action_kube_check = get_yaml_item_value(node_scenario,"kube_check",True)
+        node_action_kube_check = get_yaml_item_value(node_scenario, "kube_check", True)
         if (
             "cloud_type" not in node_scenario.keys()
             or node_scenario["cloud_type"] == "generic"
         ):
             global node_general
             node_general = True
-            return general_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status)
+            return general_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         if node_scenario["cloud_type"].lower() == "aws":
-            return aws_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status)
+            return aws_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         elif node_scenario["cloud_type"].lower() == "gcp":
-            return gcp_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status)
+            return gcp_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         elif node_scenario["cloud_type"].lower() == "openstack":
             from krkn.scenario_plugins.node_actions.openstack_node_scenarios import (
                 openstack_node_scenarios,
             )
 
-            return openstack_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status)
+            return openstack_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         elif (
             node_scenario["cloud_type"].lower() == "azure"
             or node_scenario["cloud_type"].lower() == "az"
         ):
-            return azure_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status)
+            return azure_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         elif (
             node_scenario["cloud_type"].lower() == "alibaba"
             or node_scenario["cloud_type"].lower() == "alicloud"
@@ -94,7 +112,9 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
                 alibaba_node_scenarios,
             )
 
-            return alibaba_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status)
+            return alibaba_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         elif node_scenario["cloud_type"].lower() == "bm":
             from krkn.scenario_plugins.node_actions.bm_node_scenarios import (
                 bm_node_scenarios,
@@ -106,22 +126,31 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
                 node_scenario.get("bmc_password", None),
                 kubecli,
                 node_action_kube_check,
-                affected_nodes_status
+                affected_nodes_status,
             )
         elif node_scenario["cloud_type"].lower() == "docker":
-            return docker_node_scenarios(kubecli,node_action_kube_check,
-                affected_nodes_status)
+            return docker_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         elif (
             node_scenario["cloud_type"].lower() == "vsphere"
             or node_scenario["cloud_type"].lower() == "vmware"
         ):
-            return vmware_node_scenarios(kubecli, node_action_kube_check,affected_nodes_status)
+            return vmware_node_scenarios(
+                kubecli, node_action_kube_check, affected_nodes_status
+            )
         elif (
             node_scenario["cloud_type"].lower() == "ibm"
             or node_scenario["cloud_type"].lower() == "ibmcloud"
         ):
             disable_ssl_verification = get_yaml_item_value(node_scenario, "disable_ssl_verification", True)
             return ibm_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status, disable_ssl_verification)
+        elif (
+            node_scenario["cloud_type"].lower() == "ibmpower"
+            or node_scenario["cloud_type"].lower() == "ibmcloudpower"
+        ):
+            disable_ssl_verification = get_yaml_item_value(node_scenario, "disable_ssl_verification", True)
+            return ibmcloud_power_node_scenarios(kubecli, node_action_kube_check, affected_nodes_status, disable_ssl_verification)
         else:
             logging.error(
                 "Cloud type "
@@ -139,16 +168,22 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
             )
 
     def inject_node_scenario(
-        self, action, node_scenario, node_scenario_object, kubecli: KrknKubernetes, scenario_telemetry: ScenarioTelemetry
+        self,
+        action,
+        node_scenario,
+        node_scenario_object,
+        kubecli: KrknKubernetes,
+        scenario_telemetry: ScenarioTelemetry,
     ):
-        
+
         # Get the node scenario configurations for setting nodes
-       
+
         instance_kill_count = get_yaml_item_value(node_scenario, "instance_count", 1)
         node_name = get_yaml_item_value(node_scenario, "node_name", "")
         label_selector = get_yaml_item_value(node_scenario, "label_selector", "")
+        exclude_label = get_yaml_item_value(node_scenario, "exclude_label", "")
         parallel_nodes = get_yaml_item_value(node_scenario, "parallel", False)
-        
+
         # Get the node to apply the scenario
         if node_name:
             node_name_list = node_name.split(",")
@@ -157,11 +192,22 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
             nodes = common_node_functions.get_node(
                 label_selector, instance_kill_count, kubecli
             )
-        
-        # GCP api doesn't support multiprocessing calls, will only actually run 1 
+            if exclude_label:
+                exclude_nodes = common_node_functions.get_node(
+                    exclude_label, 0, kubecli
+                )
+
+                for node in nodes:
+                    if node in exclude_nodes:
+                        logging.info(
+                            f"excluding node {node} with exclude label {exclude_nodes}"
+                        )
+                        nodes.remove(node)
+
+        # GCP api doesn't support multiprocessing calls, will only actually run 1
         if parallel_nodes:
             self.multiprocess_nodes(nodes, node_scenario_object, action, node_scenario)
-        else: 
+        else:
             for single_node in nodes:
                 self.run_node(single_node, node_scenario_object, action, node_scenario)
         affected_nodes_status = node_scenario_object.affected_nodes_status
@@ -171,21 +217,29 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
         try:
             # pool object with number of element
             pool = ThreadPool(processes=len(nodes))
-    
-            pool.starmap(self.run_node,zip(nodes, repeat(node_scenario_object), repeat(action), repeat(node_scenario)))
+
+            pool.starmap(
+                self.run_node,
+                zip(
+                    nodes,
+                    repeat(node_scenario_object),
+                    repeat(action),
+                    repeat(node_scenario),
+                ),
+            )
 
             pool.close()
         except Exception as e:
             logging.info("Error on pool multiprocessing: " + str(e))
 
-
     def run_node(self, single_node, node_scenario_object, action, node_scenario):
         # Get the scenario specifics for running action nodes
         run_kill_count = get_yaml_item_value(node_scenario, "runs", 1)
         duration = get_yaml_item_value(node_scenario, "duration", 120)
-
+        poll_interval = get_yaml_item_value(node_scenario, "poll_interval", 15)
         timeout = get_yaml_item_value(node_scenario, "timeout", 120)
         service = get_yaml_item_value(node_scenario, "service", "")
+        soft_reboot = get_yaml_item_value(node_scenario, "soft_reboot", False)
         ssh_private_key = get_yaml_item_value(
             node_scenario, "ssh_private_key", "~/.ssh/id_rsa"
         )
@@ -200,27 +254,28 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
         else:
             if action == "node_start_scenario":
                 node_scenario_object.node_start_scenario(
-                    run_kill_count, single_node, timeout
+                    run_kill_count, single_node, timeout, poll_interval
                 )
             elif action == "node_stop_scenario":
                 node_scenario_object.node_stop_scenario(
-                    run_kill_count, single_node, timeout
+                    run_kill_count, single_node, timeout, poll_interval
                 )
             elif action == "node_stop_start_scenario":
                 node_scenario_object.node_stop_start_scenario(
-                    run_kill_count, single_node, timeout, duration
+                    run_kill_count, single_node, timeout, duration, poll_interval
                 )
             elif action == "node_termination_scenario":
                 node_scenario_object.node_termination_scenario(
-                    run_kill_count, single_node, timeout
+                    run_kill_count, single_node, timeout, poll_interval
                 )
             elif action == "node_reboot_scenario":
                 node_scenario_object.node_reboot_scenario(
-                    run_kill_count, single_node, timeout
+                    run_kill_count, single_node, timeout, soft_reboot
                 )
             elif action == "node_disk_detach_attach_scenario":
                 node_scenario_object.node_disk_detach_attach_scenario(
-                    run_kill_count, single_node, timeout, duration)
+                    run_kill_count, single_node, timeout, duration
+                )
             elif action == "stop_start_kubelet_scenario":
                 node_scenario_object.stop_start_kubelet_scenario(
                     run_kill_count, single_node, timeout
@@ -248,9 +303,7 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
                 else:
                     if not node_scenario["helper_node_ip"]:
                         logging.error("Helper node IP address is not provided")
-                        raise Exception(
-                            "Helper node IP address is not provided"
-                        )
+                        raise Exception("Helper node IP address is not provided")
                     node_scenario_object.helper_node_stop_start_scenario(
                         run_kill_count, node_scenario["helper_node_ip"], timeout
                     )
@@ -270,6 +323,5 @@ class NodeActionsScenarioPlugin(AbstractScenarioPlugin):
                     % action
                 )
 
-
     def get_scenario_types(self) -> list[str]:
         return ["node_scenarios"]

krkn/scenario_plugins/node_actions/openstack_node_scenarios.py

@@ -122,7 +122,7 @@ class openstack_node_scenarios(abstract_node_scenarios):
         self.node_action_kube_check = node_action_kube_check
     
     # Node scenario to start the node
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -147,7 +147,7 @@ class openstack_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to stop the node
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:
@@ -171,7 +171,7 @@ class openstack_node_scenarios(abstract_node_scenarios):
             self.affected_nodes_status.affected_nodes.append(affected_node)
 
     # Node scenario to reboot the node
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         for _ in range(instance_kill_count):
             affected_node = AffectedNode(node)
             try:

krkn/scenario_plugins/node_actions/vmware_node_scenarios.py

@@ -73,7 +73,7 @@ class vSphere:
         vms = self.client.vcenter.VM.list(VM.FilterSpec(names=names))
 
         if len(vms) == 0:
-            logging.info("VM with name ({}) not found", instance_id)
+            logging.info("VM with name ({}) not found".format(instance_id))
             return None
         vm = vms[0].vm
 
@@ -97,7 +97,7 @@ class vSphere:
             self.client.vcenter.vm.Power.start(vm)
             self.client.vcenter.vm.Power.stop(vm)
         self.client.vcenter.VM.delete(vm)
-        logging.info("Deleted VM -- '{}-({})'", instance_id, vm)
+        logging.info("Deleted VM -- '{}-({})'".format(instance_id, vm))
 
     def reboot_instances(self, instance_id):
         """
@@ -108,11 +108,11 @@ class vSphere:
         vm = self.get_vm(instance_id)
         try:
             self.client.vcenter.vm.Power.reset(vm)
-            logging.info("Reset VM -- '{}-({})'", instance_id, vm)
+            logging.info("Reset VM -- '{}-({})'".format(instance_id, vm))
             return True
         except NotAllowedInCurrentState:
             logging.info(
-                "VM '{}'-'({})' is not Powered On. Cannot reset it", instance_id, vm
+                "VM '{}'-'({})' is not Powered On. Cannot reset it".format(instance_id, vm)
             )
             return False
 
@@ -158,7 +158,7 @@ class vSphere:
         try:
             datacenter_id = datacenter_summaries[0].datacenter
         except IndexError:
-            logging.error("Datacenter '{}' doesn't exist", datacenter)
+            logging.error("Datacenter '{}' doesn't exist".format(datacenter))
             sys.exit(1)
 
         vm_filter = self.client.vcenter.VM.FilterSpec(datacenters={datacenter_id})
@@ -389,7 +389,7 @@ class vmware_node_scenarios(abstract_node_scenarios):
         self.vsphere = vSphere()
         self.node_action_kube_check = node_action_kube_check
 
-    def node_start_scenario(self, instance_kill_count, node, timeout):
+    def node_start_scenario(self, instance_kill_count, node, timeout, poll_interval):
         try:
             for _ in range(instance_kill_count):
                 affected_node = AffectedNode(node)
@@ -409,7 +409,7 @@ class vmware_node_scenarios(abstract_node_scenarios):
                 f"node_start_scenario injection failed! " f"Error was: {str(e)}"
             )
 
-    def node_stop_scenario(self, instance_kill_count, node, timeout):
+    def node_stop_scenario(self, instance_kill_count, node, timeout, poll_interval):
         try:
             for _ in range(instance_kill_count):
                 affected_node = AffectedNode(node)
@@ -432,7 +432,7 @@ class vmware_node_scenarios(abstract_node_scenarios):
             )
                 
 
-    def node_reboot_scenario(self, instance_kill_count, node, timeout):
+    def node_reboot_scenario(self, instance_kill_count, node, timeout, soft_reboot=False):
         try:
             for _ in range(instance_kill_count):
                 affected_node = AffectedNode(node)
@@ -456,7 +456,7 @@ class vmware_node_scenarios(abstract_node_scenarios):
             )
 
 
-    def node_terminate_scenario(self, instance_kill_count, node, timeout):
+    def node_terminate_scenario(self, instance_kill_count, node, timeout, poll_interval):
         try:
             for _ in range(instance_kill_count):
                 affected_node = AffectedNode(node)

krkn/scenario_plugins/pod_disruption/models/models.py

@@ -11,6 +11,9 @@ class InputParams:
             self.label_selector = config["label_selector"] if "label_selector" in config else ""
             self.namespace_pattern = config["namespace_pattern"] if "namespace_pattern" in config else ""
             self.name_pattern = config["name_pattern"] if "name_pattern" in config else ""
+            self.node_label_selector = config["node_label_selector"] if "node_label_selector" in config else ""
+            self.node_names = config["node_names"] if "node_names" in config else []
+            self.exclude_label = config["exclude_label"] if "exclude_label" in config else ""
 
     namespace_pattern: str
     krkn_pod_recovery_time: int
@@ -18,4 +21,7 @@ class InputParams:
     duration: int
     kill: int
     label_selector: str
-    name_pattern: str
+    name_pattern: str
+    node_label_selector: str
+    node_names: list
+    exclude_label: str

krkn/scenario_plugins/pod_disruption/pod_disruption_scenario_plugin.py

@@ -2,7 +2,7 @@ import logging
 import random
 import time
 from asyncio import Future
-
+import traceback
 import yaml
 from krkn_lib.k8s import KrknKubernetes
 from krkn_lib.k8s.pod_monitor import select_and_monitor_by_namespace_pattern_and_label, \
@@ -11,6 +11,7 @@ from krkn_lib.k8s.pod_monitor import select_and_monitor_by_namespace_pattern_and
 from krkn.scenario_plugins.pod_disruption.models.models import InputParams
 from krkn_lib.models.telemetry import ScenarioTelemetry
 from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
+from krkn_lib.models.pod_monitor.models import PodsSnapshot
 from datetime import datetime
 from dataclasses import dataclass
 
@@ -40,16 +41,40 @@ class PodDisruptionScenarioPlugin(AbstractScenarioPlugin):
                         kill_scenario_config,
                         lib_telemetry
                     )
-                    self.killing_pods(
+                    ret = self.killing_pods(
                         kill_scenario_config, lib_telemetry.get_lib_kubernetes()
                     )
+                    # returning 2 if configuration issue and exiting immediately
+                    if ret > 1:
+                        # Cancel the monitoring future since killing_pods already failed
+                        logging.info("Cancelling pod monitoring future")
+                        future_snapshot.cancel()
+                        # Wait for the future to finish (monitoring will stop when stop_event is set)
+                        while not future_snapshot.done():
+                            logging.info("waiting for future to finish")
+                            time.sleep(1)
+                        logging.info("future snapshot cancelled and finished")
+                        # Get the snapshot result (even if cancelled, it will have partial data)
+                        snapshot = future_snapshot.result()
+                        result = snapshot.get_pods_status()
+                        scenario_telemetry.affected_pods = result
 
+                        logging.error("PodDisruptionScenarioPlugin failed during setup" + str(result))
+                        return 1
+                    
                     snapshot = future_snapshot.result()
                     result = snapshot.get_pods_status()
                     scenario_telemetry.affected_pods = result
+                    if len(result.unrecovered) > 0:
+                        logging.info("PodDisruptionScenarioPlugin failed with unrecovered pods")
+                        return 1
 
-
+                    if ret > 0:
+                        logging.info("PodDisruptionScenarioPlugin failed")
+                        return 1
+                    
         except (RuntimeError, Exception) as e:
+            logging.error("Stack trace:\n%s", traceback.format_exc())
             logging.error("PodDisruptionScenariosPlugin exiting due to Exception %s" % e)
             return 1
         else:
@@ -100,55 +125,131 @@ class PodDisruptionScenarioPlugin(AbstractScenarioPlugin):
             raise Exception(
                 f"impossible to determine monitor parameters, check {kill_scenario} configuration"
             )
+    
+    def _select_pods_with_field_selector(self, name_pattern, label_selector, namespace, kubecli: KrknKubernetes, field_selector: str, node_name: str = None):
+        """Helper function to select pods using either label_selector or name_pattern with field_selector, optionally filtered by node"""
+        # Combine field selectors if node targeting is specified
+        if node_name:
+            node_field_selector = f"spec.nodeName={node_name}"
+            if field_selector:
+                combined_field_selector = f"{field_selector},{node_field_selector}"
+            else:
+                combined_field_selector = node_field_selector
+        else:
+            combined_field_selector = field_selector
         
+        if label_selector:
+            return kubecli.select_pods_by_namespace_pattern_and_label(
+                label_selector=label_selector, 
+                namespace_pattern=namespace, 
+                field_selector=combined_field_selector
+            )
+        else:  # name_pattern
+            return kubecli.select_pods_by_name_pattern_and_namespace_pattern(
+                pod_name_pattern=name_pattern, 
+                namespace_pattern=namespace, 
+                field_selector=combined_field_selector
+            )
 
-    def get_pods(self, name_pattern, label_selector,namespace, kubecli: KrknKubernetes, field_selector: str =None): 
+    def get_pods(self, name_pattern, label_selector, namespace, kubecli: KrknKubernetes, field_selector: str = None, node_label_selector: str = None, node_names: list = None): 
         if label_selector and name_pattern: 
             logging.error('Only, one of name pattern or label pattern can be specified')
-        elif label_selector:
-            pods = kubecli.select_pods_by_namespace_pattern_and_label(label_selector=label_selector,namespace_pattern=namespace, field_selector=field_selector)
-        elif name_pattern:
-            pods = kubecli.select_pods_by_name_pattern_and_namespace_pattern(pod_name_pattern=name_pattern, namespace_pattern=namespace, field_selector=field_selector)
-        else: 
+            return []
+        
+        if not label_selector and not name_pattern:
             logging.error('Name pattern or label pattern must be specified ')
-        return pods
+            return []
+        
+        # If specific node names are provided, make multiple calls with field selector
+        if node_names:
+            logging.debug(f"Targeting pods on {len(node_names)} specific nodes")
+            all_pods = []
+            for node_name in node_names:
+                pods = self._select_pods_with_field_selector(
+                    name_pattern, label_selector, namespace, kubecli, field_selector, node_name
+                )
+                
+                if pods:
+                    all_pods.extend(pods)
+            
+            logging.debug(f"Found {len(all_pods)} target pods across {len(node_names)} nodes")
+            return all_pods
+        
+        #  Node label selector approach - use field selectors
+        if node_label_selector:
+            # Get nodes matching the label selector first
+            nodes_with_label = kubecli.list_nodes(label_selector=node_label_selector)
+            if not nodes_with_label:
+                logging.debug(f"No nodes found with label selector: {node_label_selector}")
+                return []
+            
+            logging.debug(f"Targeting pods on {len(nodes_with_label)} nodes with label: {node_label_selector}")
+            # Use field selector for each node
+            all_pods = []
+            for node_name in nodes_with_label:
+                pods = self._select_pods_with_field_selector(
+                    name_pattern, label_selector, namespace, kubecli, field_selector, node_name
+                )
+                
+                if pods:
+                    all_pods.extend(pods)
+            
+            logging.debug(f"Found {len(all_pods)} target pods across {len(nodes_with_label)} nodes")
+            return all_pods
+        
+        # Standard pod selection (no node targeting)
+        return self._select_pods_with_field_selector(
+            name_pattern, label_selector, namespace, kubecli, field_selector
+        )
     
     def killing_pods(self, config: InputParams, kubecli: KrknKubernetes):
         # region Select target pods
+        try:
+            namespace = config.namespace_pattern
+            if not namespace: 
+                logging.error('Namespace pattern must be specified')
+
+            pods = self.get_pods(config.name_pattern,config.label_selector,config.namespace_pattern, kubecli, field_selector="status.phase=Running", node_label_selector=config.node_label_selector, node_names=config.node_names)
+            exclude_pods = set()
+            if config.exclude_label:
+                _exclude_pods = self.get_pods("",config.exclude_label,config.namespace_pattern, kubecli, field_selector="status.phase=Running", node_label_selector=config.node_label_selector, node_names=config.node_names)
+                for pod in _exclude_pods:
+                    exclude_pods.add(pod[0])
+
+
+            pods_count = len(pods)
+            if len(pods) < config.kill:
+                logging.error("Not enough pods match the criteria, expected {} but found only {} pods".format(
+                        config.kill, len(pods)))
+                return 1
             
-        namespace = config.namespace_pattern
-        if not namespace: 
-            logging.error('Namespace pattern must be specified')
+            random.shuffle(pods)
+            for i in range(config.kill):
+                pod = pods[i]
+                logging.info(pod)
+                if pod[0] in exclude_pods:
+                    logging.info(f"Excluding {pod[0]} from chaos")
+                else:
+                    logging.info(f'Deleting pod {pod[0]}')
+                    kubecli.delete_pod(pod[0], pod[1])
+            
+            return_val = self.wait_for_pods(config.label_selector,config.name_pattern,config.namespace_pattern, pods_count, config.duration, config.timeout, kubecli, config.node_label_selector, config.node_names)
+        except Exception as e:
+            raise(e)
 
-        pods = self.get_pods(config.name_pattern,config.label_selector,config.namespace_pattern, kubecli, field_selector="status.phase=Running")
-        pods_count = len(pods)
-        if len(pods) < config.kill:
-            logging.error("Not enough pods match the criteria, expected {} but found only {} pods".format(
-                    config.kill, len(pods)))
-            return 1
-        
-        random.shuffle(pods)
-        for i in range(config.kill):
-
-            pod = pods[i]
-            logging.info(pod)
-            logging.info(f'Deleting pod {pod[0]}')
-            kubecli.delete_pod(pod[0], pod[1])
-        
-        self.wait_for_pods(config.label_selector,config.name_pattern,config.namespace_pattern, pods_count, config.duration, config.timeout, kubecli)
-        return 0
+        return return_val
 
     def wait_for_pods(
-        self, label_selector, pod_name, namespace, pod_count, duration, wait_timeout, kubecli: KrknKubernetes
+        self, label_selector, pod_name, namespace, pod_count, duration, wait_timeout, kubecli: KrknKubernetes, node_label_selector, node_names
     ):
         timeout = False
         start_time = datetime.now()
 
         while not timeout:
-            pods = self.get_pods(name_pattern=pod_name, label_selector=label_selector,namespace=namespace, field_selector="status.phase=Running", kubecli=kubecli)
+            pods = self.get_pods(name_pattern=pod_name, label_selector=label_selector,namespace=namespace, field_selector="status.phase=Running", kubecli=kubecli, node_label_selector=node_label_selector, node_names=node_names)
             if pod_count == len(pods):
-                return
-               
+                return 0
+            
             time.sleep(duration)
 
             now_time = datetime.now()
@@ -157,4 +258,5 @@ class PodDisruptionScenarioPlugin(AbstractScenarioPlugin):
             if time_diff.seconds > wait_timeout:
                 logging.error("timeout while waiting for pods to come up")
                 return 1
+
         return 0

krkn/scenario_plugins/pvc/pvc_scenario_plugin.py

@@ -1,3 +1,5 @@
+import base64
+import json
 import logging
 import random
 import re
@@ -11,9 +13,12 @@ from krkn_lib.utils import get_yaml_item_value, log_exception
 
 from krkn import cerberus, utils
 from krkn.scenario_plugins.abstract_scenario_plugin import AbstractScenarioPlugin
+from krkn.rollback.config import RollbackContent
+from krkn.rollback.handler import set_rollback_context_decorator
 
 
 class PvcScenarioPlugin(AbstractScenarioPlugin):
+    @set_rollback_context_decorator
     def run(
         self,
         run_uuid: str,
@@ -229,6 +234,24 @@ class PvcScenarioPlugin(AbstractScenarioPlugin):
                 logging.info("\n" + str(response))
                 if str(file_name).lower() in str(response).lower():
                     logging.info("%s file successfully created" % (str(full_path)))
+                    
+                    # Set rollback callable to ensure temp file cleanup on failure or interruption
+                    rollback_data = {
+                        "pod_name": pod_name,
+                        "container_name": container_name,
+                        "mount_path": mount_path,
+                        "file_name": file_name,
+                        "full_path": full_path,
+                    }
+                    json_str = json.dumps(rollback_data)
+                    encoded_data = base64.b64encode(json_str.encode('utf-8')).decode('utf-8')
+                    self.rollback_handler.set_rollback_callable(
+                        self.rollback_temp_file,
+                        RollbackContent(
+                            namespace=namespace,
+                            resource_identifier=encoded_data,
+                        ),
+                    )
                 else:
                     logging.error(
                         "PvcScenarioPlugin Failed to create tmp file with %s size"
@@ -313,5 +336,57 @@ class PvcScenarioPlugin(AbstractScenarioPlugin):
         res = int(value[:-2]) * (base**exp)
         return res
 
+    @staticmethod
+    def rollback_temp_file(
+        rollback_content: RollbackContent,
+        lib_telemetry: KrknTelemetryOpenshift,
+    ):
+        """Rollback function to remove temporary file created during the PVC scenario.
+
+        :param rollback_content: Rollback content containing namespace and encoded rollback data in resource_identifier.
+        :param lib_telemetry: Instance of KrknTelemetryOpenshift for Kubernetes operations.
+        """
+        try:
+            namespace = rollback_content.namespace
+            import base64 # noqa
+            import json # noqa
+            decoded_data = base64.b64decode(rollback_content.resource_identifier.encode('utf-8')).decode('utf-8')
+            rollback_data = json.loads(decoded_data)
+            pod_name = rollback_data["pod_name"]
+            container_name = rollback_data["container_name"]
+            full_path = rollback_data["full_path"]
+            file_name = rollback_data["file_name"]
+            mount_path = rollback_data["mount_path"]
+            
+            logging.info(
+                f"Rolling back PVC scenario: removing temp file {full_path} from pod {pod_name} in namespace {namespace}"
+            )
+            
+            # Remove the temp file
+            command = "rm -f %s" % (str(full_path))
+            logging.info("Remove temp file from the PVC command:\n %s" % command)
+            response = lib_telemetry.get_lib_kubernetes().exec_cmd_in_pod(
+                [command], pod_name, namespace, container_name
+            )
+            logging.info("\n" + str(response))
+            # Verify removal
+            command = "ls -lh %s" % (str(mount_path))
+            logging.info("Check temp file is removed command:\n %s" % command)
+            response = lib_telemetry.get_lib_kubernetes().exec_cmd_in_pod(
+                [command], pod_name, namespace, container_name
+            )
+            logging.info("\n" + str(response))
+            
+            if not (str(file_name).lower() in str(response).lower()):
+                logging.info("Temp file successfully removed during rollback")
+            else:
+                logging.warning(
+                    f"Temp file {file_name} may still exist after rollback attempt"
+                )
+            
+            logging.info("PVC scenario rollback completed successfully.")
+        except Exception as e:
+            logging.error(f"Failed to rollback PVC scenario temp file: {e}")
+
     def get_scenario_types(self) -> list[str]:
         return ["pvc_scenarios"]

krkn/scenario_plugins/service_disruption/service_disruption_scenario_plugin.py

@@ -209,7 +209,7 @@ class ServiceDisruptionScenarioPlugin(AbstractScenarioPlugin):
         try:
             statefulsets = kubecli.get_all_statefulset(namespace)
             for statefulset in statefulsets:
-                logging.info("Deleting statefulsets" + statefulsets)
+                logging.info("Deleting statefulset" + statefulset)
                 kubecli.delete_statefulset(statefulset, namespace)
         except Exception as e:
             logging.error(

krkn/scenario_plugins/service_hijacking/service_hijacking_scenario_plugin.py

@@ -1,13 +1,17 @@
+import json
 import logging
 import time
-
+import base64
 import yaml
 from krkn_lib.models.telemetry import ScenarioTelemetry
 from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
 from krkn.scenario_plugins.abstract_scenario_plugin import AbstractScenarioPlugin
 from krkn_lib.utils import get_yaml_item_value
+from krkn.rollback.config import RollbackContent
+from krkn.rollback.handler import set_rollback_context_decorator
 
 class ServiceHijackingScenarioPlugin(AbstractScenarioPlugin):
+    @set_rollback_context_decorator
     def run(
         self,
         run_uuid: str,
@@ -78,6 +82,24 @@ class ServiceHijackingScenarioPlugin(AbstractScenarioPlugin):
 
             logging.info(f"service: {service_name} successfully patched!")
             logging.info(f"original service manifest:\n\n{yaml.dump(original_service)}")
+            
+            # Set rollback callable to ensure service restoration and pod cleanup on failure or interruption
+            rollback_data = {
+                "service_name": service_name,
+                "service_namespace": service_namespace,
+                "original_selectors": original_service["spec"]["selector"],
+                "webservice_pod_name": webservice.pod_name,
+            }
+            json_str = json.dumps(rollback_data)
+            encoded_data = base64.b64encode(json_str.encode("utf-8")).decode("utf-8")
+            self.rollback_handler.set_rollback_callable(
+                self.rollback_service_hijacking,
+                RollbackContent(
+                    namespace=service_namespace,
+                    resource_identifier=encoded_data,
+                ),
+            )
+            
             logging.info(f"waiting {chaos_duration} before restoring the service")
             time.sleep(chaos_duration)
             selectors = [
@@ -106,5 +128,63 @@ class ServiceHijackingScenarioPlugin(AbstractScenarioPlugin):
             )
             return 1
 
+    @staticmethod
+    def rollback_service_hijacking(
+        rollback_content: RollbackContent,
+        lib_telemetry: KrknTelemetryOpenshift,
+    ):
+        """Rollback function to restore original service selectors and cleanup hijacker pod.
+
+        :param rollback_content: Rollback content containing namespace and encoded rollback data in resource_identifier.
+        :param lib_telemetry: Instance of KrknTelemetryOpenshift for Kubernetes operations.
+        """
+        try:
+            namespace = rollback_content.namespace
+            import json # noqa
+            import base64 # noqa
+            # Decode rollback data from resource_identifier
+            decoded_data = base64.b64decode(rollback_content.resource_identifier.encode("utf-8")).decode("utf-8")
+            rollback_data = json.loads(decoded_data)
+            service_name = rollback_data["service_name"]
+            service_namespace = rollback_data["service_namespace"]
+            original_selectors = rollback_data["original_selectors"]
+            webservice_pod_name = rollback_data["webservice_pod_name"]
+            
+            logging.info(
+                f"Rolling back service hijacking: restoring service {service_name} in namespace {service_namespace}"
+            )
+            
+            # Restore original service selectors
+            selectors = [
+                "=".join([key, original_selectors[key]])
+                for key in original_selectors.keys()
+            ]
+            logging.info(f"Restoring original service selectors: {selectors}")
+            
+            restored_service = lib_telemetry.get_lib_kubernetes().replace_service_selector(
+                selectors, service_name, service_namespace
+            )
+            
+            if restored_service is None:
+                logging.warning(
+                    f"Failed to restore service {service_name} in namespace {service_namespace}"
+                )
+            else:
+                logging.info(f"Successfully restored service {service_name}")
+            
+            # Delete the hijacker pod
+            logging.info(f"Deleting hijacker pod: {webservice_pod_name}")
+            try:
+                lib_telemetry.get_lib_kubernetes().delete_pod(
+                    webservice_pod_name, service_namespace
+                )
+                logging.info(f"Successfully deleted hijacker pod: {webservice_pod_name}")
+            except Exception as e:
+                logging.warning(f"Failed to delete hijacker pod {webservice_pod_name}: {e}")
+            
+            logging.info("Service hijacking rollback completed successfully.")
+        except Exception as e:
+            logging.error(f"Failed to rollback service hijacking: {e}")
+
     def get_scenario_types(self) -> list[str]:
         return ["service_hijacking_scenarios"]

krkn/scenario_plugins/syn_flood/syn_flood_scenario_plugin.py

@@ -1,3 +1,5 @@
+import base64
+import json
 import logging
 import os
 import time
@@ -7,9 +9,12 @@ from krkn_lib import utils as krkn_lib_utils
 from krkn_lib.models.telemetry import ScenarioTelemetry
 from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
 from krkn.scenario_plugins.abstract_scenario_plugin import AbstractScenarioPlugin
+from krkn.rollback.config import RollbackContent
+from krkn.rollback.handler import set_rollback_context_decorator
 
 
 class SynFloodScenarioPlugin(AbstractScenarioPlugin):
+    @set_rollback_context_decorator
     def run(
         self,
         run_uuid: str,
@@ -50,6 +55,16 @@ class SynFloodScenarioPlugin(AbstractScenarioPlugin):
                         config["attacker-nodes"],
                     )
                     pod_names.append(pod_name)
+                
+                # Set rollback callable to ensure pod cleanup on failure or interruption
+                rollback_data = base64.b64encode(json.dumps(pod_names).encode('utf-8')).decode('utf-8')
+                self.rollback_handler.set_rollback_callable(
+                    self.rollback_syn_flood_pods,
+                    RollbackContent(
+                        namespace=config["namespace"],
+                        resource_identifier=rollback_data,
+                    ),
+                )
 
             logging.info("waiting all the attackers to finish:")
             did_finish = False
@@ -137,3 +152,23 @@ class SynFloodScenarioPlugin(AbstractScenarioPlugin):
 
     def get_scenario_types(self) -> list[str]:
         return ["syn_flood_scenarios"]
+
+    @staticmethod
+    def rollback_syn_flood_pods(rollback_content: RollbackContent, lib_telemetry: KrknTelemetryOpenshift):
+        """
+        Rollback function to delete syn flood pods.
+
+        :param rollback_content: Rollback content containing namespace and resource_identifier.
+        :param lib_telemetry: Instance of KrknTelemetryOpenshift for Kubernetes operations
+        """
+        try:
+            namespace = rollback_content.namespace
+            import base64 # noqa
+            import json # noqa
+            pod_names = json.loads(base64.b64decode(rollback_content.resource_identifier.encode('utf-8')).decode('utf-8'))
+            logging.info(f"Rolling back syn flood pods: {pod_names} in namespace: {namespace}")
+            for pod_name in pod_names:
+                lib_telemetry.get_lib_kubernetes().delete_pod(pod_name, namespace)
+            logging.info("Rollback of syn flood pods completed successfully.")
+        except Exception as e:
+            logging.error(f"Failed to rollback syn flood pods: {e}")

krkn/scenario_plugins/time_actions/time_actions_scenario_plugin.py

@@ -144,6 +144,10 @@ class TimeActionsScenarioPlugin(AbstractScenarioPlugin):
                 node_names = scenario["object_name"]
             elif "label_selector" in scenario.keys() and scenario["label_selector"]:
                 node_names = kubecli.list_nodes(scenario["label_selector"])
+                # going to filter out nodes with the exclude_label if it is provided
+                if "exclude_label" in scenario.keys() and scenario["exclude_label"]:
+                    excluded_nodes = kubecli.list_nodes(scenario["exclude_label"])
+                    node_names = [node for node in node_names if node not in excluded_nodes]
             for node in node_names:
                 self.skew_node(node, scenario["action"], kubecli)
                 logging.info("Reset date/time on node " + str(node))
@@ -189,6 +193,10 @@ class TimeActionsScenarioPlugin(AbstractScenarioPlugin):
                     counter += 1
             elif "label_selector" in scenario.keys() and scenario["label_selector"]:
                 pod_names = kubecli.get_all_pods(scenario["label_selector"])
+                # and here filter out the pods with exclude_label if it is provided
+                if "exclude_label" in scenario.keys() and scenario["exclude_label"]:
+                    excluded_pods = kubecli.get_all_pods(scenario["exclude_label"])
+                    pod_names = [pod for pod in pod_names if pod not in excluded_pods]
 
             if len(pod_names) == 0:
                 logging.info(

krkn/tests/test_plugin_factory.py

@@ -1,10 +1,17 @@
+import json
+import tempfile
 import unittest
+from pathlib import Path
+from unittest.mock import Mock, patch
 
 from krkn.scenario_plugins.abstract_scenario_plugin import AbstractScenarioPlugin
 from krkn.scenario_plugins.scenario_plugin_factory import ScenarioPluginFactory
+from krkn.scenario_plugins.native.plugins import PluginStep, Plugins, PLUGINS
 from krkn.tests.test_classes.correct_scenario_plugin import (
     CorrectScenarioPlugin,
 )
+import yaml
+
 
 
 class TestPluginFactory(unittest.TestCase):
@@ -108,3 +115,437 @@ class TestPluginFactory(unittest.TestCase):
         self.assertEqual(
             message, "scenario plugin folder cannot contain `scenario` or `plugin` word"
         )
+
+
+class TestPluginStep(unittest.TestCase):
+    """Test cases for PluginStep class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Create a mock schema
+        self.mock_schema = Mock()
+        self.mock_schema.id = "test_step"
+
+        # Create mock output
+        mock_output = Mock()
+        mock_output.serialize = Mock(return_value={"status": "success", "message": "test"})
+        self.mock_schema.outputs = {
+            "success": mock_output,
+            "error": mock_output
+        }
+
+        self.plugin_step = PluginStep(
+            schema=self.mock_schema,
+            error_output_ids=["error"]
+        )
+
+    def test_render_output(self):
+        """Test render_output method"""
+        output_id = "success"
+        output_data = {"status": "success", "message": "test output"}
+
+        result = self.plugin_step.render_output(output_id, output_data)
+
+        # Verify it returns a JSON string
+        self.assertIsInstance(result, str)
+
+        # Verify it can be parsed as JSON
+        parsed = json.loads(result)
+        self.assertEqual(parsed["output_id"], output_id)
+        self.assertIn("output_data", parsed)
+
+
+class TestPlugins(unittest.TestCase):
+    """Test cases for Plugins class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Create mock steps with proper id attribute
+        self.mock_step1 = Mock()
+        self.mock_step1.id = "step1"
+
+        self.mock_step2 = Mock()
+        self.mock_step2.id = "step2"
+
+        self.plugin_step1 = PluginStep(schema=self.mock_step1, error_output_ids=["error"])
+        self.plugin_step2 = PluginStep(schema=self.mock_step2, error_output_ids=["error"])
+
+    def test_init_with_valid_steps(self):
+        """Test Plugins initialization with valid steps"""
+        plugins = Plugins([self.plugin_step1, self.plugin_step2])
+
+        self.assertEqual(len(plugins.steps_by_id), 2)
+        self.assertIn("step1", plugins.steps_by_id)
+        self.assertIn("step2", plugins.steps_by_id)
+
+    def test_init_with_duplicate_step_ids(self):
+        """Test Plugins initialization with duplicate step IDs raises exception"""
+        # Create two steps with the same ID
+        duplicate_step = PluginStep(schema=self.mock_step1, error_output_ids=["error"])
+
+        with self.assertRaises(Exception) as context:
+            Plugins([self.plugin_step1, duplicate_step])
+
+        self.assertIn("Duplicate step ID", str(context.exception))
+
+    def test_unserialize_scenario(self):
+        """Test unserialize_scenario method"""
+        # Create a temporary YAML file
+        test_data = [
+            {"id": "test_step", "config": {"param": "value"}}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([self.plugin_step1])
+            result = plugins.unserialize_scenario(temp_file)
+
+            self.assertIsInstance(result, list)
+        finally:
+            Path(temp_file).unlink()
+
+    def test_run_with_invalid_scenario_not_list(self):
+        """Test run method with scenario that is not a list"""
+        # Create a temporary YAML file with dict instead of list
+        test_data = {"id": "test_step", "config": {"param": "value"}}
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([self.plugin_step1])
+
+            with self.assertRaises(Exception) as context:
+                plugins.run(temp_file, "/path/to/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            self.assertIn("expected list", str(context.exception))
+        finally:
+            Path(temp_file).unlink()
+
+    def test_run_with_invalid_entry_not_dict(self):
+        """Test run method with entry that is not a dict"""
+        # Create a temporary YAML file with list of strings instead of dicts
+        test_data = ["invalid", "entries"]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([self.plugin_step1])
+
+            with self.assertRaises(Exception) as context:
+                plugins.run(temp_file, "/path/to/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            self.assertIn("expected a list of dict's", str(context.exception))
+        finally:
+            Path(temp_file).unlink()
+
+    def test_run_with_missing_id_field(self):
+        """Test run method with missing 'id' field"""
+        # Create a temporary YAML file with missing id
+        test_data = [
+            {"config": {"param": "value"}}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([self.plugin_step1])
+
+            with self.assertRaises(Exception) as context:
+                plugins.run(temp_file, "/path/to/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            self.assertIn("missing 'id' field", str(context.exception))
+        finally:
+            Path(temp_file).unlink()
+
+    def test_run_with_missing_config_field(self):
+        """Test run method with missing 'config' field"""
+        # Create a temporary YAML file with missing config
+        test_data = [
+            {"id": "step1"}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([self.plugin_step1])
+
+            with self.assertRaises(Exception) as context:
+                plugins.run(temp_file, "/path/to/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            self.assertIn("missing 'config' field", str(context.exception))
+        finally:
+            Path(temp_file).unlink()
+
+    def test_run_with_invalid_step_id(self):
+        """Test run method with invalid step ID"""
+        # Create a proper mock schema with string ID
+        mock_schema = Mock()
+        mock_schema.id = "valid_step"
+        plugin_step = PluginStep(schema=mock_schema, error_output_ids=["error"])
+
+        # Create a temporary YAML file with unknown step ID
+        test_data = [
+            {"id": "unknown_step", "config": {"param": "value"}}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([plugin_step])
+
+            with self.assertRaises(Exception) as context:
+                plugins.run(temp_file, "/path/to/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            self.assertIn("Invalid step", str(context.exception))
+            self.assertIn("expected one of", str(context.exception))
+        finally:
+            Path(temp_file).unlink()
+
+    @patch('krkn.scenario_plugins.native.plugins.logging')
+    def test_run_with_valid_scenario(self, mock_logging):
+        """Test run method with valid scenario"""
+        # Create mock schema with all necessary attributes
+        mock_schema = Mock()
+        mock_schema.id = "test_step"
+
+        # Mock input schema
+        mock_input = Mock()
+        mock_input.properties = {}
+        mock_input.unserialize = Mock(return_value=Mock(spec=[]))
+        mock_schema.input = mock_input
+
+        # Mock output
+        mock_output = Mock()
+        mock_output.serialize = Mock(return_value={"status": "success"})
+        mock_schema.outputs = {"success": mock_output}
+
+        # Mock schema call
+        mock_schema.return_value = ("success", {"status": "success"})
+
+        plugin_step = PluginStep(schema=mock_schema, error_output_ids=["error"])
+
+        # Create a temporary YAML file
+        test_data = [
+            {"id": "test_step", "config": {"param": "value"}}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([plugin_step])
+            plugins.run(temp_file, "/path/to/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            # Verify schema was called
+            mock_schema.assert_called_once()
+        finally:
+            Path(temp_file).unlink()
+
+    @patch('krkn.scenario_plugins.native.plugins.logging')
+    def test_run_with_error_output(self, mock_logging):
+        """Test run method when step returns error output"""
+        # Create mock schema with error output
+        mock_schema = Mock()
+        mock_schema.id = "test_step"
+
+        # Mock input schema
+        mock_input = Mock()
+        mock_input.properties = {}
+        mock_input.unserialize = Mock(return_value=Mock(spec=[]))
+        mock_schema.input = mock_input
+
+        # Mock output
+        mock_output = Mock()
+        mock_output.serialize = Mock(return_value={"error": "test error"})
+        mock_schema.outputs = {"error": mock_output}
+
+        # Mock schema call to return error
+        mock_schema.return_value = ("error", {"error": "test error"})
+
+        plugin_step = PluginStep(schema=mock_schema, error_output_ids=["error"])
+
+        # Create a temporary YAML file
+        test_data = [
+            {"id": "test_step", "config": {"param": "value"}}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([plugin_step])
+
+            with self.assertRaises(Exception) as context:
+                plugins.run(temp_file, "/path/to/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            self.assertIn("failed", str(context.exception))
+        finally:
+            Path(temp_file).unlink()
+
+    @patch('krkn.scenario_plugins.native.plugins.logging')
+    def test_run_with_kubeconfig_path_injection(self, mock_logging):
+        """Test run method injects kubeconfig_path when property exists"""
+        # Create mock schema with kubeconfig_path in input properties
+        mock_schema = Mock()
+        mock_schema.id = "test_step"
+
+        # Mock input schema with kubeconfig_path property
+        mock_input_instance = Mock()
+        mock_input = Mock()
+        mock_input.properties = {"kubeconfig_path": Mock()}
+        mock_input.unserialize = Mock(return_value=mock_input_instance)
+        mock_schema.input = mock_input
+
+        # Mock output
+        mock_output = Mock()
+        mock_output.serialize = Mock(return_value={"status": "success"})
+        mock_schema.outputs = {"success": mock_output}
+
+        # Mock schema call
+        mock_schema.return_value = ("success", {"status": "success"})
+
+        plugin_step = PluginStep(schema=mock_schema, error_output_ids=["error"])
+
+        # Create a temporary YAML file
+        test_data = [
+            {"id": "test_step", "config": {"param": "value"}}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([plugin_step])
+            plugins.run(temp_file, "/custom/kubeconfig", "/path/to/kraken_config", "test-uuid")
+
+            # Verify kubeconfig_path was set
+            self.assertEqual(mock_input_instance.kubeconfig_path, "/custom/kubeconfig")
+        finally:
+            Path(temp_file).unlink()
+
+    @patch('krkn.scenario_plugins.native.plugins.logging')
+    def test_run_with_kraken_config_injection(self, mock_logging):
+        """Test run method injects kraken_config when property exists"""
+        # Create mock schema with kraken_config in input properties
+        mock_schema = Mock()
+        mock_schema.id = "test_step"
+
+        # Mock input schema with kraken_config property
+        mock_input_instance = Mock()
+        mock_input = Mock()
+        mock_input.properties = {"kraken_config": Mock()}
+        mock_input.unserialize = Mock(return_value=mock_input_instance)
+        mock_schema.input = mock_input
+
+        # Mock output
+        mock_output = Mock()
+        mock_output.serialize = Mock(return_value={"status": "success"})
+        mock_schema.outputs = {"success": mock_output}
+
+        # Mock schema call
+        mock_schema.return_value = ("success", {"status": "success"})
+
+        plugin_step = PluginStep(schema=mock_schema, error_output_ids=["error"])
+
+        # Create a temporary YAML file
+        test_data = [
+            {"id": "test_step", "config": {"param": "value"}}
+        ]
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+            yaml.dump(test_data, f)
+            temp_file = f.name
+
+        try:
+            plugins = Plugins([plugin_step])
+            plugins.run(temp_file, "/path/to/kubeconfig", "/custom/kraken.yaml", "test-uuid")
+
+            # Verify kraken_config was set
+            self.assertEqual(mock_input_instance.kraken_config, "/custom/kraken.yaml")
+        finally:
+            Path(temp_file).unlink()
+
+    def test_json_schema(self):
+        """Test json_schema method"""
+        # Create mock schema with jsonschema support
+        mock_schema = Mock()
+        mock_schema.id = "test_step"
+
+        plugin_step = PluginStep(schema=mock_schema, error_output_ids=["error"])
+
+        with patch('krkn.scenario_plugins.native.plugins.jsonschema') as mock_jsonschema:
+            # Mock the step_input function
+            mock_jsonschema.step_input.return_value = {
+                "$id": "http://example.com",
+                "$schema": "http://json-schema.org/draft-07/schema#",
+                "title": "Test Schema",
+                "description": "Test description",
+                "type": "object",
+                "properties": {"param": {"type": "string"}}
+            }
+
+            plugins = Plugins([plugin_step])
+            result = plugins.json_schema()
+
+            # Verify it returns a JSON string
+            self.assertIsInstance(result, str)
+
+            # Parse and verify structure
+            parsed = json.loads(result)
+            self.assertEqual(parsed["$id"], "https://github.com/redhat-chaos/krkn/")
+            self.assertEqual(parsed["type"], "array")
+            self.assertEqual(parsed["minContains"], 1)
+            self.assertIn("items", parsed)
+            self.assertIn("oneOf", parsed["items"])
+
+            # Verify step is included
+            self.assertEqual(len(parsed["items"]["oneOf"]), 1)
+            step_schema = parsed["items"]["oneOf"][0]
+            self.assertEqual(step_schema["properties"]["id"]["const"], "test_step")
+
+
+class TestPLUGINSConstant(unittest.TestCase):
+    """Test cases for the PLUGINS constant"""
+
+    def test_plugins_initialized(self):
+        """Test that PLUGINS constant is properly initialized"""
+        self.assertIsInstance(PLUGINS, Plugins)
+
+        # Verify all expected steps are registered
+        expected_steps = [
+            "run_python",
+            "network_chaos",
+            "pod_network_outage",
+            "pod_egress_shaping",
+            "pod_ingress_shaping"
+        ]
+
+        for step_id in expected_steps:
+            self.assertIn(step_id, PLUGINS.steps_by_id)
+
+        # Ensure the registered id matches the decorator and no legacy alias is present
+        self.assertEqual(
+            PLUGINS.steps_by_id["pod_network_outage"].schema.id,
+            "pod_network_outage",
+        )
+        self.assertNotIn("pod_outage", PLUGINS.steps_by_id)
+
+    def test_plugins_step_count(self):
+        """Test that PLUGINS has the expected number of steps"""
+        self.assertEqual(len(PLUGINS.steps_by_id), 5)

krkn/utils/VirtChecker.py

@@ -1,6 +1,7 @@
 
 import time
 import logging
+import math
 import queue
 from datetime import datetime
 from krkn_lib.models.telemetry.models import VirtCheck
@@ -14,41 +15,90 @@ from krkn_lib.utils.functions import get_yaml_item_value
 class VirtChecker:
     current_iterations: int = 0
     ret_value = 0
-    def __init__(self, kubevirt_check_config, iterations, krkn_lib: KrknKubernetes, threads_limt=20):
+    def __init__(self, kubevirt_check_config, iterations, krkn_lib: KrknKubernetes, threads_limit=20):
         self.iterations = iterations
         self.namespace = get_yaml_item_value(kubevirt_check_config, "namespace", "")
         self.vm_list = []
         self.threads = []
-        self.threads_limit = threads_limt
-        if self.namespace == "":
-            logging.info("kube virt checks config is not defined, skipping them")
-            return
+        self.iteration_lock = threading.Lock()  # Lock to protect current_iterations
+        self.threads_limit = threads_limit
+        # setting to 0 in case no variables are set, so no threads later get made
+        self.batch_size = 0
+        self.ret_value = 0
         vmi_name_match = get_yaml_item_value(kubevirt_check_config, "name", ".*")
         self.krkn_lib = krkn_lib
         self.disconnected =  get_yaml_item_value(kubevirt_check_config, "disconnected", False)
         self.only_failures =  get_yaml_item_value(kubevirt_check_config, "only_failures", False)
         self.interval = get_yaml_item_value(kubevirt_check_config, "interval", 2)
+        self.ssh_node = get_yaml_item_value(kubevirt_check_config, "ssh_node", "")
+        self.node_names = get_yaml_item_value(kubevirt_check_config, "node_names", "")
+        self.exit_on_failure = get_yaml_item_value(kubevirt_check_config, "exit_on_failure", False)
+        if self.namespace == "":
+            logging.info("kube virt checks config is not defined, skipping them")
+            return
         try:
             self.kube_vm_plugin = KubevirtVmOutageScenarioPlugin()
             self.kube_vm_plugin.init_clients(k8s_client=krkn_lib)
-            vmis = self.kube_vm_plugin.get_vmis(vmi_name_match,self.namespace)
+
+            self.kube_vm_plugin.get_vmis(vmi_name_match,self.namespace)
         except Exception as e:
             logging.error('Virt Check init exception: ' + str(e))
-            return 
-        
-        for vmi in vmis:
+            return
+        # See if multiple node names exist
+        node_name_list = [node_name for node_name in self.node_names.split(',') if node_name]
+        for vmi in self.kube_vm_plugin.vmis_list:
             node_name = vmi.get("status",{}).get("nodeName")
             vmi_name = vmi.get("metadata",{}).get("name")
             ip_address = vmi.get("status",{}).get("interfaces",[])[0].get("ipAddress")
-            self.vm_list.append(VirtCheck({'vm_name':vmi_name, 'ip_address': ip_address, 'namespace':self.namespace, 'node_name':node_name}))
+            namespace = vmi.get("metadata",{}).get("namespace")
+            # If node_name_list exists, only add if node name is in list
 
-    def check_disconnected_access(self, ip_address: str, worker_name:str = ''):
+            if len(node_name_list) > 0 and node_name in node_name_list:
+                self.vm_list.append(VirtCheck({'vm_name':vmi_name, 'ip_address': ip_address, 'namespace':namespace, 'node_name':node_name, "new_ip_address":""}))
+            elif len(node_name_list) == 0:
+                # If node_name_list is blank, add all vms
+                self.vm_list.append(VirtCheck({'vm_name':vmi_name, 'ip_address': ip_address, 'namespace':namespace, 'node_name':node_name, "new_ip_address":""}))
+        self.batch_size = math.ceil(len(self.vm_list)/self.threads_limit)
 
-        virtctl_vm_cmd = f"ssh core@{worker_name} 'ssh -o BatchMode=yes -o ConnectTimeout=2 -o StrictHostKeyChecking=no root@{ip_address} 2>&1 | grep Permission' && echo 'True' || echo 'False'"
-        if 'True' in invoke_no_exit(virtctl_vm_cmd):
-            return True
+    def check_disconnected_access(self, ip_address: str, worker_name:str = '', vmi_name: str = ''):
+        
+        virtctl_vm_cmd = f"ssh core@{worker_name} -o ConnectTimeout=5 'ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@{ip_address}'"
+        
+        all_out = invoke_no_exit(virtctl_vm_cmd)
+        logging.debug(f"Checking disconnected access for {ip_address} on {worker_name} output: {all_out}")
+        virtctl_vm_cmd = f"ssh core@{worker_name} -o ConnectTimeout=5 'ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@{ip_address} 2>&1 | grep Permission' && echo 'True' || echo 'False'"
+        output = invoke_no_exit(virtctl_vm_cmd)
+        if 'True' in output:
+            logging.debug(f"Disconnected access for {ip_address} on {worker_name} is successful: {output}")
+            return True, None, None
         else:
-            return False
+            logging.debug(f"Disconnected access for {ip_address} on {worker_name} is failed: {output}")
+            vmi = self.kube_vm_plugin.get_vmi(vmi_name,self.namespace)
+            new_ip_address = vmi.get("status",{}).get("interfaces",[])[0].get("ipAddress")
+            new_node_name = vmi.get("status",{}).get("nodeName")
+            # if vm gets deleted, it'll start up with a new ip address
+            if new_ip_address != ip_address:
+                virtctl_vm_cmd = f"ssh core@{worker_name} -o ConnectTimeout=5 'ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@{new_ip_address} 2>&1 | grep Permission' && echo 'True' || echo 'False'"
+                new_output = invoke_no_exit(virtctl_vm_cmd)
+                logging.debug(f"Disconnected access for {ip_address} on {worker_name}: {new_output}")
+                if 'True' in new_output:
+                    return True, new_ip_address, None
+            # if node gets stopped, vmis will start up with a new node (and with new ip)
+            if new_node_name != worker_name:
+                virtctl_vm_cmd = f"ssh core@{new_node_name} -o ConnectTimeout=5 'ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@{new_ip_address} 2>&1 | grep Permission' && echo 'True' || echo 'False'"
+                new_output = invoke_no_exit(virtctl_vm_cmd)
+                logging.debug(f"Disconnected access for {ip_address} on {new_node_name}: {new_output}")
+                if 'True' in new_output:
+                    return True, new_ip_address, new_node_name
+            # try to connect with a common "up" node as last resort
+            if self.ssh_node:
+                # using new_ip_address here since if it hasn't changed it'll match ip_address
+                virtctl_vm_cmd = f"ssh core@{self.ssh_node} -o ConnectTimeout=5 'ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@{new_ip_address} 2>&1 | grep Permission' && echo 'True' || echo 'False'"
+                new_output = invoke_no_exit(virtctl_vm_cmd)
+                logging.debug(f"Disconnected access for {new_ip_address} on {self.ssh_node}: {new_output}")
+                if 'True' in new_output:
+                    return True, new_ip_address, None
+        return False, None, None
 
     def get_vm_access(self, vm_name: str = '', namespace: str = ''):
         """
@@ -57,8 +107,8 @@ class VirtChecker:
         :param namespace:
         :return: virtctl_status 'True' if successful, or an error message if it fails.
         """
-        virtctl_vm_cmd = f"virtctl ssh --local-ssh-opts='-o BatchMode=yes' --local-ssh-opts='-o PasswordAuthentication=no' --local-ssh-opts='-o ConnectTimeout=2' root@vmi/{vm_name} -n {namespace} 2>&1 |egrep 'denied|verification failed'  && echo 'True' || echo 'False'"
-        check_virtctl_vm_cmd = f"virtctl ssh --local-ssh-opts='-o BatchMode=yes' --local-ssh-opts='-o PasswordAuthentication=no' --local-ssh-opts='-o ConnectTimeout=2' root@{vm_name} -n {namespace} 2>&1 |egrep 'denied|verification failed'  && echo 'True' || echo 'False'"
+        virtctl_vm_cmd = f"virtctl ssh --local-ssh-opts='-o BatchMode=yes' --local-ssh-opts='-o PasswordAuthentication=no' --local-ssh-opts='-o ConnectTimeout=5' root@vmi/{vm_name} -n {namespace} 2>&1 |egrep 'denied|verification failed'  && echo 'True' || echo 'False'"
+        check_virtctl_vm_cmd = f"virtctl ssh --local-ssh-opts='-o BatchMode=yes' --local-ssh-opts='-o PasswordAuthentication=no' --local-ssh-opts='-o ConnectTimeout=5' root@{vm_name} -n {namespace} 2>&1 |egrep 'denied|verification failed'  && echo 'True' || echo 'False'"
         if 'True' in invoke_no_exit(check_virtctl_vm_cmd):
             return True
         else:
@@ -71,30 +121,54 @@ class VirtChecker:
         for thread in self.threads:
             thread.join()
 
-    def batch_list(self,  queue: queue.Queue, batch_size=20):
-        # Provided prints to easily visualize how the threads are processed.    
-        for i in range (0, len(self.vm_list),batch_size):
-            sub_list = self.vm_list[i: i+batch_size]
-            index = i
-            t = threading.Thread(target=self.run_virt_check,name=str(index), args=(sub_list,queue))
-            self.threads.append(t)
-            t.start()
+    def batch_list(self, queue: queue.SimpleQueue = None):
+        if self.batch_size > 0:
+            # Provided prints to easily visualize how the threads are processed.    
+            for i in range (0, len(self.vm_list),self.batch_size):
+                if i+self.batch_size > len(self.vm_list):
+                    sub_list = self.vm_list[i:]
+                else:
+                    sub_list = self.vm_list[i: i+self.batch_size]
+                index = i
+                t = threading.Thread(target=self.run_virt_check,name=str(index), args=(sub_list,queue))
+                self.threads.append(t)
+                t.start()
 
-    
-    def run_virt_check(self, vm_list_batch, virt_check_telemetry_queue: queue.Queue):
+    def increment_iterations(self):
+        """Thread-safe method to increment current_iterations"""
+        with self.iteration_lock:
+            self.current_iterations += 1
+
+    def run_virt_check(self, vm_list_batch, virt_check_telemetry_queue: queue.SimpleQueue):
         
         virt_check_telemetry = []
         virt_check_tracker = {}
-        while self.current_iterations < self.iterations:
+        while True:
+            # Thread-safe read of current_iterations
+            with self.iteration_lock:
+                current = self.current_iterations
+            if current >= self.iterations:
+                break
             for vm in vm_list_batch:
+                start_time= datetime.now()
                 try: 
                     if not self.disconnected: 
                         vm_status = self.get_vm_access(vm.vm_name, vm.namespace)
                     else:
-                        vm_status = self.check_disconnected_access(vm.ip_address, vm.node_name)
+                        # if new ip address exists use it 
+                        if vm.new_ip_address: 
+                            vm_status, new_ip_address, new_node_name = self.check_disconnected_access(vm.new_ip_address, vm.node_name, vm.vm_name)
+                            # since we already set the new ip address, we don't want to reset to none each time
+                        else: 
+                            vm_status, new_ip_address, new_node_name = self.check_disconnected_access(vm.ip_address, vm.node_name, vm.vm_name)
+                            if new_ip_address and vm.ip_address != new_ip_address:
+                                vm.new_ip_address = new_ip_address
+                            if new_node_name and vm.node_name != new_node_name:
+                                vm.node_name = new_node_name
                 except Exception:
+                    logging.info('Exception in get vm status')
                     vm_status = False
-                
+
                 if vm.vm_name not in virt_check_tracker:
                     start_timestamp = datetime.now()
                     virt_check_tracker[vm.vm_name] = {
@@ -103,9 +177,11 @@ class VirtChecker:
                         "namespace": vm.namespace,
                         "node_name": vm.node_name,
                         "status": vm_status,
-                        "start_timestamp": start_timestamp
+                        "start_timestamp": start_timestamp,
+                        "new_ip_address": vm.new_ip_address
                     }
                 else:
+                    
                     if vm_status != virt_check_tracker[vm.vm_name]["status"]:
                         end_timestamp = datetime.now()
                         start_timestamp = virt_check_tracker[vm.vm_name]["start_timestamp"]
@@ -113,6 +189,8 @@ class VirtChecker:
                         virt_check_tracker[vm.vm_name]["end_timestamp"] = end_timestamp.isoformat()
                         virt_check_tracker[vm.vm_name]["duration"] = duration
                         virt_check_tracker[vm.vm_name]["start_timestamp"] = start_timestamp.isoformat()
+                        if vm.new_ip_address:
+                            virt_check_tracker[vm.vm_name]["new_ip_address"] = vm.new_ip_address
                         if self.only_failures: 
                             if not virt_check_tracker[vm.vm_name]["status"]:
                                 virt_check_telemetry.append(VirtCheck(virt_check_tracker[vm.vm_name]))
@@ -132,4 +210,66 @@ class VirtChecker:
                     virt_check_telemetry.append(VirtCheck(virt_check_tracker[vm]))
             else:
                 virt_check_telemetry.append(VirtCheck(virt_check_tracker[vm]))
-        virt_check_telemetry_queue.put(virt_check_telemetry)
+        try:
+            virt_check_telemetry_queue.put(virt_check_telemetry)
+        except Exception as e:
+            logging.error('Put queue error ' + str(e))
+    def run_post_virt_check(self, vm_list_batch, virt_check_telemetry, post_virt_check_queue: queue.SimpleQueue):
+        
+        virt_check_telemetry = []
+        virt_check_tracker = {}
+        start_timestamp = datetime.now()
+        for vm in vm_list_batch:
+            
+            try: 
+                if not self.disconnected: 
+                    vm_status = self.get_vm_access(vm.vm_name, vm.namespace)
+                else:
+                    vm_status, new_ip_address, new_node_name = self.check_disconnected_access(vm.ip_address, vm.node_name, vm.vm_name)
+                    if new_ip_address and vm.ip_address != new_ip_address:
+                        vm.new_ip_address = new_ip_address
+                    if new_node_name and vm.node_name != new_node_name:
+                        vm.node_name = new_node_name
+            except Exception:
+                vm_status = False
+            
+            if not vm_status:
+
+                virt_check_tracker= {
+                    "vm_name": vm.vm_name,
+                    "ip_address": vm.ip_address,
+                    "namespace": vm.namespace,
+                    "node_name": vm.node_name,
+                    "status": vm_status,
+                    "start_timestamp": start_timestamp.isoformat(),
+                    "new_ip_address": vm.new_ip_address,
+                    "duration": 0,
+                    "end_timestamp": start_timestamp.isoformat()
+                }
+                
+                virt_check_telemetry.append(VirtCheck(virt_check_tracker))
+        post_virt_check_queue.put(virt_check_telemetry)
+    
+
+    def gather_post_virt_checks(self, kubevirt_check_telem):
+
+        post_kubevirt_check_queue = queue.SimpleQueue()
+        post_threads = []
+
+        if self.batch_size > 0:
+            for i in range (0, len(self.vm_list),self.batch_size):
+                sub_list = self.vm_list[i: i+self.batch_size]
+                index = i
+                t = threading.Thread(target=self.run_post_virt_check,name=str(index), args=(sub_list,kubevirt_check_telem, post_kubevirt_check_queue))
+                post_threads.append(t)
+                t.start()
+
+            kubevirt_check_telem = []
+            for thread in post_threads:
+                thread.join()
+                if not post_kubevirt_check_queue.empty():
+                    kubevirt_check_telem.extend(post_kubevirt_check_queue.get_nowait())
+        
+        if self.exit_on_failure and len(kubevirt_check_telem) > 0:
+            self.ret_value = 2
+        return kubevirt_check_telem

requirements.txt

@@ -9,16 +9,16 @@ azure-mgmt-network==27.0.0
 itsdangerous==2.0.1
 coverage==7.6.12
 datetime==5.4
-docker==7.0.0
+docker>=6.0,<7.0  # docker 7.0+ has breaking changes with Unix sockets
 gitpython==3.1.41
 google-auth==2.37.0
 google-cloud-compute==1.22.0
 ibm_cloud_sdk_core==3.18.0
 ibm_vpc==0.20.0
 jinja2==3.1.6
-krkn-lib==5.1.5
+krkn-lib==5.1.13
 lxml==5.1.0
-kubernetes==28.1.0
+kubernetes==34.1.0
 numpy==1.26.4
 pandas==2.2.0
 openshift-client==1.0.21
@@ -28,11 +28,12 @@ pyfiglet==1.0.2
 pytest==8.0.0
 python-ipmi==0.5.4
 python-openstackclient==6.5.0
-requests==2.32.4
+requests<2.32  # requests 2.32+ breaks Unix socket support (http+docker scheme)
+requests-unixsocket>=0.4.0  # Required for Docker Unix socket support
 service_identity==24.1.0
 PyYAML==6.0.1
 setuptools==78.1.1
-werkzeug==3.0.6
+werkzeug==3.1.4
 wheel==0.42.0
 zope.interface==5.4.0
 

run_kraken.py

@@ -133,7 +133,7 @@ def main(options, command: Optional[str]) -> int:
         telemetry_api_url = config["telemetry"].get("api_url")
         health_check_config = get_yaml_item_value(config, "health_checks",{})
         kubevirt_check_config = get_yaml_item_value(config, "kubevirt_checks", {})
-
+        
         # Initialize clients
         if not os.path.isfile(kubeconfig_path) and not os.path.isfile(
                 "/var/run/secrets/kubernetes.io/serviceaccount/token"
@@ -141,7 +141,7 @@ def main(options, command: Optional[str]) -> int:
             logging.error(
                 "Cannot read the kubeconfig file at %s, please check" % kubeconfig_path
             )
-            return 1
+            return -1
         logging.info("Initializing client to talk to the Kubernetes cluster")
 
         # Generate uuid for the run
@@ -184,10 +184,10 @@ def main(options, command: Optional[str]) -> int:
         # Set up kraken url to track signal
         if not 0 <= int(port) <= 65535:
             logging.error("%s isn't a valid port number, please check" % (port))
-            return 1
+            return -1
         if not signal_address:
             logging.error("Please set the signal address in the config")
-            return 1
+            return -1
         address = (signal_address, port)
 
         # If publish_running_status is False this should keep us going
@@ -220,7 +220,7 @@ def main(options, command: Optional[str]) -> int:
                         "invalid distribution selected, running openshift scenarios against kubernetes cluster."
                         "Please set 'kubernetes' in config.yaml krkn.platform and try again"
                     )
-                    return 1
+                    return -1
         if cv != "":
             logging.info(cv)
         else:
@@ -326,7 +326,7 @@ def main(options, command: Optional[str]) -> int:
                                                args=(health_check_config, health_check_telemetry_queue))
         health_check_worker.start()
 
-        kubevirt_check_telemetry_queue = queue.Queue()
+        kubevirt_check_telemetry_queue = queue.SimpleQueue()
         kubevirt_checker = VirtChecker(kubevirt_check_config, iterations=iterations, krkn_lib=kubecli)
         kubevirt_checker.batch_list(kubevirt_check_telemetry_queue)
 
@@ -361,7 +361,7 @@ def main(options, command: Optional[str]) -> int:
                             logging.error(
                                 f"impossible to find scenario {scenario_type}, plugin not found. Exiting"
                             )
-                            sys.exit(1)
+                            sys.exit(-1)
 
                         failed_post_scenarios, scenario_telemetries = (
                             scenario_plugin.run_scenarios(
@@ -375,10 +375,12 @@ def main(options, command: Optional[str]) -> int:
                             prometheus_plugin.critical_alerts(
                                 prometheus,
                                 summary,
+                                elastic_search,
                                 run_uuid,
                                 scenario_type,
                                 start_time,
                                 datetime.datetime.now(),
+                                elastic_alerts_index
                             )
 
                             chaos_output.critical_alerts = summary
@@ -391,8 +393,7 @@ def main(options, command: Optional[str]) -> int:
 
             iteration += 1
             health_checker.current_iterations += 1
-            kubevirt_checker.current_iterations += 1
-
+            kubevirt_checker.increment_iterations()
         # telemetry
         # in order to print decoded telemetry data even if telemetry collection
         # is disabled, it's necessary to serialize the ChaosRunTelemetry object
@@ -406,15 +407,12 @@ def main(options, command: Optional[str]) -> int:
         
         kubevirt_checker.thread_join()
         kubevirt_check_telem = []
-        i =0
-        while i <= kubevirt_checker.threads_limit:
-            if not kubevirt_check_telemetry_queue.empty():
-                kubevirt_check_telem.extend(kubevirt_check_telemetry_queue.get_nowait())
-            else:
-                break
-            i+= 1
+        while not kubevirt_check_telemetry_queue.empty():
+            kubevirt_check_telem.extend(kubevirt_check_telemetry_queue.get_nowait())
         chaos_telemetry.virt_checks = kubevirt_check_telem
-
+        
+        post_kubevirt_check = kubevirt_checker.gather_post_virt_checks(kubevirt_check_telem)
+        chaos_telemetry.post_virt_checks = post_kubevirt_check
         # if platform is openshift will be collected
         # Cloud platform and network plugins metadata
         # through OCP specific APIs
@@ -524,7 +522,7 @@ def main(options, command: Optional[str]) -> int:
 
             else:
                 logging.error("Alert profile is not defined")
-                return 1
+                return -1
                 # sys.exit(1)
         if enable_metrics:
             logging.info(f'Capturing metrics using file {metrics_profile}')
@@ -539,21 +537,28 @@ def main(options, command: Optional[str]) -> int:
                 telemetry_json
             )
 
+        # want to exit with 1 first to show failure of scenario 
+        # even if alerts failing
+        if failed_post_scenarios:
+            logging.error(
+                "Post scenarios are still failing at the end of all iterations"
+            )
+            # sys.exit(1)
+            return 1
+
         if post_critical_alerts > 0:
             logging.error("Critical alerts are firing, please check; exiting")
             # sys.exit(2)
             return 2
 
-        if failed_post_scenarios:
-            logging.error(
-                "Post scenarios are still failing at the end of all iterations"
-            )
-            # sys.exit(2)
-            return 2
         if health_checker.ret_value != 0:
             logging.error("Health check failed for the applications, Please check; exiting")
             return health_checker.ret_value
 
+        if kubevirt_checker.ret_value != 0:
+            logging.error("Kubevirt check still had failed VMIs at end of run, Please check; exiting")
+            return kubevirt_checker.ret_value
+
         logging.info(
             "Successfully finished running Kraken. UUID for the run: "
             "%s. Report generated at %s. Exiting" % (run_uuid, report_file)
@@ -561,7 +566,7 @@ def main(options, command: Optional[str]) -> int:
     else:
         logging.error("Cannot find a config at %s, please check" % (cfg))
         # sys.exit(1)
-        return 2
+        return -1
 
     return 0
 
@@ -628,6 +633,14 @@ if __name__ == "__main__":
         default=None,
     )
 
+    parser.add_option(
+        "-d",
+        "--debug",
+        dest="debug",
+        help="enable debug logging",
+        default=False,
+    )
+
     (options, args) = parser.parse_args()
     
     # If no command or regular execution, continue with existing logic
@@ -640,7 +653,7 @@ if __name__ == "__main__":
     ]
 
     logging.basicConfig(
-        level=logging.INFO,
+        level=logging.DEBUG if options.debug else logging.INFO,
         format="%(asctime)s [%(levelname)s] %(message)s",
         handlers=handlers,
     )
@@ -722,4 +735,4 @@ if __name__ == "__main__":
         with open(junit_testcase_file_path, "w") as stream:
             stream.write(junit_testcase_xml)
 
-    sys.exit(retval)
+    sys.exit(retval)

scenarios/kind/node_scenarios_example.yml

@@ -1,16 +1,18 @@
 node_scenarios:
   - actions:                                                        # node chaos scenarios to be injected
     - node_stop_start_scenario
-    node_name: kind-worker                                          # node on which scenario has to be injected; can set multiple names separated by comma
-    # label_selector: node-role.kubernetes.io/worker                # when node_name is not specified, a node with matching label_selector is selected for node chaos scenario injection
+    # node_name: kind-control-plane                                        # node on which scenario has to be injected; can set multiple names separated by comma
+    label_selector: kubernetes.io/hostname=kind-worker              # when node_name is not specified, a node with matching label_selector is selected for node chaos scenario injection
     instance_count: 1                                               # Number of nodes to perform action/select that match the label selector
     runs: 1                                                         # number of times to inject each scenario under actions (will perform on same node each time)
     timeout: 120                                                    # duration to wait for completion of node scenario injection
     cloud_type: docker                                                # cloud type on which Kubernetes/OpenShift runs
+    duration: 10
   - actions:
     - node_reboot_scenario
-    node_name: kind-worker
-    # label_selector: node-role.kubernetes.io/infra
+    node_name: kind-control-plane
+    # label_selector: kubernetes.io/hostname=kind-worker
     instance_count: 1
     timeout: 120
     cloud_type: docker
+    kube_check: false

scenarios/kind/pod_etcd.yml

@@ -3,3 +3,4 @@
     namespace_pattern: "kube-system"
     label_selector: "component=etcd"
     krkn_pod_recovery_time: 120
+    kill: 1

scenarios/kind/pvc_scenario.yaml

@@ -0,0 +1,7 @@
+pvc_scenario:
+  pvc_name: kraken-test-pvc         # Name of the target PVC
+  pod_name: kraken-test-pod      # Name of the pod where the PVC is mounted, it will be ignored if the pvc_name is defined
+  namespace: kraken  # Namespace where the PVC is
+  fill_percentage: 98           # Target percentage to fill up the cluster, value must be higher than current percentage, valid values are between 0 and 99
+  duration: 10                  # Duration in seconds for the fault
+  block_size: 102400            # used only by dd if fallocate not present in the container

scenarios/kube/container_dns.yml

@@ -6,3 +6,4 @@ scenarios:
   action: 1
   count: 1
   retry_wait: 60
+  exclude_label: ""

scenarios/openshift/app_outage.yaml

@@ -3,3 +3,4 @@ application_outage:                                  # Scenario to create an out
   namespace: <namespace-with-application>            # Namespace to target - all application routes will go inaccessible if pod selector is empty
   pod_selector: {app: foo}                            # Pods to target
   block: [Ingress, Egress]                           # It can be Ingress or Egress or Ingress, Egress
+  exclude_label: ""                                  # Optional label selector to exclude pods. Supports dict, string, or list format

scenarios/openshift/aws_node_scenarios.yml

@@ -10,6 +10,7 @@ node_scenarios:
     cloud_type: aws                                               # cloud type on which Kubernetes/OpenShift runs  
     parallel: true                                                # Run action on label or node name in parallel or sequential, defaults to sequential
     kube_check: true                                              # Run the kubernetes api calls to see if the node gets to a certain state during the node scenario
+    poll_interval: 15                                             # Time interval(in seconds) to periodically check the node's status
   - actions:
     - node_reboot_scenario
     node_name:

scenarios/openshift/container_etcd.yml

@@ -6,3 +6,4 @@ scenarios:
   action: 1
   count: 1
   expected_recovery_time: 120
+  exclude_label: ""

scenarios/openshift/customapp_pod.yaml

@@ -1,6 +1,15 @@
 # yaml-language-server: $schema=../plugin.schema.json
 - id: kill-pods
   config:
-    namespace_pattern: ^acme-air$
+    namespace_pattern: "kube-system"
     name_pattern: .*
-    krkn_pod_recovery_time: 120
+    krkn_pod_recovery_time: 60
+    kill: 1     # num of pods to kill
+    #Not needed by default, but can be used if you want to target pods on specific nodes
+    # Option 1: Target pods on nodes with specific labels [master/worker nodes]
+    node_label_selector: node-role.kubernetes.io/control-plane=      # Target control-plane nodes (works on both k8s and openshift) 
+    # Option 2: Target pods of specific nodes (testing mixed node types)
+    # node_names: 
+    #   - ip-10-0-31-8.us-east-2.compute.internal      # Worker node 1
+    #   - ip-10-0-48-188.us-east-2.compute.internal    # Worker node 2  
+    #   - ip-10-0-14-59.us-east-2.compute.internal     # Master node 1

scenarios/openshift/egress_ip_ovn.yml

@@ -0,0 +1,20 @@
+# EgressIP failovr scenario - blocks OVN healthcheck port 9107 for EgressIP to move to another node
+
+- id: node_network_filter
+  image: "quay.io/krkn-chaos/krkn-network-chaos:latest"
+  wait_duration: 60
+  test_duration: 30
+  label_selector: "k8s.ovn.org/egress-assignable="
+  service_account: ""
+  taints: []
+  namespace: 'default'
+  instance_count: 1
+  execution: serial
+  ingress: true
+  egress: false
+  target: ''
+  interfaces: []
+  ports:
+    - 9107
+  protocols:
+    - tcp

scenarios/openshift/etcd.yml

@@ -4,3 +4,4 @@
     namespace_pattern: ^openshift-etcd$
     label_selector: k8s-app=etcd
     krkn_pod_recovery_time: 120
+    exclude_label: "" # excludes pods marked with this label from chaos

scenarios/openshift/openshift-apiserver.yml

@@ -4,4 +4,5 @@
     namespace_pattern: ^openshift-apiserver$
     label_selector: app=openshift-apiserver-a
     krkn_pod_recovery_time: 120
+    exclude_label: "" # excludes pods marked with this label from chaos
 

scenarios/openshift/openshift-kube-apiserver.yml

@@ -4,4 +4,5 @@
     namespace_pattern: ^openshift-kube-apiserver$
     label_selector: app=openshift-kube-apiserver
     krkn_pod_recovery_time: 120
+    exclude_label: "" # excludes pods marked with this label from chaos
 

scenarios/openshift/prom_kill.yml

@@ -2,4 +2,5 @@
   config:
     namespace_pattern: ^openshift-monitoring$
     label_selector: statefulset.kubernetes.io/pod-name=prometheus-k8s-0
-    krkn_pod_recovery_time: 120
+    krkn_pod_recovery_time: 120
+    exclude_label: "" # excludes pods marked with this label from chaos

scenarios/openshift/regex_openshift_pod_kill.yml

@@ -5,3 +5,4 @@
     name_pattern: .*
     kill: 3
     krkn_pod_recovery_time: 120
+    exclude_label: "" # excludes pods marked with this label from chaos

scenarios/plugin.schema.README.md

@@ -1,5 +0,0 @@
-This file is generated by running the "plugins" module in the kraken project:
-
-```
-python -m kraken.plugins >scenarios/plugin.schema.json
-```

scenarios/plugin.schema.json

@@ -1,584 +0,0 @@
-{
-	"$id": "https://github.com/redhat-chaos/krkn/",
-	"$schema": "https://json-schema.org/draft/2020-12/schema",
-	"title": "Kraken Arcaflow scenarios",
-	"description": "Serial execution of Arcaflow Python plugins. See https://github.com/arcaflow for details.",
-	"type": "array",
-	"minContains": 1,
-	"items": {
-		"oneOf": [
-			{
-				"type": "object",
-				"properties": {
-					"id": {
-						"type": "string",
-						"const": "run_python"
-					},
-					"config": {
-						"$defs": {
-							"RunPythonFileInput": {
-								"type": "object",
-								"properties": {
-									"filename": {
-										"type": "string"
-									}
-								},
-								"required": [
-									"filename"
-								],
-								"additionalProperties": false,
-								"dependentRequired": {}
-							}
-						},
-						"type": "object",
-						"properties": {
-							"filename": {
-								"type": "string"
-							}
-						},
-						"required": [
-							"filename"
-						],
-						"additionalProperties": false,
-						"dependentRequired": {}
-					}
-				},
-				"required": [
-					"id",
-					"config"
-				]
-			},
-			{
-				"type": "object",
-				"title": "pod_network_outage Arcaflow scenarios",
-				"properties": {
-					"id": {
-						"type": "string",
-						"const": "pod_network_outage"
-					},
-					"config": {
-						"$defs": {
-							"InputParams": {
-								"type": "object",
-								"properties": {
-									"namespace": {
-										"type": "string",
-										"minLength": 1,
-										"title": "Namespace",
-										"description": "Namespace of the pod to which filter need to be appliedfor details."
-									},
-									"image": {
-										"type": "string",
-										"minLength": 1,
-										"title": "Image",
-										"default": "image: quay.io/krkn-chaos/krkn:tools",
-										"description": "Image of the krkn tools to run network outage."
-									},
-									"direction": {
-										"type": "array",
-										"items": {
-											"type": "string"
-										},
-										"default": [
-											"ingress",
-											"egress"
-										],
-										"title": "Direction",
-										"description": "List of directions to apply filtersDefault both egress and ingress."
-									},
-									"ingress_ports": {
-										"type": "array",
-										"items": {
-											"type": "integer"
-										},
-										"default": [],
-										"title": "Ingress ports",
-										"description": "List of ports to block traffic onDefault [], i.e. all ports"
-									},
-									"egress_ports": {
-										"type": "array",
-										"items": {
-											"type": "integer"
-										},
-										"default": [],
-										"title": "Egress ports",
-										"description": "List of ports to block traffic onDefault [], i.e. all ports"
-									},
-									"kubeconfig_path": {
-										"type": "string",
-										"title": "Kubeconfig path",
-										"description": "Kubeconfig file as string\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details."
-									},
-									"pod_name": {
-										"type": "string",
-										"title": "Pod name",
-										"description": "When label_selector is not specified, pod matching the name will beselected for the chaos scenario"
-									},
-									"label_selector": {
-										"type": "string",
-										"title": "Label selector",
-										"description": "Kubernetes label selector for the target pod. When pod_name is not specified, pod with matching label_selector is selected for chaos scenario"
-									},
-									"kraken_config": {
-										"type": "string",
-										"title": "Kraken Config",
-										"description": "Path to the config file of Kraken. Set this field if you wish to publish status onto Cerberus"
-									},
-									"test_duration": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 120,
-										"title": "Test duration",
-										"description": "Duration for which each step of the ingress chaos testing is to be performed."
-									},
-									"wait_duration": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 300,
-										"title": "Wait Duration",
-										"description": "Wait duration for finishing a test and its cleanup.Ensure that it is significantly greater than wait_duration"
-									},
-									"instance_count": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 1,
-										"title": "Instance Count",
-										"description": "Number of pods to perform action/select that match the label selector."
-									}
-								},
-								"required": [
-									"namespace"
-								],
-								"additionalProperties": false,
-								"dependentRequired": {}
-							}
-						},
-						"type": "object",
-						"properties": {
-							"namespace": {
-								"type": "string",
-								"minLength": 1,
-								"title": "Namespace",
-								"description": "Namespace of the pod to which filter need to be appliedfor details."
-							},
-							"direction": {
-								"type": "array",
-								"items": {
-									"type": "string"
-								},
-								"default": [
-									"ingress",
-									"egress"
-								],
-								"title": "Direction",
-								"description": "List of directions to apply filtersDefault both egress and ingress."
-							},
-							"ingress_ports": {
-								"type": "array",
-								"items": {
-									"type": "integer"
-								},
-								"default": [],
-								"title": "Ingress ports",
-								"description": "List of ports to block traffic onDefault [], i.e. all ports"
-							},
-							"egress_ports": {
-								"type": "array",
-								"items": {
-									"type": "integer"
-								},
-								"default": [],
-								"title": "Egress ports",
-								"description": "List of ports to block traffic onDefault [], i.e. all ports"
-							},
-							"kubeconfig_path": {
-								"type": "string",
-								"title": "Kubeconfig path",
-								"description": "Kubeconfig file as string\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details."
-							},
-							"pod_name": {
-								"type": "string",
-								"title": "Pod name",
-								"description": "When label_selector is not specified, pod matching the name will beselected for the chaos scenario"
-							},
-							"label_selector": {
-								"type": "string",
-								"title": "Label selector",
-								"description": "Kubernetes label selector for the target pod. When pod_name is not specified, pod with matching label_selector is selected for chaos scenario"
-							},
-							"kraken_config": {
-								"type": "string",
-								"title": "Kraken Config",
-								"description": "Path to the config file of Kraken. Set this field if you wish to publish status onto Cerberus"
-							},
-							"test_duration": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 120,
-								"title": "Test duration",
-								"description": "Duration for which each step of the ingress chaos testing is to be performed."
-							},
-							"wait_duration": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 300,
-								"title": "Wait Duration",
-								"description": "Wait duration for finishing a test and its cleanup.Ensure that it is significantly greater than wait_duration"
-							},
-							"instance_count": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 1,
-								"title": "Instance Count",
-								"description": "Number of pods to perform action/select that match the label selector."
-							}
-						},
-						"required": [
-							"namespace"
-						],
-						"additionalProperties": false,
-						"dependentRequired": {}
-					}
-				},
-				"required": [
-					"id",
-					"config"
-				]
-			},
-			{
-				"type": "object",
-				"title": "pod_egress_shaping Arcaflow scenarios",
-				"properties": {
-					"id": {
-						"type": "string",
-						"const": "pod_egress_shaping"
-					},
-					"config": {
-						"$defs": {
-							"EgressParams": {
-								"type": "object",
-								"properties": {
-									"namespace": {
-										"type": "string",
-										"minLength": 1,
-										"title": "Namespace",
-										"description": "Namespace of the pod to which filter need to be appliedfor details."
-									},
-									"image": {
-										"type": "string",
-										"minLength": 1,
-										"title": "Image",
-										"default": "image: quay.io/krkn-chaos/krkn:tools",
-										"description": "Image of the krkn tools to run network outage."
-									},
-									"kubeconfig_path": {
-										"type": "string",
-										"title": "Kubeconfig path",
-										"description": "Kubeconfig file as string\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details."
-									},
-									"pod_name": {
-										"type": "string",
-										"title": "Pod name",
-										"description": "When label_selector is not specified, pod matching the name will beselected for the chaos scenario"
-									},
-									"label_selector": {
-										"type": "string",
-										"title": "Label selector",
-										"description": "Kubernetes label selector for the target pod. When pod_name is not specified, pod with matching label_selector is selected for chaos scenario"
-									},
-									"kraken_config": {
-										"type": "string",
-										"title": "Kraken Config",
-										"description": "Path to the config file of Kraken. Set this field if you wish to publish status onto Cerberus"
-									},
-									"test_duration": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 90,
-										"title": "Test duration",
-										"description": "Duration for which each step of the ingress chaos testing is to be performed."
-									},
-									"wait_duration": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 300,
-										"title": "Wait Duration",
-										"description": "Wait duration for finishing a test and its cleanup.Ensure that it is significantly greater than wait_duration"
-									},
-									"instance_count": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 1,
-										"title": "Instance Count",
-										"description": "Number of pods to perform action/select that match the label selector."
-									},
-									"execution_type": {
-										"type": "string",
-										"default": "parallel",
-										"title": "Execution Type",
-										"description": "The order in which the ingress filters are applied. Execution type can be 'serial' or 'parallel'"
-									},
-									"network_params": {
-										"type": "object",
-										"propertyNames": {},
-										"additionalProperties": {
-											"type": "string"
-										},
-										"title": "Network Parameters",
-										"description": "The network filters that are applied on the interface. The currently supported filters are latency, loss and bandwidth"
-									}
-								},
-								"required": [
-									"namespace"
-								],
-								"additionalProperties": false,
-								"dependentRequired": {}
-							}
-						},
-						"type": "object",
-						"properties": {
-							"namespace": {
-								"type": "string",
-								"minLength": 1,
-								"title": "Namespace",
-								"description": "Namespace of the pod to which filter need to be appliedfor details."
-							},
-							"kubeconfig_path": {
-								"type": "string",
-								"title": "Kubeconfig path",
-								"description": "Kubeconfig file as string\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details."
-							},
-							"pod_name": {
-								"type": "string",
-								"title": "Pod name",
-								"description": "When label_selector is not specified, pod matching the name will beselected for the chaos scenario"
-							},
-							"label_selector": {
-								"type": "string",
-								"title": "Label selector",
-								"description": "Kubernetes label selector for the target pod. When pod_name is not specified, pod with matching label_selector is selected for chaos scenario"
-							},
-							"kraken_config": {
-								"type": "string",
-								"title": "Kraken Config",
-								"description": "Path to the config file of Kraken. Set this field if you wish to publish status onto Cerberus"
-							},
-							"test_duration": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 90,
-								"title": "Test duration",
-								"description": "Duration for which each step of the ingress chaos testing is to be performed."
-							},
-							"wait_duration": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 300,
-								"title": "Wait Duration",
-								"description": "Wait duration for finishing a test and its cleanup.Ensure that it is significantly greater than wait_duration"
-							},
-							"instance_count": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 1,
-								"title": "Instance Count",
-								"description": "Number of pods to perform action/select that match the label selector."
-							},
-							"execution_type": {
-								"type": "string",
-								"default": "parallel",
-								"title": "Execution Type",
-								"description": "The order in which the ingress filters are applied. Execution type can be 'serial' or 'parallel'"
-							},
-							"network_params": {
-								"type": "object",
-								"propertyNames": {},
-								"additionalProperties": {
-									"type": "string"
-								},
-								"title": "Network Parameters",
-								"description": "The network filters that are applied on the interface. The currently supported filters are latency, loss and bandwidth"
-							}
-						},
-						"required": [
-							"namespace"
-						],
-						"additionalProperties": false,
-						"dependentRequired": {}
-					}
-				},
-				"required": [
-					"id",
-					"config"
-				]
-			},
-			{
-				"type": "object",
-				"title": "pod_ingress_shaping Arcaflow scenarios",
-				"properties": {
-					"id": {
-						"type": "string",
-						"const": "pod_ingress_shaping"
-					},
-					"config": {
-						"$defs": {
-							"IngressParams": {
-								"type": "object",
-								"properties": {
-									"namespace": {
-										"type": "string",
-										"minLength": 1,
-										"title": "Namespace",
-										"description": "Namespace of the pod to which filter need to be appliedfor details."
-									},
-									"image": {
-										"type": "string",
-										"minLength": 1,
-										"title": "Image",
-										"default": "image: quay.io/krkn-chaos/krkn:tools",
-										"description": "Image of the krkn tools to run network outage."
-									},
-									"network_params": {
-										"type": "object",
-										"propertyNames": {},
-										"additionalProperties": {
-											"type": "string"
-										},
-										"title": "Network Parameters",
-										"description": "The network filters that are applied on the interface. The currently supported filters are latency, loss and bandwidth"
-									},
-									"kubeconfig_path": {
-										"type": "string",
-										"title": "Kubeconfig path",
-										"description": "Kubeconfig file as string\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details."
-									},
-									"pod_name": {
-										"type": "string",
-										"title": "Pod name",
-										"description": "When label_selector is not specified, pod matching the name will beselected for the chaos scenario"
-									},
-									"label_selector": {
-										"type": "string",
-										"title": "Label selector",
-										"description": "Kubernetes label selector for the target pod. When pod_name is not specified, pod with matching label_selector is selected for chaos scenario"
-									},
-									"kraken_config": {
-										"type": "string",
-										"title": "Kraken Config",
-										"description": "Path to the config file of Kraken. Set this field if you wish to publish status onto Cerberus"
-									},
-									"test_duration": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 90,
-										"title": "Test duration",
-										"description": "Duration for which each step of the ingress chaos testing is to be performed."
-									},
-									"wait_duration": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 300,
-										"title": "Wait Duration",
-										"description": "Wait duration for finishing a test and its cleanup.Ensure that it is significantly greater than wait_duration"
-									},
-									"instance_count": {
-										"type": "integer",
-										"minimum": 1,
-										"default": 1,
-										"title": "Instance Count",
-										"description": "Number of pods to perform action/select that match the label selector."
-									},
-									"execution_type": {
-										"type": "string",
-										"default": "parallel",
-										"title": "Execution Type",
-										"description": "The order in which the ingress filters are applied. Execution type can be 'serial' or 'parallel'"
-									}
-								},
-								"required": [
-									"namespace"
-								],
-								"additionalProperties": false,
-								"dependentRequired": {}
-							}
-						},
-						"type": "object",
-						"properties": {
-							"namespace": {
-								"type": "string",
-								"minLength": 1,
-								"title": "Namespace",
-								"description": "Namespace of the pod to which filter need to be appliedfor details."
-							},
-							"network_params": {
-								"type": "object",
-								"propertyNames": {},
-								"additionalProperties": {
-									"type": "string"
-								},
-								"title": "Network Parameters",
-								"description": "The network filters that are applied on the interface. The currently supported filters are latency, loss and bandwidth"
-							},
-							"kubeconfig_path": {
-								"type": "string",
-								"title": "Kubeconfig path",
-								"description": "Kubeconfig file as string\nSee https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/ for details."
-							},
-							"pod_name": {
-								"type": "string",
-								"title": "Pod name",
-								"description": "When label_selector is not specified, pod matching the name will beselected for the chaos scenario"
-							},
-							"label_selector": {
-								"type": "string",
-								"title": "Label selector",
-								"description": "Kubernetes label selector for the target pod. When pod_name is not specified, pod with matching label_selector is selected for chaos scenario"
-							},
-							"kraken_config": {
-								"type": "string",
-								"title": "Kraken Config",
-								"description": "Path to the config file of Kraken. Set this field if you wish to publish status onto Cerberus"
-							},
-							"test_duration": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 90,
-								"title": "Test duration",
-								"description": "Duration for which each step of the ingress chaos testing is to be performed."
-							},
-							"wait_duration": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 300,
-								"title": "Wait Duration",
-								"description": "Wait duration for finishing a test and its cleanup.Ensure that it is significantly greater than wait_duration"
-							},
-							"instance_count": {
-								"type": "integer",
-								"minimum": 1,
-								"default": 1,
-								"title": "Instance Count",
-								"description": "Number of pods to perform action/select that match the label selector."
-							},
-							"execution_type": {
-								"type": "string",
-								"default": "parallel",
-								"title": "Execution Type",
-								"description": "The order in which the ingress filters are applied. Execution type can be 'serial' or 'parallel'"
-							}
-						},
-						"required": [
-							"namespace"
-						],
-						"additionalProperties": false,
-						"dependentRequired": {}
-					}
-				},
-				"required": [
-					"id",
-					"config"
-				]
-			}
-		]
-	}
-}

tests/kubevirt_vm_outage/test_kubevirt_vm_outage.py

@@ -1,215 +0,0 @@
-import unittest
-import time
-from unittest.mock import MagicMock, patch
-
-import yaml
-from krkn_lib.k8s import KrknKubernetes
-from krkn_lib.models.telemetry import ScenarioTelemetry
-from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
-
-from krkn.scenario_plugins.kubevirt_vm_outage.kubevirt_vm_outage_scenario_plugin import KubevirtVmOutageScenarioPlugin
-
-
-class TestKubevirtVmOutageScenarioPlugin(unittest.TestCase):
-    
-    def setUp(self):
-        """
-        Set up test fixtures for KubevirtVmOutageScenarioPlugin
-        """
-        self.plugin = KubevirtVmOutageScenarioPlugin()
-        
-        # Create mock k8s client
-        self.k8s_client = MagicMock()
-        self.custom_object_client = MagicMock()
-        self.k8s_client.custom_object_client = self.custom_object_client
-        self.plugin.k8s_client = self.k8s_client
-        
-        # Mock methods needed for KubeVirt operations
-        self.k8s_client.list_custom_resource_definition = MagicMock()
-        
-        # Mock custom resource definition list with KubeVirt CRDs
-        crd_list = MagicMock()
-        crd_item = MagicMock()
-        crd_item.spec = MagicMock()
-        crd_item.spec.group = "kubevirt.io"
-        crd_list.items = [crd_item]
-        self.k8s_client.list_custom_resource_definition.return_value = crd_list
-        
-        # Mock VMI data
-        self.mock_vmi = {
-            "metadata": {
-                "name": "test-vm",
-                "namespace": "default"
-            },
-            "status": {
-                "phase": "Running"
-            }
-        }
-        
-        # Create test config
-        self.config = {
-            "scenarios": [
-                {
-                    "name": "kubevirt outage test",
-                    "scenario": "kubevirt_vm_outage",
-                    "parameters": {
-                        "vm_name": "test-vm",
-                        "namespace": "default",
-                        "duration": 0  
-                    }
-                }
-            ]
-        }
-        
-        # Create a temporary config file
-        import tempfile, os
-        temp_dir = tempfile.gettempdir()
-        self.scenario_file = os.path.join(temp_dir, "test_kubevirt_scenario.yaml")
-        with open(self.scenario_file, "w") as f:
-            yaml.dump(self.config, f)
-            
-        # Mock dependencies
-        self.telemetry = MagicMock(spec=KrknTelemetryOpenshift)
-        self.scenario_telemetry = MagicMock(spec=ScenarioTelemetry)
-        self.telemetry.get_lib_kubernetes.return_value = self.k8s_client
-        
-    def test_successful_injection_and_recovery(self):
-        """
-        Test successful deletion and recovery of a VMI
-        """
-        # Mock get_vmi to return our mock VMI
-        with patch.object(self.plugin, 'get_vmi', return_value=self.mock_vmi):
-            # Mock inject and recover to simulate success
-            with patch.object(self.plugin, 'inject', return_value=0) as mock_inject:
-                with patch.object(self.plugin, 'recover', return_value=0) as mock_recover:
-                    with patch("builtins.open", unittest.mock.mock_open(read_data=yaml.dump(self.config))):
-                        result = self.plugin.run("test-uuid", self.scenario_file, {}, self.telemetry, self.scenario_telemetry)
-                    
-        self.assertEqual(result, 0)
-        mock_inject.assert_called_once_with("test-vm", "default", False)
-        mock_recover.assert_called_once_with("test-vm", "default", False)
-        
-    def test_injection_failure(self):
-        """
-        Test failure during VMI deletion
-        """
-        # Mock get_vmi to return our mock VMI
-        with patch.object(self.plugin, 'get_vmi', return_value=self.mock_vmi):
-            # Mock inject to simulate failure
-            with patch.object(self.plugin, 'inject', return_value=1) as mock_inject:
-                with patch.object(self.plugin, 'recover', return_value=0) as mock_recover:
-                    with patch("builtins.open", unittest.mock.mock_open(read_data=yaml.dump(self.config))):
-                        result = self.plugin.run("test-uuid", self.scenario_file, {}, self.telemetry, self.scenario_telemetry)
-                    
-        self.assertEqual(result, 1)
-        mock_inject.assert_called_once_with("test-vm", "default", False)
-        mock_recover.assert_not_called()
-        
-    def test_disable_auto_restart(self):
-        """
-        Test VM auto-restart can be disabled
-        """
-        # Configure test with disable_auto_restart=True
-        self.config["scenarios"][0]["parameters"]["disable_auto_restart"] = True
-        
-        # Mock VM object for patching
-        mock_vm = {
-            "metadata": {"name": "test-vm", "namespace": "default"},
-            "spec": {}
-        }
-        
-        # Mock get_vmi to return our mock VMI
-        with patch.object(self.plugin, 'get_vmi', return_value=self.mock_vmi):
-            # Mock VM patch operation
-            with patch.object(self.plugin, 'patch_vm_spec') as mock_patch_vm:
-                mock_patch_vm.return_value = True
-                # Mock inject and recover
-                with patch.object(self.plugin, 'inject', return_value=0) as mock_inject:
-                    with patch.object(self.plugin, 'recover', return_value=0) as mock_recover:
-                        with patch("builtins.open", unittest.mock.mock_open(read_data=yaml.dump(self.config))):
-                            result = self.plugin.run("test-uuid", self.scenario_file, {}, self.telemetry, self.scenario_telemetry)
-        
-        self.assertEqual(result, 0)
-        # Should call patch_vm_spec to disable auto-restart
-        mock_patch_vm.assert_any_call("test-vm", "default", False)
-        # Should call patch_vm_spec to re-enable auto-restart during recovery
-        mock_patch_vm.assert_any_call("test-vm", "default", True)
-        mock_inject.assert_called_once_with("test-vm", "default", True)
-        mock_recover.assert_called_once_with("test-vm", "default", True)
-        
-    def test_recovery_when_vmi_does_not_exist(self):
-        """
-        Test recovery logic when VMI does not exist after deletion
-        """
-        # Store the original VMI in the plugin for recovery
-        self.plugin.original_vmi = self.mock_vmi.copy()
-        
-        # Create a cleaned vmi_dict as the plugin would
-        vmi_dict = self.mock_vmi.copy()
-        
-        # Set up running VMI data for after recovery
-        running_vmi = {
-            "metadata": {"name": "test-vm", "namespace": "default"},
-            "status": {"phase": "Running"}
-        }
-        
-        # Set up time.time to immediately exceed the timeout for auto-recovery
-        with patch('time.time', side_effect=[0, 301, 301, 301, 301, 310, 320]):
-            # Mock get_vmi to always return None (not auto-recovered)
-            with patch.object(self.plugin, 'get_vmi', side_effect=[None, None, running_vmi]):
-                # Mock the custom object API to return success
-                self.custom_object_client.create_namespaced_custom_object = MagicMock(return_value=running_vmi)
-                
-                # Run recovery with mocked time.sleep
-                with patch('time.sleep'):
-                    result = self.plugin.recover("test-vm", "default", False)
-        
-        self.assertEqual(result, 0)
-        # Verify create was called with the right arguments for our API version and kind
-        self.custom_object_client.create_namespaced_custom_object.assert_called_once_with(
-            group="kubevirt.io",
-            version="v1",
-            namespace="default",
-            plural="virtualmachineinstances",
-            body=vmi_dict
-        )
-    
-    def test_validation_failure(self):
-        """
-        Test validation failure when KubeVirt is not installed
-        """
-        # Mock empty CRD list (no KubeVirt CRDs)
-        empty_crd_list = MagicMock()
-        empty_crd_list.items = []
-        self.k8s_client.list_custom_resource_definition.return_value = empty_crd_list
-        
-        with patch("builtins.open", unittest.mock.mock_open(read_data=yaml.dump(self.config))):
-            result = self.plugin.run("test-uuid", self.scenario_file, {}, self.telemetry, self.scenario_telemetry)
-            
-        self.assertEqual(result, 1)
-        
-    def test_delete_vmi_timeout(self):
-        """
-        Test timeout during VMI deletion
-        """
-        # Mock successful delete operation
-        self.custom_object_client.delete_namespaced_custom_object = MagicMock(return_value={})
-        
-        # Mock that get_vmi always returns VMI (never gets deleted)
-        with patch.object(self.plugin, 'get_vmi', return_value=self.mock_vmi):
-            # Simulate timeout by making time.time return values that exceed the timeout
-            with patch('time.sleep'), patch('time.time', side_effect=[0, 10, 20, 130, 130, 130, 130, 140]):
-                result = self.plugin.inject("test-vm", "default", False)
-            
-        self.assertEqual(result, 1)
-        self.custom_object_client.delete_namespaced_custom_object.assert_called_once_with(
-            group="kubevirt.io",
-            version="v1",
-            namespace="default",
-            plural="virtualmachineinstances",
-            name="test-vm"
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/run_python_plugin.py

@@ -0,0 +1,37 @@
+import tempfile
+import unittest
+
+from krkn.scenario_plugins.native.run_python_plugin import (
+    RunPythonFileInput,
+    run_python_file,
+)
+
+
+class RunPythonPluginTest(unittest.TestCase):
+    def test_success_execution(self):
+        tmp_file = tempfile.NamedTemporaryFile()
+        tmp_file.write(bytes("print('Hello world!')", "utf-8"))
+        tmp_file.flush()
+        output_id, output_data = run_python_file(
+            params=RunPythonFileInput(tmp_file.name),
+            run_id="test-python-plugin-success",
+        )
+        self.assertEqual("success", output_id)
+        self.assertEqual("Hello world!\n", output_data.stdout)
+
+    def test_error_execution(self):
+        tmp_file = tempfile.NamedTemporaryFile()
+        tmp_file.write(
+            bytes("import sys\nprint('Hello world!')\nsys.exit(42)\n", "utf-8")
+        )
+        tmp_file.flush()
+        output_id, output_data = run_python_file(
+            params=RunPythonFileInput(tmp_file.name), run_id="test-python-plugin-error"
+        )
+        self.assertEqual("error", output_id)
+        self.assertEqual(42, output_data.exit_code)
+        self.assertEqual("Hello world!\n", output_data.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_abstract_node_scenarios.py

@@ -0,0 +1,415 @@
+"""
+Test suite for AbstractNode Scenarios
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_abstract_node_scenarios.py
+
+Assisted By: Claude Code
+"""
+
+import unittest
+from unittest.mock import Mock, patch
+from krkn.scenario_plugins.node_actions.abstract_node_scenarios import abstract_node_scenarios
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
+
+
+class TestAbstractNodeScenarios(unittest.TestCase):
+    """Test suite for abstract_node_scenarios class"""
+
+    def setUp(self):
+        """Set up test fixtures before each test method"""
+        self.mock_kubecli = Mock(spec=KrknKubernetes)
+        self.mock_affected_nodes_status = Mock(spec=AffectedNodeStatus)
+        self.mock_affected_nodes_status.affected_nodes = []
+        self.node_action_kube_check = True
+
+        self.scenarios = abstract_node_scenarios(
+            kubecli=self.mock_kubecli,
+            node_action_kube_check=self.node_action_kube_check,
+            affected_nodes_status=self.mock_affected_nodes_status
+        )
+
+    def test_init(self):
+        """Test initialization of abstract_node_scenarios"""
+        self.assertEqual(self.scenarios.kubecli, self.mock_kubecli)
+        self.assertEqual(self.scenarios.affected_nodes_status, self.mock_affected_nodes_status)
+        self.assertTrue(self.scenarios.node_action_kube_check)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    def test_node_stop_start_scenario(self, mock_logging, mock_sleep):
+        """Test node_stop_start_scenario calls stop and start in sequence"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+        duration = 60
+        poll_interval = 10
+
+        self.scenarios.node_stop_scenario = Mock()
+        self.scenarios.node_start_scenario = Mock()
+
+        # Act
+        self.scenarios.node_stop_start_scenario(
+            instance_kill_count, node, timeout, duration, poll_interval
+        )
+
+        # Assert
+        self.scenarios.node_stop_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout, poll_interval
+        )
+        mock_sleep.assert_called_once_with(duration)
+        self.scenarios.node_start_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout, poll_interval
+        )
+        self.mock_affected_nodes_status.merge_affected_nodes.assert_called_once()
+
+    @patch('logging.info')
+    def test_helper_node_stop_start_scenario(self, mock_logging):
+        """Test helper_node_stop_start_scenario calls helper stop and start"""
+        # Arrange
+        instance_kill_count = 1
+        node = "helper-node"
+        timeout = 300
+
+        self.scenarios.helper_node_stop_scenario = Mock()
+        self.scenarios.helper_node_start_scenario = Mock()
+
+        # Act
+        self.scenarios.helper_node_stop_start_scenario(instance_kill_count, node, timeout)
+
+        # Assert
+        self.scenarios.helper_node_stop_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout
+        )
+        self.scenarios.helper_node_start_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout
+        )
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    def test_node_disk_detach_attach_scenario_success(self, mock_logging, mock_sleep):
+        """Test disk detach/attach scenario with valid disk attachment"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+        duration = 60
+        disk_details = {"disk_id": "disk-123", "device": "/dev/sdb"}
+
+        self.scenarios.get_disk_attachment_info = Mock(return_value=disk_details)
+        self.scenarios.disk_detach_scenario = Mock()
+        self.scenarios.disk_attach_scenario = Mock()
+
+        # Act
+        self.scenarios.node_disk_detach_attach_scenario(
+            instance_kill_count, node, timeout, duration
+        )
+
+        # Assert
+        self.scenarios.get_disk_attachment_info.assert_called_once_with(
+            instance_kill_count, node
+        )
+        self.scenarios.disk_detach_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout
+        )
+        mock_sleep.assert_called_once_with(duration)
+        self.scenarios.disk_attach_scenario.assert_called_once_with(
+            instance_kill_count, disk_details, timeout
+        )
+
+    @patch('logging.error')
+    @patch('logging.info')
+    def test_node_disk_detach_attach_scenario_no_disk(self, mock_info, mock_error):
+        """Test disk detach/attach scenario when only root disk exists"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+        duration = 60
+
+        self.scenarios.get_disk_attachment_info = Mock(return_value=None)
+        self.scenarios.disk_detach_scenario = Mock()
+        self.scenarios.disk_attach_scenario = Mock()
+
+        # Act
+        self.scenarios.node_disk_detach_attach_scenario(
+            instance_kill_count, node, timeout, duration
+        )
+
+        # Assert
+        self.scenarios.disk_detach_scenario.assert_not_called()
+        self.scenarios.disk_attach_scenario.assert_not_called()
+        mock_error.assert_any_call("Node %s has only root disk attached" % node)
+
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.nodeaction.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.runcommand.run')
+    @patch('logging.info')
+    def test_stop_kubelet_scenario_success(self, mock_logging, mock_run, mock_wait):
+        """Test successful kubelet stop scenario"""
+        # Arrange
+        instance_kill_count = 2
+        node = "test-node"
+        timeout = 300
+        mock_affected_node = Mock(spec=AffectedNode)
+        mock_wait.return_value = None
+
+        # Act
+        with patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.AffectedNode') as mock_affected_node_class:
+            mock_affected_node_class.return_value = mock_affected_node
+            self.scenarios.stop_kubelet_scenario(instance_kill_count, node, timeout)
+
+        # Assert
+        self.assertEqual(mock_run.call_count, 2)
+        expected_command = "oc debug node/" + node + " -- chroot /host systemctl stop kubelet"
+        mock_run.assert_called_with(expected_command)
+        self.assertEqual(mock_wait.call_count, 2)
+        self.assertEqual(len(self.mock_affected_nodes_status.affected_nodes), 2)
+
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.nodeaction.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.runcommand.run')
+    @patch('logging.error')
+    @patch('logging.info')
+    def test_stop_kubelet_scenario_failure(self, mock_info, mock_error, mock_run, mock_wait):
+        """Test kubelet stop scenario when command fails"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+        error_msg = "Command failed"
+        mock_run.side_effect = Exception(error_msg)
+
+        # Act & Assert
+        with self.assertRaises(Exception):
+            with patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.AffectedNode'):
+                self.scenarios.stop_kubelet_scenario(instance_kill_count, node, timeout)
+
+        mock_error.assert_any_call(
+            "Failed to stop the kubelet of the node. Encountered following "
+            "exception: %s. Test Failed" % error_msg
+        )
+
+    @patch('logging.info')
+    def test_stop_start_kubelet_scenario(self, mock_logging):
+        """Test stop/start kubelet scenario"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+
+        self.scenarios.stop_kubelet_scenario = Mock()
+        self.scenarios.node_reboot_scenario = Mock()
+
+        # Act
+        self.scenarios.stop_start_kubelet_scenario(instance_kill_count, node, timeout)
+
+        # Assert
+        self.scenarios.stop_kubelet_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout
+        )
+        self.scenarios.node_reboot_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout
+        )
+        self.mock_affected_nodes_status.merge_affected_nodes.assert_called_once()
+
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.nodeaction.wait_for_ready_status')
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.runcommand.run')
+    @patch('logging.info')
+    def test_restart_kubelet_scenario_success(self, mock_logging, mock_run, mock_wait):
+        """Test successful kubelet restart scenario"""
+        # Arrange
+        instance_kill_count = 2
+        node = "test-node"
+        timeout = 300
+        mock_affected_node = Mock(spec=AffectedNode)
+        mock_wait.return_value = None
+
+        # Act
+        with patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.AffectedNode') as mock_affected_node_class:
+            mock_affected_node_class.return_value = mock_affected_node
+            self.scenarios.restart_kubelet_scenario(instance_kill_count, node, timeout)
+
+        # Assert
+        self.assertEqual(mock_run.call_count, 2)
+        expected_command = "oc debug node/" + node + " -- chroot /host systemctl restart kubelet &"
+        mock_run.assert_called_with(expected_command)
+        self.assertEqual(mock_wait.call_count, 2)
+        self.assertEqual(len(self.mock_affected_nodes_status.affected_nodes), 2)
+
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.nodeaction.wait_for_ready_status')
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.runcommand.run')
+    @patch('logging.error')
+    @patch('logging.info')
+    def test_restart_kubelet_scenario_failure(self, mock_info, mock_error, mock_run, mock_wait):
+        """Test kubelet restart scenario when command fails"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+        error_msg = "Restart failed"
+        mock_run.side_effect = Exception(error_msg)
+
+        # Act & Assert
+        with self.assertRaises(Exception):
+            with patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.AffectedNode'):
+                self.scenarios.restart_kubelet_scenario(instance_kill_count, node, timeout)
+
+        mock_error.assert_any_call(
+            "Failed to restart the kubelet of the node. Encountered following "
+            "exception: %s. Test Failed" % error_msg
+        )
+
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.runcommand.run')
+    @patch('logging.info')
+    def test_node_crash_scenario_success(self, mock_logging, mock_run):
+        """Test successful node crash scenario"""
+        # Arrange
+        instance_kill_count = 2
+        node = "test-node"
+        timeout = 300
+
+        # Act
+        result = self.scenarios.node_crash_scenario(instance_kill_count, node, timeout)
+
+        # Assert
+        self.assertEqual(mock_run.call_count, 2)
+        expected_command = (
+            "oc debug node/" + node + " -- chroot /host "
+            "dd if=/dev/urandom of=/proc/sysrq-trigger"
+        )
+        mock_run.assert_called_with(expected_command)
+        self.assertIsNone(result)
+
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.runcommand.run')
+    @patch('logging.error')
+    @patch('logging.info')
+    def test_node_crash_scenario_failure(self, mock_info, mock_error, mock_run):
+        """Test node crash scenario when command fails"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+        error_msg = "Crash command failed"
+        mock_run.side_effect = Exception(error_msg)
+
+        # Act
+        result = self.scenarios.node_crash_scenario(instance_kill_count, node, timeout)
+
+        # Assert
+        self.assertEqual(result, 1)
+        mock_error.assert_any_call(
+            "Failed to crash the node. Encountered following exception: %s. "
+            "Test Failed" % error_msg
+        )
+
+    def test_node_start_scenario_not_implemented(self):
+        """Test that node_start_scenario returns None (not implemented)"""
+        result = self.scenarios.node_start_scenario(1, "test-node", 300, 10)
+        self.assertIsNone(result)
+
+    def test_node_stop_scenario_not_implemented(self):
+        """Test that node_stop_scenario returns None (not implemented)"""
+        result = self.scenarios.node_stop_scenario(1, "test-node", 300, 10)
+        self.assertIsNone(result)
+
+    def test_node_termination_scenario_not_implemented(self):
+        """Test that node_termination_scenario returns None (not implemented)"""
+        result = self.scenarios.node_termination_scenario(1, "test-node", 300, 10)
+        self.assertIsNone(result)
+
+    def test_node_reboot_scenario_not_implemented(self):
+        """Test that node_reboot_scenario returns None (not implemented)"""
+        result = self.scenarios.node_reboot_scenario(1, "test-node", 300)
+        self.assertIsNone(result)
+
+    def test_node_service_status_not_implemented(self):
+        """Test that node_service_status returns None (not implemented)"""
+        result = self.scenarios.node_service_status("test-node", "service", "key", 300)
+        self.assertIsNone(result)
+
+    def test_node_block_scenario_not_implemented(self):
+        """Test that node_block_scenario returns None (not implemented)"""
+        result = self.scenarios.node_block_scenario(1, "test-node", 300, 60)
+        self.assertIsNone(result)
+
+
+class TestAbstractNodeScenariosIntegration(unittest.TestCase):
+    """Integration tests for abstract_node_scenarios workflows"""
+
+    def setUp(self):
+        """Set up test fixtures before each test method"""
+        self.mock_kubecli = Mock(spec=KrknKubernetes)
+        self.mock_affected_nodes_status = Mock(spec=AffectedNodeStatus)
+        self.mock_affected_nodes_status.affected_nodes = []
+
+        self.scenarios = abstract_node_scenarios(
+            kubecli=self.mock_kubecli,
+            node_action_kube_check=True,
+            affected_nodes_status=self.mock_affected_nodes_status
+        )
+
+    @patch('time.sleep')
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.nodeaction.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.runcommand.run')
+    def test_complete_stop_start_kubelet_workflow(self, mock_run, mock_wait, mock_sleep):
+        """Test complete workflow of stop/start kubelet scenario"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+
+        self.scenarios.node_reboot_scenario = Mock()
+
+        # Act
+        with patch('krkn.scenario_plugins.node_actions.abstract_node_scenarios.AffectedNode'):
+            self.scenarios.stop_start_kubelet_scenario(instance_kill_count, node, timeout)
+
+        # Assert - verify stop kubelet was called
+        expected_stop_command = "oc debug node/" + node + " -- chroot /host systemctl stop kubelet"
+        mock_run.assert_any_call(expected_stop_command)
+
+        # Verify reboot was called
+        self.scenarios.node_reboot_scenario.assert_called_once_with(
+            instance_kill_count, node, timeout
+        )
+
+        # Verify merge was called
+        self.mock_affected_nodes_status.merge_affected_nodes.assert_called_once()
+
+    @patch('time.sleep')
+    def test_node_stop_start_scenario_workflow(self, mock_sleep):
+        """Test complete workflow of node stop/start scenario"""
+        # Arrange
+        instance_kill_count = 1
+        node = "test-node"
+        timeout = 300
+        duration = 60
+        poll_interval = 10
+
+        self.scenarios.node_stop_scenario = Mock()
+        self.scenarios.node_start_scenario = Mock()
+
+        # Act
+        self.scenarios.node_stop_start_scenario(
+            instance_kill_count, node, timeout, duration, poll_interval
+        )
+
+        # Assert - verify order of operations
+        call_order = []
+
+        # Verify stop was called first
+        self.scenarios.node_stop_scenario.assert_called_once()
+
+        # Verify sleep was called
+        mock_sleep.assert_called_once_with(duration)
+
+        # Verify start was called after sleep
+        self.scenarios.node_start_scenario.assert_called_once()
+
+        # Verify merge was called
+        self.mock_affected_nodes_status.merge_affected_nodes.assert_called_once()
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_alibaba_node_scenarios.py

@@ -0,0 +1,680 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for alibaba_node_scenarios class
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_alibaba_node_scenarios.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+from unittest.mock import MagicMock, Mock, patch, PropertyMock, call
+import logging
+import json
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
+
+from krkn.scenario_plugins.node_actions.alibaba_node_scenarios import Alibaba, alibaba_node_scenarios
+
+
+class TestAlibaba(unittest.TestCase):
+    """Test suite for Alibaba class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Mock environment variables
+        self.env_patcher = patch.dict('os.environ', {
+            'ALIBABA_ID': 'test-access-key',
+            'ALIBABA_SECRET': 'test-secret-key',
+            'ALIBABA_REGION_ID': 'cn-hangzhou'
+        })
+        self.env_patcher.start()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.env_patcher.stop()
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_alibaba_init_success(self, mock_acs_client, mock_logging):
+        """Test Alibaba class initialization"""
+        mock_client = Mock()
+        mock_acs_client.return_value = mock_client
+
+        alibaba = Alibaba()
+
+        mock_acs_client.assert_called_once_with('test-access-key', 'test-secret-key', 'cn-hangzhou')
+        self.assertEqual(alibaba.compute_client, mock_client)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_alibaba_init_failure(self, mock_acs_client, mock_logging):
+        """Test Alibaba initialization handles errors"""
+        mock_acs_client.side_effect = Exception("Credential error")
+
+        alibaba = Alibaba()
+
+        mock_logging.assert_called()
+        self.assertIn("Initializing alibaba", str(mock_logging.call_args))
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_send_request_success(self, mock_acs_client):
+        """Test _send_request successfully sends request"""
+        alibaba = Alibaba()
+
+        mock_request = Mock()
+        mock_response = {'Instances': {'Instance': []}}
+        alibaba.compute_client.do_action.return_value = json.dumps(mock_response).encode('utf-8')
+
+        result = alibaba._send_request(mock_request)
+
+        mock_request.set_accept_format.assert_called_once_with('json')
+        alibaba.compute_client.do_action.assert_called_once_with(mock_request)
+        self.assertEqual(result, mock_response)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_send_request_failure(self, mock_acs_client, mock_logging):
+        """Test _send_request handles errors"""
+        alibaba = Alibaba()
+
+        mock_request = Mock()
+        alibaba.compute_client.do_action.side_effect = Exception("API error")
+
+        # The actual code has a bug in the format string (%S instead of %s)
+        # So we expect this to raise a ValueError
+        with self.assertRaises(ValueError):
+            alibaba._send_request(mock_request)
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_list_instances_success(self, mock_acs_client):
+        """Test list_instances returns instance list"""
+        alibaba = Alibaba()
+
+        mock_instances = [
+            {'InstanceId': 'i-123', 'InstanceName': 'node1'},
+            {'InstanceId': 'i-456', 'InstanceName': 'node2'}
+        ]
+        mock_response = {'Instances': {'Instance': mock_instances}}
+        alibaba.compute_client.do_action.return_value = json.dumps(mock_response).encode('utf-8')
+
+        result = alibaba.list_instances()
+
+        self.assertEqual(result, mock_instances)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_list_instances_no_instances_key(self, mock_acs_client, mock_logging):
+        """Test list_instances handles missing Instances key"""
+        alibaba = Alibaba()
+
+        mock_response = {'SomeOtherKey': 'value'}
+        alibaba.compute_client.do_action.return_value = json.dumps(mock_response).encode('utf-8')
+
+        with self.assertRaises(RuntimeError):
+            alibaba.list_instances()
+
+        mock_logging.assert_called()
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_list_instances_none_response(self, mock_acs_client):
+        """Test list_instances handles None response"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(return_value=None)
+
+        result = alibaba.list_instances()
+
+        self.assertEqual(result, [])
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_list_instances_exception(self, mock_acs_client, mock_logging):
+        """Test list_instances handles exceptions"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(side_effect=Exception("Network error"))
+
+        with self.assertRaises(Exception):
+            alibaba.list_instances()
+
+        mock_logging.assert_called()
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_get_instance_id_found(self, mock_acs_client):
+        """Test get_instance_id when instance is found"""
+        alibaba = Alibaba()
+
+        mock_instances = [
+            {'InstanceId': 'i-123', 'InstanceName': 'test-node'},
+            {'InstanceId': 'i-456', 'InstanceName': 'other-node'}
+        ]
+        alibaba.list_instances = Mock(return_value=mock_instances)
+
+        result = alibaba.get_instance_id('test-node')
+
+        self.assertEqual(result, 'i-123')
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_get_instance_id_not_found(self, mock_acs_client, mock_logging):
+        """Test get_instance_id when instance is not found"""
+        alibaba = Alibaba()
+
+        alibaba.list_instances = Mock(return_value=[])
+
+        with self.assertRaises(RuntimeError):
+            alibaba.get_instance_id('nonexistent-node')
+
+        mock_logging.assert_called()
+        self.assertIn("Couldn't find vm", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_start_instances_success(self, mock_acs_client, mock_logging):
+        """Test start_instances successfully starts instance"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(return_value={'RequestId': 'req-123'})
+
+        alibaba.start_instances('i-123')
+
+        alibaba._send_request.assert_called_once()
+        mock_logging.assert_called()
+        call_str = str(mock_logging.call_args_list)
+        self.assertTrue('started' in call_str or 'submit successfully' in call_str)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_start_instances_failure(self, mock_acs_client, mock_logging):
+        """Test start_instances handles failure"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(side_effect=Exception("Start failed"))
+
+        with self.assertRaises(Exception):
+            alibaba.start_instances('i-123')
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to start", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_stop_instances_success(self, mock_acs_client, mock_logging):
+        """Test stop_instances successfully stops instance"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(return_value={'RequestId': 'req-123'})
+
+        alibaba.stop_instances('i-123', force_stop=True)
+
+        alibaba._send_request.assert_called_once()
+        mock_logging.assert_called()
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_stop_instances_failure(self, mock_acs_client, mock_logging):
+        """Test stop_instances handles failure"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(side_effect=Exception("Stop failed"))
+
+        with self.assertRaises(Exception):
+            alibaba.stop_instances('i-123')
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to stop", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_release_instance_success(self, mock_acs_client, mock_logging):
+        """Test release_instance successfully releases instance"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(return_value={'RequestId': 'req-123'})
+
+        alibaba.release_instance('i-123', force_release=True)
+
+        alibaba._send_request.assert_called_once()
+        mock_logging.assert_called()
+        self.assertIn("released", str(mock_logging.call_args))
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_release_instance_failure(self, mock_acs_client, mock_logging):
+        """Test release_instance handles failure"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(side_effect=Exception("Release failed"))
+
+        with self.assertRaises(Exception):
+            alibaba.release_instance('i-123')
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to terminate", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_reboot_instances_success(self, mock_acs_client, mock_logging):
+        """Test reboot_instances successfully reboots instance"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(return_value={'RequestId': 'req-123'})
+
+        alibaba.reboot_instances('i-123', force_reboot=True)
+
+        alibaba._send_request.assert_called_once()
+        mock_logging.assert_called()
+        self.assertIn("rebooted", str(mock_logging.call_args))
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_reboot_instances_failure(self, mock_acs_client, mock_logging):
+        """Test reboot_instances handles failure"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(side_effect=Exception("Reboot failed"))
+
+        with self.assertRaises(Exception):
+            alibaba.reboot_instances('i-123')
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to reboot", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_get_vm_status_success(self, mock_acs_client, mock_logging):
+        """Test get_vm_status returns instance status"""
+        alibaba = Alibaba()
+
+        mock_response = {
+            'Instances': {
+                'Instance': [{'Status': 'Running'}]
+            }
+        }
+        alibaba._send_request = Mock(return_value=mock_response)
+
+        result = alibaba.get_vm_status('i-123')
+
+        self.assertEqual(result, 'Running')
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_get_vm_status_no_instances(self, mock_acs_client, mock_logging):
+        """Test get_vm_status when no instances found"""
+        alibaba = Alibaba()
+
+        mock_response = {
+            'Instances': {
+                'Instance': []
+            }
+        }
+        alibaba._send_request = Mock(return_value=mock_response)
+
+        result = alibaba.get_vm_status('i-123')
+
+        self.assertIsNone(result)
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_get_vm_status_none_response(self, mock_acs_client, mock_logging):
+        """Test get_vm_status with None response"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(return_value=None)
+
+        result = alibaba.get_vm_status('i-123')
+
+        self.assertEqual(result, 'Unknown')
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_get_vm_status_exception(self, mock_acs_client, mock_logging):
+        """Test get_vm_status handles exceptions"""
+        alibaba = Alibaba()
+        alibaba._send_request = Mock(side_effect=Exception("API error"))
+
+        result = alibaba.get_vm_status('i-123')
+
+        self.assertIsNone(result)
+        mock_logging.assert_called()
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_wait_until_running_success(self, mock_acs_client, mock_logging, mock_sleep):
+        """Test wait_until_running waits for instance to be running"""
+        alibaba = Alibaba()
+
+        alibaba.get_vm_status = Mock(side_effect=['Starting', 'Running'])
+        mock_affected_node = Mock(spec=AffectedNode)
+
+        result = alibaba.wait_until_running('i-123', 300, mock_affected_node)
+
+        self.assertTrue(result)
+        mock_affected_node.set_affected_node_status.assert_called_once()
+        args = mock_affected_node.set_affected_node_status.call_args[0]
+        self.assertEqual(args[0], 'running')
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_wait_until_running_timeout(self, mock_acs_client, mock_logging, mock_sleep):
+        """Test wait_until_running returns False on timeout"""
+        alibaba = Alibaba()
+
+        alibaba.get_vm_status = Mock(return_value='Starting')
+
+        result = alibaba.wait_until_running('i-123', 10, None)
+
+        self.assertFalse(result)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_wait_until_stopped_success(self, mock_acs_client, mock_logging, mock_sleep):
+        """Test wait_until_stopped waits for instance to be stopped"""
+        alibaba = Alibaba()
+
+        alibaba.get_vm_status = Mock(side_effect=['Stopping', 'Stopped'])
+        mock_affected_node = Mock(spec=AffectedNode)
+
+        result = alibaba.wait_until_stopped('i-123', 300, mock_affected_node)
+
+        self.assertTrue(result)
+        mock_affected_node.set_affected_node_status.assert_called_once()
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_wait_until_stopped_timeout(self, mock_acs_client, mock_logging, mock_sleep):
+        """Test wait_until_stopped returns False on timeout"""
+        alibaba = Alibaba()
+
+        alibaba.get_vm_status = Mock(return_value='Stopping')
+
+        result = alibaba.wait_until_stopped('i-123', 10, None)
+
+        self.assertFalse(result)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_wait_until_released_success(self, mock_acs_client, mock_logging, mock_sleep):
+        """Test wait_until_released waits for instance to be released"""
+        alibaba = Alibaba()
+
+        alibaba.get_vm_status = Mock(side_effect=['Deleting', 'Released'])
+        mock_affected_node = Mock(spec=AffectedNode)
+
+        result = alibaba.wait_until_released('i-123', 300, mock_affected_node)
+
+        self.assertTrue(result)
+        mock_affected_node.set_affected_node_status.assert_called_once()
+        args = mock_affected_node.set_affected_node_status.call_args[0]
+        self.assertEqual(args[0], 'terminated')
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_wait_until_released_timeout(self, mock_acs_client, mock_logging, mock_sleep):
+        """Test wait_until_released returns False on timeout"""
+        alibaba = Alibaba()
+
+        alibaba.get_vm_status = Mock(return_value='Deleting')
+
+        result = alibaba.wait_until_released('i-123', 10, None)
+
+        self.assertFalse(result)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.AcsClient')
+    def test_wait_until_released_none_status(self, mock_acs_client, mock_logging, mock_sleep):
+        """Test wait_until_released when status becomes None"""
+        alibaba = Alibaba()
+
+        alibaba.get_vm_status = Mock(side_effect=['Deleting', None])
+        mock_affected_node = Mock(spec=AffectedNode)
+
+        result = alibaba.wait_until_released('i-123', 300, mock_affected_node)
+
+        self.assertTrue(result)
+
+
+class TestAlibabaNodeScenarios(unittest.TestCase):
+    """Test suite for alibaba_node_scenarios class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        self.env_patcher = patch.dict('os.environ', {
+            'ALIBABA_ID': 'test-access-key',
+            'ALIBABA_SECRET': 'test-secret-key',
+            'ALIBABA_REGION_ID': 'cn-hangzhou'
+        })
+        self.env_patcher.start()
+
+        self.mock_kubecli = Mock(spec=KrknKubernetes)
+        self.affected_nodes_status = AffectedNodeStatus()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.env_patcher.stop()
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_init(self, mock_alibaba_class, mock_logging):
+        """Test alibaba_node_scenarios initialization"""
+        mock_alibaba_instance = Mock()
+        mock_alibaba_class.return_value = mock_alibaba_instance
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        self.assertEqual(scenarios.kubecli, self.mock_kubecli)
+        self.assertTrue(scenarios.node_action_kube_check)
+        self.assertEqual(scenarios.alibaba, mock_alibaba_instance)
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_start_scenario_success(self, mock_alibaba_class, mock_logging, mock_nodeaction):
+        """Test node_start_scenario successfully starts node"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.wait_until_running.return_value = True
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_start_scenario(1, 'test-node', 300, 15)
+
+        mock_alibaba.get_instance_id.assert_called_once_with('test-node')
+        mock_alibaba.start_instances.assert_called_once_with('i-123')
+        mock_alibaba.wait_until_running.assert_called_once()
+        mock_nodeaction.wait_for_ready_status.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_start_scenario_no_kube_check(self, mock_alibaba_class, mock_logging, mock_nodeaction):
+        """Test node_start_scenario without Kubernetes check"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.wait_until_running.return_value = True
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        scenarios.node_start_scenario(1, 'test-node', 300, 15)
+
+        mock_alibaba.start_instances.assert_called_once()
+        mock_nodeaction.wait_for_ready_status.assert_not_called()
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_start_scenario_failure(self, mock_alibaba_class, mock_logging):
+        """Test node_start_scenario handles failure"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.start_instances.side_effect = Exception('Start failed')
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(Exception):
+            scenarios.node_start_scenario(1, 'test-node', 300, 15)
+
+        mock_logging.assert_called()
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_start_scenario_multiple_runs(self, mock_alibaba_class, mock_logging, mock_nodeaction):
+        """Test node_start_scenario with multiple runs"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.wait_until_running.return_value = True
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_start_scenario(3, 'test-node', 300, 15)
+
+        self.assertEqual(mock_alibaba.start_instances.call_count, 3)
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 3)
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_stop_scenario_success(self, mock_alibaba_class, mock_logging, mock_nodeaction):
+        """Test node_stop_scenario successfully stops node"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.wait_until_stopped.return_value = True
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_stop_scenario(1, 'test-node', 300, 15)
+
+        mock_alibaba.get_instance_id.assert_called_once_with('test-node')
+        mock_alibaba.stop_instances.assert_called_once_with('i-123')
+        mock_alibaba.wait_until_stopped.assert_called_once()
+        mock_nodeaction.wait_for_unknown_status.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_stop_scenario_failure(self, mock_alibaba_class, mock_logging):
+        """Test node_stop_scenario handles failure"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.stop_instances.side_effect = Exception('Stop failed')
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(Exception):
+            scenarios.node_stop_scenario(1, 'test-node', 300, 15)
+
+        mock_logging.assert_called()
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_termination_scenario_success(self, mock_alibaba_class, mock_logging):
+        """Test node_termination_scenario successfully terminates node"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.wait_until_stopped.return_value = True
+        mock_alibaba.wait_until_released.return_value = True
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        scenarios.node_termination_scenario(1, 'test-node', 300, 15)
+
+        mock_alibaba.stop_instances.assert_called_once_with('i-123')
+        mock_alibaba.wait_until_stopped.assert_called_once()
+        mock_alibaba.release_instance.assert_called_once_with('i-123')
+        mock_alibaba.wait_until_released.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_termination_scenario_failure(self, mock_alibaba_class, mock_logging):
+        """Test node_termination_scenario handles failure"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.stop_instances.side_effect = Exception('Stop failed')
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(Exception):
+            scenarios.node_termination_scenario(1, 'test-node', 300, 15)
+
+        mock_logging.assert_called()
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_reboot_scenario_success(self, mock_alibaba_class, mock_logging, mock_nodeaction):
+        """Test node_reboot_scenario successfully reboots node"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_reboot_scenario(1, 'test-node', 300, soft_reboot=False)
+
+        mock_alibaba.reboot_instances.assert_called_once_with('i-123')
+        mock_nodeaction.wait_for_unknown_status.assert_called_once()
+        mock_nodeaction.wait_for_ready_status.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_reboot_scenario_no_kube_check(self, mock_alibaba_class, mock_logging, mock_nodeaction):
+        """Test node_reboot_scenario without Kubernetes check"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        scenarios.node_reboot_scenario(1, 'test-node', 300)
+
+        mock_alibaba.reboot_instances.assert_called_once()
+        mock_nodeaction.wait_for_unknown_status.assert_not_called()
+        mock_nodeaction.wait_for_ready_status.assert_not_called()
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_reboot_scenario_failure(self, mock_alibaba_class, mock_logging):
+        """Test node_reboot_scenario handles failure"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+        mock_alibaba.reboot_instances.side_effect = Exception('Reboot failed')
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(Exception):
+            scenarios.node_reboot_scenario(1, 'test-node', 300)
+
+        mock_logging.assert_called()
+
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.alibaba_node_scenarios.Alibaba')
+    def test_node_reboot_scenario_multiple_runs(self, mock_alibaba_class, mock_logging, mock_nodeaction):
+        """Test node_reboot_scenario with multiple runs"""
+        mock_alibaba = Mock()
+        mock_alibaba_class.return_value = mock_alibaba
+        mock_alibaba.get_instance_id.return_value = 'i-123'
+
+        scenarios = alibaba_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_reboot_scenario(2, 'test-node', 300)
+
+        self.assertEqual(mock_alibaba.reboot_instances.call_count, 2)
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_application_outage_scenario_plugin.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for ApplicationOutageScenarioPlugin class
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_application_outage_scenario_plugin.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+from unittest.mock import MagicMock
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
+
+from krkn.scenario_plugins.application_outage.application_outage_scenario_plugin import ApplicationOutageScenarioPlugin
+
+
+class TestApplicationOutageScenarioPlugin(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Set up test fixtures for ApplicationOutageScenarioPlugin
+        """
+        self.plugin = ApplicationOutageScenarioPlugin()
+
+    def test_get_scenario_types(self):
+        """
+        Test get_scenario_types returns correct scenario type
+        """
+        result = self.plugin.get_scenario_types()
+
+        self.assertEqual(result, ["application_outages_scenarios"])
+        self.assertEqual(len(result), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_aws_node_scenarios.py

@@ -0,0 +1,984 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for AWS node scenarios
+
+This test suite covers both the AWS class and aws_node_scenarios class
+using mocks to avoid actual AWS API calls.
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_aws_node_scenarios.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+import sys
+from unittest.mock import MagicMock, patch
+
+# Mock external dependencies before any imports that use them
+sys.modules['boto3'] = MagicMock()
+sys.modules['paramiko'] = MagicMock()
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
+from krkn.scenario_plugins.node_actions.aws_node_scenarios import AWS, aws_node_scenarios
+
+
+class TestAWS(unittest.TestCase):
+    """Test cases for AWS class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Mock boto3 to avoid actual AWS calls
+        self.boto_client_patcher = patch('boto3.client')
+        self.boto_resource_patcher = patch('boto3.resource')
+
+        self.mock_client = self.boto_client_patcher.start()
+        self.mock_resource = self.boto_resource_patcher.start()
+
+        # Create AWS instance with mocked boto3
+        self.aws = AWS()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.boto_client_patcher.stop()
+        self.boto_resource_patcher.stop()
+
+    def test_aws_init(self):
+        """Test AWS class initialization"""
+        self.assertIsNotNone(self.aws.boto_client)
+        self.assertIsNotNone(self.aws.boto_resource)
+        self.assertIsNotNone(self.aws.boto_instance)
+
+    def test_get_instance_id_by_dns_name(self):
+        """Test getting instance ID by DNS name"""
+        mock_response = {
+            'Reservations': [{
+                'Instances': [{
+                    'InstanceId': 'i-1234567890abcdef0'
+                }]
+            }]
+        }
+        self.aws.boto_client.describe_instances = MagicMock(return_value=mock_response)
+
+        instance_id = self.aws.get_instance_id('ip-10-0-1-100.ec2.internal')
+
+        self.assertEqual(instance_id, 'i-1234567890abcdef0')
+        self.aws.boto_client.describe_instances.assert_called_once()
+
+    def test_get_instance_id_by_ip_address(self):
+        """Test getting instance ID by IP address when DNS name fails"""
+        # First call returns empty, second call returns the instance
+        mock_response_empty = {'Reservations': []}
+        mock_response_with_instance = {
+            'Reservations': [{
+                'Instances': [{
+                    'InstanceId': 'i-1234567890abcdef0'
+                }]
+            }]
+        }
+        self.aws.boto_client.describe_instances = MagicMock(
+            side_effect=[mock_response_empty, mock_response_with_instance]
+        )
+
+        instance_id = self.aws.get_instance_id('ip-10-0-1-100')
+
+        self.assertEqual(instance_id, 'i-1234567890abcdef0')
+        self.assertEqual(self.aws.boto_client.describe_instances.call_count, 2)
+
+    def test_start_instances_success(self):
+        """Test starting instances successfully"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.start_instances = MagicMock()
+
+        self.aws.start_instances(instance_id)
+
+        self.aws.boto_client.start_instances.assert_called_once_with(
+            InstanceIds=[instance_id]
+        )
+
+    def test_start_instances_failure(self):
+        """Test starting instances with failure"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.start_instances = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.start_instances(instance_id)
+
+    def test_stop_instances_success(self):
+        """Test stopping instances successfully"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.stop_instances = MagicMock()
+
+        self.aws.stop_instances(instance_id)
+
+        self.aws.boto_client.stop_instances.assert_called_once_with(
+            InstanceIds=[instance_id]
+        )
+
+    def test_stop_instances_failure(self):
+        """Test stopping instances with failure"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.stop_instances = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.stop_instances(instance_id)
+
+    def test_terminate_instances_success(self):
+        """Test terminating instances successfully"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.terminate_instances = MagicMock()
+
+        self.aws.terminate_instances(instance_id)
+
+        self.aws.boto_client.terminate_instances.assert_called_once_with(
+            InstanceIds=[instance_id]
+        )
+
+    def test_terminate_instances_failure(self):
+        """Test terminating instances with failure"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.terminate_instances = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.terminate_instances(instance_id)
+
+    def test_reboot_instances_success(self):
+        """Test rebooting instances successfully"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.reboot_instances = MagicMock()
+
+        self.aws.reboot_instances(instance_id)
+
+        self.aws.boto_client.reboot_instances.assert_called_once_with(
+            InstanceIds=[instance_id]
+        )
+
+    def test_reboot_instances_failure(self):
+        """Test rebooting instances with failure"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_client.reboot_instances = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.reboot_instances(instance_id)
+
+    def test_wait_until_running_success(self):
+        """Test waiting until instance is running successfully"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_instance.wait_until_running = MagicMock()
+
+        result = self.aws.wait_until_running(instance_id, timeout=600, poll_interval=15)
+
+        self.assertTrue(result)
+        self.aws.boto_instance.wait_until_running.assert_called_once()
+
+    def test_wait_until_running_with_affected_node(self):
+        """Test waiting until running with affected node tracking"""
+        instance_id = 'i-1234567890abcdef0'
+        affected_node = MagicMock(spec=AffectedNode)
+        self.aws.boto_instance.wait_until_running = MagicMock()
+
+        with patch('time.time', side_effect=[100, 110]):
+            result = self.aws.wait_until_running(
+                instance_id,
+                timeout=600,
+                affected_node=affected_node,
+                poll_interval=15
+            )
+
+        self.assertTrue(result)
+        affected_node.set_affected_node_status.assert_called_once_with("running", 10)
+
+    def test_wait_until_running_failure(self):
+        """Test waiting until running with failure"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_instance.wait_until_running = MagicMock(
+            side_effect=Exception("Timeout")
+        )
+
+        result = self.aws.wait_until_running(instance_id)
+
+        self.assertFalse(result)
+
+    def test_wait_until_stopped_success(self):
+        """Test waiting until instance is stopped successfully"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_instance.wait_until_stopped = MagicMock()
+
+        result = self.aws.wait_until_stopped(instance_id, timeout=600, poll_interval=15)
+
+        self.assertTrue(result)
+        self.aws.boto_instance.wait_until_stopped.assert_called_once()
+
+    def test_wait_until_stopped_with_affected_node(self):
+        """Test waiting until stopped with affected node tracking"""
+        instance_id = 'i-1234567890abcdef0'
+        affected_node = MagicMock(spec=AffectedNode)
+        self.aws.boto_instance.wait_until_stopped = MagicMock()
+
+        with patch('time.time', side_effect=[100, 115]):
+            result = self.aws.wait_until_stopped(
+                instance_id,
+                timeout=600,
+                affected_node=affected_node,
+                poll_interval=15
+            )
+
+        self.assertTrue(result)
+        affected_node.set_affected_node_status.assert_called_once_with("stopped", 15)
+
+    def test_wait_until_stopped_failure(self):
+        """Test waiting until stopped with failure"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_instance.wait_until_stopped = MagicMock(
+            side_effect=Exception("Timeout")
+        )
+
+        result = self.aws.wait_until_stopped(instance_id)
+
+        self.assertFalse(result)
+
+    def test_wait_until_terminated_success(self):
+        """Test waiting until instance is terminated successfully"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_instance.wait_until_terminated = MagicMock()
+
+        result = self.aws.wait_until_terminated(instance_id, timeout=600, poll_interval=15)
+
+        self.assertTrue(result)
+        self.aws.boto_instance.wait_until_terminated.assert_called_once()
+
+    def test_wait_until_terminated_with_affected_node(self):
+        """Test waiting until terminated with affected node tracking"""
+        instance_id = 'i-1234567890abcdef0'
+        affected_node = MagicMock(spec=AffectedNode)
+        self.aws.boto_instance.wait_until_terminated = MagicMock()
+
+        with patch('time.time', side_effect=[100, 120]):
+            result = self.aws.wait_until_terminated(
+                instance_id,
+                timeout=600,
+                affected_node=affected_node,
+                poll_interval=15
+            )
+
+        self.assertTrue(result)
+        affected_node.set_affected_node_status.assert_called_once_with("terminated", 20)
+
+    def test_wait_until_terminated_failure(self):
+        """Test waiting until terminated with failure"""
+        instance_id = 'i-1234567890abcdef0'
+        self.aws.boto_instance.wait_until_terminated = MagicMock(
+            side_effect=Exception("Timeout")
+        )
+
+        result = self.aws.wait_until_terminated(instance_id)
+
+        self.assertFalse(result)
+
+    def test_create_default_network_acl_success(self):
+        """Test creating default network ACL successfully"""
+        vpc_id = 'vpc-12345678'
+        acl_id = 'acl-12345678'
+        mock_response = {
+            'NetworkAcl': {
+                'NetworkAclId': acl_id
+            }
+        }
+        self.aws.boto_client.create_network_acl = MagicMock(return_value=mock_response)
+
+        result = self.aws.create_default_network_acl(vpc_id)
+
+        self.assertEqual(result, acl_id)
+        self.aws.boto_client.create_network_acl.assert_called_once_with(VpcId=vpc_id)
+
+    def test_create_default_network_acl_failure(self):
+        """Test creating default network ACL with failure"""
+        vpc_id = 'vpc-12345678'
+        self.aws.boto_client.create_network_acl = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.create_default_network_acl(vpc_id)
+
+    def test_replace_network_acl_association_success(self):
+        """Test replacing network ACL association successfully"""
+        association_id = 'aclassoc-12345678'
+        acl_id = 'acl-12345678'
+        new_association_id = 'aclassoc-87654321'
+        mock_response = {
+            'NewAssociationId': new_association_id
+        }
+        self.aws.boto_client.replace_network_acl_association = MagicMock(
+            return_value=mock_response
+        )
+
+        result = self.aws.replace_network_acl_association(association_id, acl_id)
+
+        self.assertEqual(result, new_association_id)
+        self.aws.boto_client.replace_network_acl_association.assert_called_once_with(
+            AssociationId=association_id, NetworkAclId=acl_id
+        )
+
+    def test_replace_network_acl_association_failure(self):
+        """Test replacing network ACL association with failure"""
+        association_id = 'aclassoc-12345678'
+        acl_id = 'acl-12345678'
+        self.aws.boto_client.replace_network_acl_association = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.replace_network_acl_association(association_id, acl_id)
+
+    def test_describe_network_acls_success(self):
+        """Test describing network ACLs successfully"""
+        vpc_id = 'vpc-12345678'
+        subnet_id = 'subnet-12345678'
+        acl_id = 'acl-12345678'
+        associations = [{'NetworkAclId': acl_id, 'SubnetId': subnet_id}]
+        mock_response = {
+            'NetworkAcls': [{
+                'Associations': associations
+            }]
+        }
+        self.aws.boto_client.describe_network_acls = MagicMock(return_value=mock_response)
+
+        result_associations, result_acl_id = self.aws.describe_network_acls(vpc_id, subnet_id)
+
+        self.assertEqual(result_associations, associations)
+        self.assertEqual(result_acl_id, acl_id)
+
+    def test_describe_network_acls_failure(self):
+        """Test describing network ACLs with failure"""
+        vpc_id = 'vpc-12345678'
+        subnet_id = 'subnet-12345678'
+        self.aws.boto_client.describe_network_acls = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.describe_network_acls(vpc_id, subnet_id)
+
+    def test_delete_network_acl_success(self):
+        """Test deleting network ACL successfully"""
+        acl_id = 'acl-12345678'
+        self.aws.boto_client.delete_network_acl = MagicMock()
+
+        self.aws.delete_network_acl(acl_id)
+
+        self.aws.boto_client.delete_network_acl.assert_called_once_with(NetworkAclId=acl_id)
+
+    def test_delete_network_acl_failure(self):
+        """Test deleting network ACL with failure"""
+        acl_id = 'acl-12345678'
+        self.aws.boto_client.delete_network_acl = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.delete_network_acl(acl_id)
+
+    def test_detach_volumes_success(self):
+        """Test detaching volumes successfully"""
+        volume_ids = ['vol-12345678', 'vol-87654321']
+        self.aws.boto_client.detach_volume = MagicMock()
+
+        self.aws.detach_volumes(volume_ids)
+
+        self.assertEqual(self.aws.boto_client.detach_volume.call_count, 2)
+        self.aws.boto_client.detach_volume.assert_any_call(VolumeId='vol-12345678', Force=True)
+        self.aws.boto_client.detach_volume.assert_any_call(VolumeId='vol-87654321', Force=True)
+
+    def test_detach_volumes_partial_failure(self):
+        """Test detaching volumes with partial failure"""
+        volume_ids = ['vol-12345678', 'vol-87654321']
+        # First call succeeds, second fails - should not raise exception
+        self.aws.boto_client.detach_volume = MagicMock(
+            side_effect=[None, Exception("AWS error")]
+        )
+
+        # Should not raise exception, just log error
+        self.aws.detach_volumes(volume_ids)
+
+        self.assertEqual(self.aws.boto_client.detach_volume.call_count, 2)
+
+    def test_attach_volume_success(self):
+        """Test attaching volume successfully"""
+        attachment = {
+            'VolumeId': 'vol-12345678',
+            'InstanceId': 'i-1234567890abcdef0',
+            'Device': '/dev/sdf'
+        }
+        mock_volume = MagicMock()
+        mock_volume.state = 'available'
+        self.aws.boto_resource.Volume = MagicMock(return_value=mock_volume)
+        self.aws.boto_client.attach_volume = MagicMock()
+
+        self.aws.attach_volume(attachment)
+
+        self.aws.boto_client.attach_volume.assert_called_once_with(
+            InstanceId=attachment['InstanceId'],
+            Device=attachment['Device'],
+            VolumeId=attachment['VolumeId']
+        )
+
+    def test_attach_volume_already_in_use(self):
+        """Test attaching volume that is already in use"""
+        attachment = {
+            'VolumeId': 'vol-12345678',
+            'InstanceId': 'i-1234567890abcdef0',
+            'Device': '/dev/sdf'
+        }
+        mock_volume = MagicMock()
+        mock_volume.state = 'in-use'
+        self.aws.boto_resource.Volume = MagicMock(return_value=mock_volume)
+        self.aws.boto_client.attach_volume = MagicMock()
+
+        self.aws.attach_volume(attachment)
+
+        # Should not attempt to attach
+        self.aws.boto_client.attach_volume.assert_not_called()
+
+    def test_attach_volume_failure(self):
+        """Test attaching volume with failure"""
+        attachment = {
+            'VolumeId': 'vol-12345678',
+            'InstanceId': 'i-1234567890abcdef0',
+            'Device': '/dev/sdf'
+        }
+        mock_volume = MagicMock()
+        mock_volume.state = 'available'
+        self.aws.boto_resource.Volume = MagicMock(return_value=mock_volume)
+        self.aws.boto_client.attach_volume = MagicMock(
+            side_effect=Exception("AWS error")
+        )
+
+        with self.assertRaises(RuntimeError):
+            self.aws.attach_volume(attachment)
+
+    def test_get_volumes_ids(self):
+        """Test getting volume IDs from instance"""
+        instance_id = ['i-1234567890abcdef0']
+        mock_response = {
+            'Reservations': [{
+                'Instances': [{
+                    'BlockDeviceMappings': [
+                        {'DeviceName': '/dev/sda1', 'Ebs': {'VolumeId': 'vol-root'}},
+                        {'DeviceName': '/dev/sdf', 'Ebs': {'VolumeId': 'vol-12345678'}},
+                        {'DeviceName': '/dev/sdg', 'Ebs': {'VolumeId': 'vol-87654321'}}
+                    ]
+                }]
+            }]
+        }
+        mock_instance = MagicMock()
+        mock_instance.root_device_name = '/dev/sda1'
+        self.aws.boto_resource.Instance = MagicMock(return_value=mock_instance)
+        self.aws.boto_client.describe_instances = MagicMock(return_value=mock_response)
+
+        volume_ids = self.aws.get_volumes_ids(instance_id)
+
+        self.assertEqual(len(volume_ids), 2)
+        self.assertIn('vol-12345678', volume_ids)
+        self.assertIn('vol-87654321', volume_ids)
+        self.assertNotIn('vol-root', volume_ids)
+
+    def test_get_volume_attachment_details(self):
+        """Test getting volume attachment details"""
+        volume_ids = ['vol-12345678', 'vol-87654321']
+        mock_response = {
+            'Volumes': [
+                {'VolumeId': 'vol-12345678', 'State': 'in-use'},
+                {'VolumeId': 'vol-87654321', 'State': 'available'}
+            ]
+        }
+        self.aws.boto_client.describe_volumes = MagicMock(return_value=mock_response)
+
+        details = self.aws.get_volume_attachment_details(volume_ids)
+
+        self.assertEqual(len(details), 2)
+        self.assertEqual(details[0]['VolumeId'], 'vol-12345678')
+        self.assertEqual(details[1]['VolumeId'], 'vol-87654321')
+
+    def test_get_root_volume_id(self):
+        """Test getting root volume ID"""
+        instance_id = ['i-1234567890abcdef0']
+        mock_instance = MagicMock()
+        mock_instance.root_device_name = '/dev/sda1'
+        self.aws.boto_resource.Instance = MagicMock(return_value=mock_instance)
+
+        root_volume = self.aws.get_root_volume_id(instance_id)
+
+        self.assertEqual(root_volume, '/dev/sda1')
+
+    def test_get_volume_state(self):
+        """Test getting volume state"""
+        volume_id = 'vol-12345678'
+        mock_volume = MagicMock()
+        mock_volume.state = 'available'
+        self.aws.boto_resource.Volume = MagicMock(return_value=mock_volume)
+
+        state = self.aws.get_volume_state(volume_id)
+
+        self.assertEqual(state, 'available')
+
+
+class TestAWSNodeScenarios(unittest.TestCase):
+    """Test cases for aws_node_scenarios class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        self.kubecli = MagicMock(spec=KrknKubernetes)
+        self.affected_nodes_status = AffectedNodeStatus()
+
+        # Mock the AWS class
+        with patch('krkn.scenario_plugins.node_actions.aws_node_scenarios.AWS') as mock_aws_class:
+            self.mock_aws = MagicMock()
+            mock_aws_class.return_value = self.mock_aws
+            self.scenario = aws_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=True,
+                affected_nodes_status=self.affected_nodes_status
+            )
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_success(self, mock_wait_ready):
+        """Test node start scenario successfully"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.start_instances.return_value = None
+        self.mock_aws.wait_until_running.return_value = True
+
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600,
+            poll_interval=15
+        )
+
+        self.mock_aws.get_instance_id.assert_called_once_with(node)
+        self.mock_aws.start_instances.assert_called_once_with(instance_id)
+        self.mock_aws.wait_until_running.assert_called_once()
+        mock_wait_ready.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+        self.assertEqual(self.affected_nodes_status.affected_nodes[0].node_name, node)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_no_kube_check(self, mock_wait_ready):
+        """Test node start scenario without kube check"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        # Create scenario with node_action_kube_check=False
+        with patch('krkn.scenario_plugins.node_actions.aws_node_scenarios.AWS') as mock_aws_class:
+            mock_aws = MagicMock()
+            mock_aws_class.return_value = mock_aws
+            scenario = aws_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=False,
+                affected_nodes_status=AffectedNodeStatus()
+            )
+
+            mock_aws.get_instance_id.return_value = instance_id
+            mock_aws.start_instances.return_value = None
+            mock_aws.wait_until_running.return_value = True
+
+            scenario.node_start_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+            # Should not call wait_for_ready_status
+            mock_wait_ready.assert_not_called()
+
+    def test_node_start_scenario_failure(self):
+        """Test node start scenario with failure"""
+        node = 'ip-10-0-1-100.ec2.internal'
+
+        self.mock_aws.get_instance_id.side_effect = Exception("AWS error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_start_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    def test_node_stop_scenario_success(self, mock_wait_unknown):
+        """Test node stop scenario successfully"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.stop_instances.return_value = None
+        self.mock_aws.wait_until_stopped.return_value = True
+
+        self.scenario.node_stop_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600,
+            poll_interval=15
+        )
+
+        self.mock_aws.get_instance_id.assert_called_once_with(node)
+        self.mock_aws.stop_instances.assert_called_once_with(instance_id)
+        self.mock_aws.wait_until_stopped.assert_called_once()
+        mock_wait_unknown.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    def test_node_stop_scenario_no_kube_check(self, mock_wait_unknown):
+        """Test node stop scenario without kube check"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        # Create scenario with node_action_kube_check=False
+        with patch('krkn.scenario_plugins.node_actions.aws_node_scenarios.AWS') as mock_aws_class:
+            mock_aws = MagicMock()
+            mock_aws_class.return_value = mock_aws
+            scenario = aws_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=False,
+                affected_nodes_status=AffectedNodeStatus()
+            )
+
+            mock_aws.get_instance_id.return_value = instance_id
+            mock_aws.stop_instances.return_value = None
+            mock_aws.wait_until_stopped.return_value = True
+
+            scenario.node_stop_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+            # Should not call wait_for_unknown_status
+            mock_wait_unknown.assert_not_called()
+
+    def test_node_stop_scenario_failure(self):
+        """Test node stop scenario with failure"""
+        node = 'ip-10-0-1-100.ec2.internal'
+
+        self.mock_aws.get_instance_id.side_effect = Exception("AWS error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_stop_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+    @patch('time.sleep')
+    def test_node_termination_scenario_success(self, _mock_sleep):
+        """Test node termination scenario successfully"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.terminate_instances.return_value = None
+        self.mock_aws.wait_until_terminated.return_value = True
+        self.kubecli.list_nodes.return_value = []
+
+        self.scenario.node_termination_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600,
+            poll_interval=15
+        )
+
+        self.mock_aws.get_instance_id.assert_called_once_with(node)
+        self.mock_aws.terminate_instances.assert_called_once_with(instance_id)
+        self.mock_aws.wait_until_terminated.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('time.sleep')
+    def test_node_termination_scenario_node_still_exists(self, _mock_sleep):
+        """Test node termination scenario when node still exists"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.terminate_instances.return_value = None
+        self.mock_aws.wait_until_terminated.return_value = True
+        # Node still in list after timeout
+        self.kubecli.list_nodes.return_value = [node]
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_termination_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=2,
+                poll_interval=15
+            )
+
+    def test_node_termination_scenario_failure(self):
+        """Test node termination scenario with failure"""
+        node = 'ip-10-0-1-100.ec2.internal'
+
+        self.mock_aws.get_instance_id.side_effect = Exception("AWS error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_termination_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_reboot_scenario_success(self, mock_wait_ready, mock_wait_unknown):
+        """Test node reboot scenario successfully"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.reboot_instances.return_value = None
+
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600
+        )
+
+        self.mock_aws.get_instance_id.assert_called_once_with(node)
+        self.mock_aws.reboot_instances.assert_called_once_with(instance_id)
+        mock_wait_unknown.assert_called_once()
+        mock_wait_ready.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_reboot_scenario_no_kube_check(self, mock_wait_ready, mock_wait_unknown):
+        """Test node reboot scenario without kube check"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        # Create scenario with node_action_kube_check=False
+        with patch('krkn.scenario_plugins.node_actions.aws_node_scenarios.AWS') as mock_aws_class:
+            mock_aws = MagicMock()
+            mock_aws_class.return_value = mock_aws
+            scenario = aws_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=False,
+                affected_nodes_status=AffectedNodeStatus()
+            )
+
+            mock_aws.get_instance_id.return_value = instance_id
+            mock_aws.reboot_instances.return_value = None
+
+            scenario.node_reboot_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600
+            )
+
+            # Should not call wait functions
+            mock_wait_unknown.assert_not_called()
+            mock_wait_ready.assert_not_called()
+
+    def test_node_reboot_scenario_failure(self):
+        """Test node reboot scenario with failure"""
+        node = 'ip-10-0-1-100.ec2.internal'
+
+        self.mock_aws.get_instance_id.side_effect = Exception("AWS error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_reboot_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600
+            )
+
+    def test_node_reboot_scenario_multiple_kills(self):
+        """Test node reboot scenario with multiple kill counts"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        with patch('krkn.scenario_plugins.node_actions.aws_node_scenarios.AWS') as mock_aws_class:
+            mock_aws = MagicMock()
+            mock_aws_class.return_value = mock_aws
+            scenario = aws_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=False,
+                affected_nodes_status=AffectedNodeStatus()
+            )
+
+            mock_aws.get_instance_id.return_value = instance_id
+            mock_aws.reboot_instances.return_value = None
+
+            scenario.node_reboot_scenario(
+                instance_kill_count=3,
+                node=node,
+                timeout=600
+            )
+
+            self.assertEqual(mock_aws.reboot_instances.call_count, 3)
+            self.assertEqual(len(scenario.affected_nodes_status.affected_nodes), 3)
+
+    def test_get_disk_attachment_info_success(self):
+        """Test getting disk attachment info successfully"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+        volume_ids = ['vol-12345678']
+        attachment_details = [
+            {
+                'VolumeId': 'vol-12345678',
+                'Attachments': [{
+                    'InstanceId': instance_id,
+                    'Device': '/dev/sdf'
+                }]
+            }
+        ]
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.get_volumes_ids.return_value = volume_ids
+        self.mock_aws.get_volume_attachment_details.return_value = attachment_details
+
+        result = self.scenario.get_disk_attachment_info(
+            instance_kill_count=1,
+            node=node
+        )
+
+        self.assertEqual(result, attachment_details)
+        self.mock_aws.get_instance_id.assert_called_once_with(node)
+        self.mock_aws.get_volumes_ids.assert_called_once()
+        self.mock_aws.get_volume_attachment_details.assert_called_once_with(volume_ids)
+
+    def test_get_disk_attachment_info_no_volumes(self):
+        """Test getting disk attachment info when no volumes exist"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.get_volumes_ids.return_value = []
+
+        result = self.scenario.get_disk_attachment_info(
+            instance_kill_count=1,
+            node=node
+        )
+
+        self.assertIsNone(result)
+        self.mock_aws.get_volume_attachment_details.assert_not_called()
+
+    def test_get_disk_attachment_info_failure(self):
+        """Test getting disk attachment info with failure"""
+        node = 'ip-10-0-1-100.ec2.internal'
+
+        self.mock_aws.get_instance_id.side_effect = Exception("AWS error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.get_disk_attachment_info(
+                instance_kill_count=1,
+                node=node
+            )
+
+    def test_disk_detach_scenario_success(self):
+        """Test disk detach scenario successfully"""
+        node = 'ip-10-0-1-100.ec2.internal'
+        instance_id = 'i-1234567890abcdef0'
+        volume_ids = ['vol-12345678', 'vol-87654321']
+
+        self.mock_aws.get_instance_id.return_value = instance_id
+        self.mock_aws.get_volumes_ids.return_value = volume_ids
+        self.mock_aws.detach_volumes.return_value = None
+
+        self.scenario.disk_detach_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600
+        )
+
+        self.mock_aws.get_instance_id.assert_called_once_with(node)
+        self.mock_aws.get_volumes_ids.assert_called_once()
+        self.mock_aws.detach_volumes.assert_called_once_with(volume_ids)
+
+    def test_disk_detach_scenario_failure(self):
+        """Test disk detach scenario with failure"""
+        node = 'ip-10-0-1-100.ec2.internal'
+
+        self.mock_aws.get_instance_id.side_effect = Exception("AWS error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.disk_detach_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600
+            )
+
+    def test_disk_attach_scenario_success(self):
+        """Test disk attach scenario successfully"""
+        attachment_details = [
+            {
+                'VolumeId': 'vol-12345678',
+                'Attachments': [{
+                    'InstanceId': 'i-1234567890abcdef0',
+                    'Device': '/dev/sdf',
+                    'VolumeId': 'vol-12345678'
+                }]
+            },
+            {
+                'VolumeId': 'vol-87654321',
+                'Attachments': [{
+                    'InstanceId': 'i-1234567890abcdef0',
+                    'Device': '/dev/sdg',
+                    'VolumeId': 'vol-87654321'
+                }]
+            }
+        ]
+
+        self.mock_aws.attach_volume.return_value = None
+
+        self.scenario.disk_attach_scenario(
+            instance_kill_count=1,
+            attachment_details=attachment_details,
+            timeout=600
+        )
+
+        self.assertEqual(self.mock_aws.attach_volume.call_count, 2)
+
+    def test_disk_attach_scenario_multiple_kills(self):
+        """Test disk attach scenario with multiple kill counts"""
+        attachment_details = [
+            {
+                'VolumeId': 'vol-12345678',
+                'Attachments': [{
+                    'InstanceId': 'i-1234567890abcdef0',
+                    'Device': '/dev/sdf',
+                    'VolumeId': 'vol-12345678'
+                }]
+            }
+        ]
+
+        self.mock_aws.attach_volume.return_value = None
+
+        self.scenario.disk_attach_scenario(
+            instance_kill_count=3,
+            attachment_details=attachment_details,
+            timeout=600
+        )
+
+        # Should call attach_volume 3 times (once per kill count)
+        self.assertEqual(self.mock_aws.attach_volume.call_count, 3)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_az_node_scenarios.py

@@ -0,0 +1,784 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for azure_node_scenarios class
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_az_node_scenarios.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+from unittest.mock import Mock, patch
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
+
+from krkn.scenario_plugins.node_actions.az_node_scenarios import Azure, azure_node_scenarios
+
+
+class TestAzure(unittest.TestCase):
+    """Test suite for Azure class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Mock environment variable
+        self.env_patcher = patch.dict('os.environ', {'AZURE_SUBSCRIPTION_ID': 'test-subscription-id'})
+        self.env_patcher.start()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.env_patcher.stop()
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    @patch('logging.info')
+    def test_azure_init(self, mock_logging, mock_credential, mock_compute, mock_network):
+        """Test Azure class initialization"""
+        mock_creds = Mock()
+        mock_credential.return_value = mock_creds
+
+        azure = Azure()
+
+        mock_credential.assert_called_once()
+        mock_compute.assert_called_once()
+        mock_network.assert_called_once()
+        self.assertIsNotNone(azure.compute_client)
+        self.assertIsNotNone(azure.network_client)
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_get_instance_id_found(self, mock_credential, mock_compute, mock_network):
+        """Test get_instance_id when VM is found"""
+        azure = Azure()
+
+        # Mock VM
+        mock_vm = Mock()
+        mock_vm.name = "test-node"
+        mock_vm.id = "/subscriptions/sub-id/resourceGroups/test-rg/providers/Microsoft.Compute/virtualMachines/test-node"
+
+        azure.compute_client.virtual_machines.list_all.return_value = [mock_vm]
+
+        vm_name, resource_group = azure.get_instance_id("test-node")
+
+        self.assertEqual(vm_name, "test-node")
+        self.assertEqual(resource_group, "test-rg")
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_get_instance_id_not_found(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test get_instance_id when VM is not found"""
+        azure = Azure()
+
+        azure.compute_client.virtual_machines.list_all.return_value = []
+
+        result = azure.get_instance_id("nonexistent-node")
+
+        self.assertIsNone(result)
+        mock_logging.assert_called()
+        self.assertIn("Couldn't find vm", str(mock_logging.call_args))
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_get_network_interface(self, mock_credential, mock_compute, mock_network):
+        """Test get_network_interface retrieves network details"""
+        azure = Azure()
+
+        # Mock VM with network profile
+        mock_vm = Mock()
+        mock_nic_ref = Mock()
+        mock_nic_ref.id = "/subscriptions/sub-id/resourceGroups/test-rg/providers/Microsoft.Network/networkInterfaces/test-nic"
+        mock_vm.network_profile.network_interfaces = [mock_nic_ref]
+
+        # Mock NIC
+        mock_nic = Mock()
+        mock_nic.location = "eastus"
+        mock_ip_config = Mock()
+        mock_ip_config.private_ip_address = "10.0.1.5"
+        mock_ip_config.subnet.id = "/subscriptions/sub-id/resourceGroups/network-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet"
+        mock_nic.ip_configurations = [mock_ip_config]
+
+        azure.compute_client.virtual_machines.get.return_value = mock_vm
+        azure.network_client.network_interfaces.get.return_value = mock_nic
+
+        subnet, vnet, ip, net_rg, location = azure.get_network_interface("test-node", "test-rg")
+
+        self.assertEqual(subnet, "test-subnet")
+        self.assertEqual(vnet, "test-vnet")
+        self.assertEqual(ip, "10.0.1.5")
+        self.assertEqual(net_rg, "network-rg")
+        self.assertEqual(location, "eastus")
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_start_instances_success(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test start_instances successfully starts VM"""
+        azure = Azure()
+
+        mock_operation = Mock()
+        azure.compute_client.virtual_machines.begin_start.return_value = mock_operation
+
+        azure.start_instances("test-rg", "test-vm")
+
+        azure.compute_client.virtual_machines.begin_start.assert_called_once_with("test-rg", "test-vm")
+        mock_logging.assert_called()
+        self.assertIn("started", str(mock_logging.call_args))
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_start_instances_failure(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test start_instances handles failure"""
+        azure = Azure()
+
+        azure.compute_client.virtual_machines.begin_start.side_effect = Exception("Start failed")
+
+        with self.assertRaises(RuntimeError):
+            azure.start_instances("test-rg", "test-vm")
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to start", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_stop_instances_success(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test stop_instances successfully stops VM"""
+        azure = Azure()
+
+        mock_operation = Mock()
+        azure.compute_client.virtual_machines.begin_power_off.return_value = mock_operation
+
+        azure.stop_instances("test-rg", "test-vm")
+
+        azure.compute_client.virtual_machines.begin_power_off.assert_called_once_with("test-rg", "test-vm")
+        mock_logging.assert_called()
+        self.assertIn("stopped", str(mock_logging.call_args))
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_stop_instances_failure(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test stop_instances handles failure"""
+        azure = Azure()
+
+        azure.compute_client.virtual_machines.begin_power_off.side_effect = Exception("Stop failed")
+
+        with self.assertRaises(RuntimeError):
+            azure.stop_instances("test-rg", "test-vm")
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to stop", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_terminate_instances_success(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test terminate_instances successfully deletes VM"""
+        azure = Azure()
+
+        mock_operation = Mock()
+        azure.compute_client.virtual_machines.begin_delete.return_value = mock_operation
+
+        azure.terminate_instances("test-rg", "test-vm")
+
+        azure.compute_client.virtual_machines.begin_delete.assert_called_once_with("test-rg", "test-vm")
+        mock_logging.assert_called()
+        self.assertIn("terminated", str(mock_logging.call_args))
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_terminate_instances_failure(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test terminate_instances handles failure"""
+        azure = Azure()
+
+        azure.compute_client.virtual_machines.begin_delete.side_effect = Exception("Delete failed")
+
+        with self.assertRaises(RuntimeError):
+            azure.terminate_instances("test-rg", "test-vm")
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to terminate", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_reboot_instances_success(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test reboot_instances successfully reboots VM"""
+        azure = Azure()
+
+        mock_operation = Mock()
+        azure.compute_client.virtual_machines.begin_restart.return_value = mock_operation
+
+        azure.reboot_instances("test-rg", "test-vm")
+
+        azure.compute_client.virtual_machines.begin_restart.assert_called_once_with("test-rg", "test-vm")
+        mock_logging.assert_called()
+        self.assertIn("rebooted", str(mock_logging.call_args))
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_reboot_instances_failure(self, mock_credential, mock_compute, mock_network, mock_logging):
+        """Test reboot_instances handles failure"""
+        azure = Azure()
+
+        azure.compute_client.virtual_machines.begin_restart.side_effect = Exception("Reboot failed")
+
+        with self.assertRaises(RuntimeError):
+            azure.reboot_instances("test-rg", "test-vm")
+
+        mock_logging.assert_called()
+        self.assertIn("Failed to reboot", str(mock_logging.call_args))
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_get_vm_status(self, mock_credential, mock_compute, mock_network):
+        """Test get_vm_status returns VM power state"""
+        azure = Azure()
+
+        mock_status1 = Mock()
+        mock_status1.code = "ProvisioningState/succeeded"
+        mock_status2 = Mock()
+        mock_status2.code = "PowerState/running"
+
+        mock_instance_view = Mock()
+        mock_instance_view.statuses = [mock_status1, mock_status2]
+        azure.compute_client.virtual_machines.instance_view.return_value = mock_instance_view
+
+        status = azure.get_vm_status("test-rg", "test-vm")
+
+        self.assertEqual(status.code, "PowerState/running")
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_wait_until_running_success(self, mock_credential, mock_compute, mock_network, mock_logging, mock_sleep):
+        """Test wait_until_running waits for VM to be running"""
+        azure = Azure()
+
+        mock_status_starting = Mock()
+        mock_status_starting.code = "PowerState/starting"
+        mock_status_running = Mock()
+        mock_status_running.code = "PowerState/running"
+
+        mock_instance_view1 = Mock()
+        mock_instance_view1.statuses = [Mock(), mock_status_starting]
+        mock_instance_view2 = Mock()
+        mock_instance_view2.statuses = [Mock(), mock_status_running]
+
+        azure.compute_client.virtual_machines.instance_view.side_effect = [
+            mock_instance_view1,
+            mock_instance_view2
+        ]
+
+        mock_affected_node = Mock(spec=AffectedNode)
+
+        result = azure.wait_until_running("test-rg", "test-vm", 300, mock_affected_node)
+
+        self.assertTrue(result)
+        mock_affected_node.set_affected_node_status.assert_called_once()
+        args = mock_affected_node.set_affected_node_status.call_args[0]
+        self.assertEqual(args[0], "running")
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_wait_until_running_timeout(self, mock_credential, mock_compute, mock_network, mock_logging, mock_sleep):
+        """Test wait_until_running returns False on timeout"""
+        azure = Azure()
+
+        mock_status = Mock()
+        mock_status.code = "PowerState/starting"
+        mock_instance_view = Mock()
+        mock_instance_view.statuses = [Mock(), mock_status]
+
+        azure.compute_client.virtual_machines.instance_view.return_value = mock_instance_view
+
+        result = azure.wait_until_running("test-rg", "test-vm", 10, None)
+
+        self.assertFalse(result)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_wait_until_stopped_success(self, mock_credential, mock_compute, mock_network, mock_logging, mock_sleep):
+        """Test wait_until_stopped waits for VM to be stopped"""
+        azure = Azure()
+
+        mock_status_stopping = Mock()
+        mock_status_stopping.code = "PowerState/stopping"
+        mock_status_stopped = Mock()
+        mock_status_stopped.code = "PowerState/stopped"
+
+        mock_instance_view1 = Mock()
+        mock_instance_view1.statuses = [Mock(), mock_status_stopping]
+        mock_instance_view2 = Mock()
+        mock_instance_view2.statuses = [Mock(), mock_status_stopped]
+
+        azure.compute_client.virtual_machines.instance_view.side_effect = [
+            mock_instance_view1,
+            mock_instance_view2
+        ]
+
+        mock_affected_node = Mock(spec=AffectedNode)
+
+        result = azure.wait_until_stopped("test-rg", "test-vm", 300, mock_affected_node)
+
+        self.assertTrue(result)
+        mock_affected_node.set_affected_node_status.assert_called_once()
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_wait_until_stopped_timeout(self, mock_credential, mock_compute, mock_network, mock_logging, mock_sleep):
+        """Test wait_until_stopped returns False on timeout"""
+        azure = Azure()
+
+        mock_status = Mock()
+        mock_status.code = "PowerState/stopping"
+        mock_instance_view = Mock()
+        mock_instance_view.statuses = [Mock(), mock_status]
+
+        azure.compute_client.virtual_machines.instance_view.return_value = mock_instance_view
+
+        result = azure.wait_until_stopped("test-rg", "test-vm", 10, None)
+
+        self.assertFalse(result)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_wait_until_terminated_success(self, mock_credential, mock_compute, mock_network, mock_logging, mock_sleep):
+        """Test wait_until_terminated waits for VM deletion"""
+        azure = Azure()
+
+        mock_status_deleting = Mock()
+        mock_status_deleting.code = "ProvisioningState/deleting"
+        mock_instance_view = Mock()
+        mock_instance_view.statuses = [mock_status_deleting]
+
+        # First call returns deleting, second raises exception (VM deleted)
+        azure.compute_client.virtual_machines.instance_view.side_effect = [
+            mock_instance_view,
+            Exception("VM not found")
+        ]
+
+        mock_affected_node = Mock(spec=AffectedNode)
+
+        result = azure.wait_until_terminated("test-rg", "test-vm", 300, mock_affected_node)
+
+        self.assertTrue(result)
+        mock_affected_node.set_affected_node_status.assert_called_once()
+        args = mock_affected_node.set_affected_node_status.call_args[0]
+        self.assertEqual(args[0], "terminated")
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_wait_until_terminated_timeout(self, mock_credential, mock_compute, mock_network, mock_logging, mock_sleep):
+        """Test wait_until_terminated returns False on timeout"""
+        azure = Azure()
+
+        mock_status = Mock()
+        mock_status.code = "ProvisioningState/deleting"
+        mock_instance_view = Mock()
+        mock_instance_view.statuses = [mock_status]
+
+        azure.compute_client.virtual_machines.instance_view.return_value = mock_instance_view
+
+        result = azure.wait_until_terminated("test-rg", "test-vm", 10, None)
+
+        self.assertFalse(result)
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_create_security_group(self, mock_credential, mock_compute, mock_network):
+        """Test create_security_group creates NSG with deny rules"""
+        azure = Azure()
+
+        mock_nsg_result = Mock()
+        mock_nsg_result.id = "/subscriptions/sub-id/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/chaos"
+        mock_operation = Mock()
+        mock_operation.result.return_value = mock_nsg_result
+
+        azure.network_client.network_security_groups.begin_create_or_update.return_value = mock_operation
+
+        nsg_id = azure.create_security_group("test-rg", "chaos", "eastus", "10.0.1.5")
+
+        self.assertEqual(nsg_id, mock_nsg_result.id)
+        azure.network_client.network_security_groups.begin_create_or_update.assert_called_once()
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_delete_security_group(self, mock_credential, mock_compute, mock_network):
+        """Test delete_security_group deletes NSG"""
+        azure = Azure()
+
+        mock_operation = Mock()
+        mock_operation.result.return_value = None
+        azure.network_client.network_security_groups.begin_delete.return_value = mock_operation
+
+        azure.delete_security_group("test-rg", "chaos")
+
+        azure.network_client.network_security_groups.begin_delete.assert_called_once_with("test-rg", "chaos")
+
+    @patch('builtins.print')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_delete_security_group_with_result(self, mock_credential, mock_compute, mock_network, mock_print):
+        """Test delete_security_group deletes NSG with non-None result"""
+        azure = Azure()
+
+        mock_result = Mock()
+        mock_result.as_dict.return_value = {"id": "/test-nsg-id", "name": "chaos"}
+        mock_operation = Mock()
+        mock_operation.result.return_value = mock_result
+        azure.network_client.network_security_groups.begin_delete.return_value = mock_operation
+
+        azure.delete_security_group("test-rg", "chaos")
+
+        azure.network_client.network_security_groups.begin_delete.assert_called_once_with("test-rg", "chaos")
+        mock_result.as_dict.assert_called_once()
+        mock_print.assert_called_once_with({"id": "/test-nsg-id", "name": "chaos"})
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.NetworkManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.ComputeManagementClient')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.DefaultAzureCredential')
+    def test_update_subnet(self, mock_credential, mock_compute, mock_network):
+        """Test update_subnet updates subnet NSG"""
+        azure = Azure()
+
+        # Mock existing subnet
+        mock_old_nsg = Mock()
+        mock_old_nsg.id = "/old-nsg-id"
+        mock_subnet = Mock()
+        mock_subnet.network_security_group = mock_old_nsg
+
+        azure.network_client.subnets.get.return_value = mock_subnet
+
+        old_nsg = azure.update_subnet("/new-nsg-id", "test-rg", "test-subnet", "test-vnet")
+
+        self.assertEqual(old_nsg, "/old-nsg-id")
+        azure.network_client.subnets.begin_create_or_update.assert_called_once()
+
+
+class TestAzureNodeScenarios(unittest.TestCase):
+    """Test suite for azure_node_scenarios class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        self.env_patcher = patch.dict('os.environ', {'AZURE_SUBSCRIPTION_ID': 'test-subscription-id'})
+        self.env_patcher.start()
+
+        self.mock_kubecli = Mock(spec=KrknKubernetes)
+        self.affected_nodes_status = AffectedNodeStatus()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.env_patcher.stop()
+
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_init(self, mock_azure_class, mock_logging):
+        """Test azure_node_scenarios initialization"""
+        mock_azure_instance = Mock()
+        mock_azure_class.return_value = mock_azure_instance
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        self.assertEqual(scenarios.kubecli, self.mock_kubecli)
+        self.assertTrue(scenarios.node_action_kube_check)
+        self.assertEqual(scenarios.azure, mock_azure_instance)
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_start_scenario_success(self, mock_azure_class, mock_logging, mock_nodeaction):
+        """Test node_start_scenario successfully starts node"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.wait_until_running.return_value = True
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_start_scenario(1, "test-node", 300, 15)
+
+        mock_azure.get_instance_id.assert_called_once_with("test-node")
+        mock_azure.start_instances.assert_called_once_with("test-rg", "test-vm")
+        mock_azure.wait_until_running.assert_called_once()
+        mock_nodeaction.wait_for_ready_status.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_start_scenario_no_kube_check(self, mock_azure_class, mock_logging, mock_nodeaction):
+        """Test node_start_scenario without Kubernetes check"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.wait_until_running.return_value = True
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        scenarios.node_start_scenario(1, "test-node", 300, 15)
+
+        mock_azure.start_instances.assert_called_once()
+        mock_nodeaction.wait_for_ready_status.assert_not_called()
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_start_scenario_failure(self, mock_azure_class, mock_logging):
+        """Test node_start_scenario handles failure"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.start_instances.side_effect = Exception("Start failed")
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(RuntimeError):
+            scenarios.node_start_scenario(1, "test-node", 300, 15)
+
+        mock_logging.assert_called()
+        # Check that failure was logged (either specific or general injection failed message)
+        call_str = str(mock_logging.call_args)
+        self.assertTrue("Failed to start" in call_str or "injection failed" in call_str)
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_start_scenario_multiple_runs(self, mock_azure_class, mock_logging, mock_nodeaction):
+        """Test node_start_scenario with multiple runs"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.wait_until_running.return_value = True
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_start_scenario(3, "test-node", 300, 15)
+
+        self.assertEqual(mock_azure.start_instances.call_count, 3)
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 3)
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_stop_scenario_success(self, mock_azure_class, mock_logging, mock_nodeaction):
+        """Test node_stop_scenario successfully stops node"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.wait_until_stopped.return_value = True
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_stop_scenario(1, "test-node", 300, 15)
+
+        mock_azure.get_instance_id.assert_called_once_with("test-node")
+        mock_azure.stop_instances.assert_called_once_with("test-rg", "test-vm")
+        mock_azure.wait_until_stopped.assert_called_once()
+        mock_nodeaction.wait_for_unknown_status.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_stop_scenario_failure(self, mock_azure_class, mock_logging):
+        """Test node_stop_scenario handles failure"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.stop_instances.side_effect = Exception("Stop failed")
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(RuntimeError):
+            scenarios.node_stop_scenario(1, "test-node", 300, 15)
+
+        mock_logging.assert_called()
+        # Check that failure was logged
+        call_str = str(mock_logging.call_args)
+        self.assertTrue("Failed to stop" in call_str or "injection failed" in call_str)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_termination_scenario_success(self, mock_azure_class, mock_logging, mock_sleep):
+        """Test node_termination_scenario successfully terminates node"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.wait_until_terminated.return_value = True
+
+        self.mock_kubecli.list_nodes.return_value = ["other-node"]
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        scenarios.node_termination_scenario(1, "test-node", 300, 15)
+
+        mock_azure.terminate_instances.assert_called_once_with("test-rg", "test-vm")
+        mock_azure.wait_until_terminated.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('time.sleep')
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_termination_scenario_node_still_exists(self, mock_azure_class, mock_logging, mock_sleep):
+        """Test node_termination_scenario when node still exists after timeout"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.wait_until_terminated.return_value = True
+
+        # Node still in list after termination attempt
+        self.mock_kubecli.list_nodes.return_value = ["test-vm", "other-node"]
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(RuntimeError):
+            scenarios.node_termination_scenario(1, "test-node", 5, 15)
+
+        mock_logging.assert_called()
+        # Check that failure was logged
+        call_str = str(mock_logging.call_args)
+        self.assertTrue("Failed to terminate" in call_str or "injection failed" in call_str)
+
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.nodeaction')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_reboot_scenario_success(self, mock_azure_class, mock_logging, mock_nodeaction):
+        """Test node_reboot_scenario successfully reboots node"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, True, self.affected_nodes_status)
+
+        scenarios.node_reboot_scenario(1, "test-node", 300, soft_reboot=False)
+
+        mock_azure.reboot_instances.assert_called_once_with("test-rg", "test-vm")
+        mock_nodeaction.wait_for_ready_status.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_reboot_scenario_failure(self, mock_azure_class, mock_logging):
+        """Test node_reboot_scenario handles failure"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.reboot_instances.side_effect = Exception("Reboot failed")
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(RuntimeError):
+            scenarios.node_reboot_scenario(1, "test-node", 300)
+
+        mock_logging.assert_called()
+        # Check that failure was logged
+        call_str = str(mock_logging.call_args)
+        self.assertTrue("Failed to reboot" in call_str or "injection failed" in call_str)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_block_scenario_success(self, mock_azure_class, mock_logging, mock_sleep):
+        """Test node_block_scenario successfully blocks and unblocks node"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.get_network_interface.return_value = (
+            "test-subnet", "test-vnet", "10.0.1.5", "network-rg", "eastus"
+        )
+        mock_azure.create_security_group.return_value = "/new-nsg-id"
+        mock_azure.update_subnet.return_value = "/old-nsg-id"
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        scenarios.node_block_scenario(1, "test-node", 300, 60)
+
+        mock_azure.create_security_group.assert_called_once()
+        # Should be called twice: once to apply block, once to remove
+        self.assertEqual(mock_azure.update_subnet.call_count, 2)
+        mock_azure.delete_security_group.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('time.sleep')
+    @patch('logging.error')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_block_scenario_failure(self, mock_azure_class, mock_logging, mock_sleep):
+        """Test node_block_scenario handles failure"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.get_network_interface.side_effect = Exception("Network error")
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        with self.assertRaises(RuntimeError):
+            scenarios.node_block_scenario(1, "test-node", 300, 60)
+
+        mock_logging.assert_called()
+        # Check that failure was logged
+        call_str = str(mock_logging.call_args)
+        self.assertTrue("Failed to block" in call_str or "injection failed" in call_str)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.az_node_scenarios.Azure')
+    def test_node_block_scenario_duration_timing(self, mock_azure_class, mock_logging, mock_sleep):
+        """Test node_block_scenario waits for specified duration"""
+        mock_azure = Mock()
+        mock_azure_class.return_value = mock_azure
+        mock_azure.get_instance_id.return_value = ("test-vm", "test-rg")
+        mock_azure.get_network_interface.return_value = (
+            "test-subnet", "test-vnet", "10.0.1.5", "network-rg", "eastus"
+        )
+        mock_azure.create_security_group.return_value = "/new-nsg-id"
+        mock_azure.update_subnet.return_value = "/old-nsg-id"
+
+        scenarios = azure_node_scenarios(self.mock_kubecli, False, self.affected_nodes_status)
+
+        scenarios.node_block_scenario(1, "test-node", 300, 120)
+
+        # Verify sleep was called with the correct duration
+        mock_sleep.assert_called_with(120)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_common_node_functions.py

@@ -0,0 +1,476 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for common_node_functions module
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_common_node_functions.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+from unittest.mock import MagicMock, Mock, patch, call
+import logging
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode
+
+from krkn.scenario_plugins.node_actions import common_node_functions
+
+
+class TestCommonNodeFunctions(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Set up test fixtures before each test
+        """
+        self.mock_kubecli = Mock(spec=KrknKubernetes)
+        self.mock_affected_node = Mock(spec=AffectedNode)
+
+    def test_get_node_by_name_all_nodes_exist(self):
+        """
+        Test get_node_by_name returns list when all nodes exist
+        """
+        node_name_list = ["node1", "node2", "node3"]
+        self.mock_kubecli.list_killable_nodes.return_value = ["node1", "node2", "node3", "node4"]
+
+        result = common_node_functions.get_node_by_name(node_name_list, self.mock_kubecli)
+
+        self.assertEqual(result, node_name_list)
+        self.mock_kubecli.list_killable_nodes.assert_called_once()
+
+    def test_get_node_by_name_single_node(self):
+        """
+        Test get_node_by_name with single node
+        """
+        node_name_list = ["worker-1"]
+        self.mock_kubecli.list_killable_nodes.return_value = ["worker-1", "worker-2"]
+
+        result = common_node_functions.get_node_by_name(node_name_list, self.mock_kubecli)
+
+        self.assertEqual(result, node_name_list)
+
+    @patch('logging.info')
+    def test_get_node_by_name_node_not_exist(self, mock_logging):
+        """
+        Test get_node_by_name returns None when node doesn't exist
+        """
+        node_name_list = ["node1", "nonexistent-node"]
+        self.mock_kubecli.list_killable_nodes.return_value = ["node1", "node2", "node3"]
+
+        result = common_node_functions.get_node_by_name(node_name_list, self.mock_kubecli)
+
+        self.assertIsNone(result)
+        mock_logging.assert_called()
+        self.assertIn("does not exist", str(mock_logging.call_args))
+
+    @patch('logging.info')
+    def test_get_node_by_name_empty_killable_list(self, mock_logging):
+        """
+        Test get_node_by_name when no killable nodes exist
+        """
+        node_name_list = ["node1"]
+        self.mock_kubecli.list_killable_nodes.return_value = []
+
+        result = common_node_functions.get_node_by_name(node_name_list, self.mock_kubecli)
+
+        self.assertIsNone(result)
+        mock_logging.assert_called()
+
+    @patch('logging.info')
+    def test_get_node_single_label_selector(self, mock_logging):
+        """
+        Test get_node with single label selector
+        """
+        label_selector = "node-role.kubernetes.io/worker"
+        instance_kill_count = 2
+        self.mock_kubecli.list_killable_nodes.return_value = ["worker-1", "worker-2", "worker-3"]
+
+        result = common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertEqual(len(result), 2)
+        self.assertTrue(all(node in ["worker-1", "worker-2", "worker-3"] for node in result))
+        self.mock_kubecli.list_killable_nodes.assert_called_once_with(label_selector)
+        mock_logging.assert_called()
+
+    @patch('logging.info')
+    def test_get_node_multiple_label_selectors(self, mock_logging):
+        """
+        Test get_node with multiple comma-separated label selectors
+        """
+        label_selector = "node-role.kubernetes.io/worker,topology.kubernetes.io/zone=us-east-1a"
+        instance_kill_count = 3
+        self.mock_kubecli.list_killable_nodes.side_effect = [
+            ["worker-1", "worker-2"],
+            ["worker-3", "worker-4"]
+        ]
+
+        result = common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertEqual(len(result), 3)
+        self.assertTrue(all(node in ["worker-1", "worker-2", "worker-3", "worker-4"] for node in result))
+        self.assertEqual(self.mock_kubecli.list_killable_nodes.call_count, 2)
+
+    @patch('logging.info')
+    def test_get_node_return_all_when_count_equals_total(self, mock_logging):
+        """
+        Test get_node returns all nodes when instance_kill_count equals number of nodes
+        """
+        label_selector = "node-role.kubernetes.io/worker"
+        nodes = ["worker-1", "worker-2", "worker-3"]
+        instance_kill_count = 3
+        self.mock_kubecli.list_killable_nodes.return_value = nodes
+
+        result = common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertEqual(result, nodes)
+
+    @patch('logging.info')
+    def test_get_node_return_all_when_count_is_zero(self, mock_logging):
+        """
+        Test get_node returns all nodes when instance_kill_count is 0
+        """
+        label_selector = "node-role.kubernetes.io/worker"
+        nodes = ["worker-1", "worker-2", "worker-3"]
+        instance_kill_count = 0
+        self.mock_kubecli.list_killable_nodes.return_value = nodes
+
+        result = common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertEqual(result, nodes)
+
+    @patch('logging.info')
+    @patch('random.randint')
+    def test_get_node_random_selection(self, mock_randint, mock_logging):
+        """
+        Test get_node randomly selects nodes when count is less than total
+        """
+        label_selector = "node-role.kubernetes.io/worker"
+        instance_kill_count = 2
+        self.mock_kubecli.list_killable_nodes.return_value = ["worker-1", "worker-2", "worker-3", "worker-4"]
+        # Mock random selection to return predictable values
+        mock_randint.side_effect = [1, 0]  # Select index 1, then index 0
+
+        result = common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertEqual(len(result), 2)
+        # Verify nodes were removed after selection to avoid duplicates
+        self.assertEqual(len(set(result)), 2)
+
+    def test_get_node_no_nodes_with_label(self):
+        """
+        Test get_node raises exception when no nodes match label selector
+        """
+        label_selector = "nonexistent-label"
+        instance_kill_count = 1
+        self.mock_kubecli.list_killable_nodes.return_value = []
+
+        with self.assertRaises(Exception) as context:
+            common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertIn("Ready nodes with the provided label selector do not exist", str(context.exception))
+
+    def test_get_node_single_node_available(self):
+        """
+        Test get_node when only one node is available
+        """
+        label_selector = "node-role.kubernetes.io/master"
+        instance_kill_count = 1
+        self.mock_kubecli.list_killable_nodes.return_value = ["master-1"]
+
+        result = common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertEqual(result, ["master-1"])
+
+    def test_wait_for_ready_status_without_affected_node(self):
+        """
+        Test wait_for_ready_status without providing affected_node
+        """
+        node = "test-node"
+        timeout = 300
+        expected_affected_node = Mock(spec=AffectedNode)
+        self.mock_kubecli.watch_node_status.return_value = expected_affected_node
+
+        result = common_node_functions.wait_for_ready_status(node, timeout, self.mock_kubecli)
+
+        self.assertEqual(result, expected_affected_node)
+        self.mock_kubecli.watch_node_status.assert_called_once_with(node, "True", timeout, None)
+
+    def test_wait_for_ready_status_with_affected_node(self):
+        """
+        Test wait_for_ready_status with provided affected_node
+        """
+        node = "test-node"
+        timeout = 300
+        self.mock_kubecli.watch_node_status.return_value = self.mock_affected_node
+
+        result = common_node_functions.wait_for_ready_status(
+            node, timeout, self.mock_kubecli, self.mock_affected_node
+        )
+
+        self.assertEqual(result, self.mock_affected_node)
+        self.mock_kubecli.watch_node_status.assert_called_once_with(
+            node, "True", timeout, self.mock_affected_node
+        )
+
+    def test_wait_for_not_ready_status_without_affected_node(self):
+        """
+        Test wait_for_not_ready_status without providing affected_node
+        """
+        node = "test-node"
+        timeout = 300
+        expected_affected_node = Mock(spec=AffectedNode)
+        self.mock_kubecli.watch_node_status.return_value = expected_affected_node
+
+        result = common_node_functions.wait_for_not_ready_status(node, timeout, self.mock_kubecli)
+
+        self.assertEqual(result, expected_affected_node)
+        self.mock_kubecli.watch_node_status.assert_called_once_with(node, "False", timeout, None)
+
+    def test_wait_for_not_ready_status_with_affected_node(self):
+        """
+        Test wait_for_not_ready_status with provided affected_node
+        """
+        node = "test-node"
+        timeout = 300
+        self.mock_kubecli.watch_node_status.return_value = self.mock_affected_node
+
+        result = common_node_functions.wait_for_not_ready_status(
+            node, timeout, self.mock_kubecli, self.mock_affected_node
+        )
+
+        self.assertEqual(result, self.mock_affected_node)
+        self.mock_kubecli.watch_node_status.assert_called_once_with(
+            node, "False", timeout, self.mock_affected_node
+        )
+
+    def test_wait_for_unknown_status_without_affected_node(self):
+        """
+        Test wait_for_unknown_status without providing affected_node
+        """
+        node = "test-node"
+        timeout = 300
+        expected_affected_node = Mock(spec=AffectedNode)
+        self.mock_kubecli.watch_node_status.return_value = expected_affected_node
+
+        result = common_node_functions.wait_for_unknown_status(node, timeout, self.mock_kubecli)
+
+        self.assertEqual(result, expected_affected_node)
+        self.mock_kubecli.watch_node_status.assert_called_once_with(node, "Unknown", timeout, None)
+
+    def test_wait_for_unknown_status_with_affected_node(self):
+        """
+        Test wait_for_unknown_status with provided affected_node
+        """
+        node = "test-node"
+        timeout = 300
+        self.mock_kubecli.watch_node_status.return_value = self.mock_affected_node
+
+        result = common_node_functions.wait_for_unknown_status(
+            node, timeout, self.mock_kubecli, self.mock_affected_node
+        )
+
+        self.assertEqual(result, self.mock_affected_node)
+        self.mock_kubecli.watch_node_status.assert_called_once_with(
+            node, "Unknown", timeout, self.mock_affected_node
+        )
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.paramiko.SSHClient')
+    def test_check_service_status_success(self, mock_ssh_client, mock_logging, mock_sleep):
+        """
+        Test check_service_status successfully checks service status
+        """
+        node = "192.168.1.100"
+        service = ["neutron-server", "nova-compute"]
+        ssh_private_key = "~/.ssh/id_rsa"
+        timeout = 60
+
+        # Mock SSH client
+        mock_ssh = Mock()
+        mock_ssh_client.return_value = mock_ssh
+        mock_ssh.connect.return_value = None
+
+        # Mock exec_command to return active status
+        mock_stdout = Mock()
+        mock_stdout.readlines.return_value = ["active\n"]
+        mock_ssh.exec_command.return_value = (Mock(), mock_stdout, Mock())
+
+        common_node_functions.check_service_status(node, service, ssh_private_key, timeout)
+
+        # Verify SSH connection was attempted
+        mock_ssh.connect.assert_called()
+        # Verify service status was checked for each service
+        self.assertEqual(mock_ssh.exec_command.call_count, 2)
+        # Verify SSH connection was closed
+        mock_ssh.close.assert_called_once()
+
+    @patch('time.sleep')
+    @patch('logging.error')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.paramiko.SSHClient')
+    def test_check_service_status_service_inactive(self, mock_ssh_client, mock_logging_info, mock_logging_error, mock_sleep):
+        """
+        Test check_service_status logs error when service is inactive
+        """
+        node = "192.168.1.100"
+        service = ["neutron-server"]
+        ssh_private_key = "~/.ssh/id_rsa"
+        timeout = 60
+
+        # Mock SSH client
+        mock_ssh = Mock()
+        mock_ssh_client.return_value = mock_ssh
+        mock_ssh.connect.return_value = None
+
+        # Mock exec_command to return inactive status
+        mock_stdout = Mock()
+        mock_stdout.readlines.return_value = ["inactive\n"]
+        mock_ssh.exec_command.return_value = (Mock(), mock_stdout, Mock())
+
+        common_node_functions.check_service_status(node, service, ssh_private_key, timeout)
+
+        # Verify error was logged for inactive service
+        mock_logging_error.assert_called()
+        error_call_str = str(mock_logging_error.call_args)
+        self.assertIn("inactive", error_call_str)
+        mock_ssh.close.assert_called_once()
+
+    @patch('time.sleep')
+    @patch('logging.error')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.paramiko.SSHClient')
+    def test_check_service_status_ssh_connection_fails(self, mock_ssh_client, mock_logging_info, mock_logging_error, mock_sleep):
+        """
+        Test check_service_status handles SSH connection failures
+        """
+        node = "192.168.1.100"
+        service = ["neutron-server"]
+        ssh_private_key = "~/.ssh/id_rsa"
+        timeout = 5
+
+        # Mock SSH client to raise exception
+        mock_ssh = Mock()
+        mock_ssh_client.return_value = mock_ssh
+        mock_ssh.connect.side_effect = Exception("Connection timeout")
+
+        # Mock exec_command for when connection eventually works (or doesn't)
+        mock_stdout = Mock()
+        mock_stdout.readlines.return_value = ["active\n"]
+        mock_ssh.exec_command.return_value = (Mock(), mock_stdout, Mock())
+
+        common_node_functions.check_service_status(node, service, ssh_private_key, timeout)
+
+        # Verify error was logged for SSH connection failure
+        mock_logging_error.assert_called()
+        error_call_str = str(mock_logging_error.call_args)
+        self.assertIn("Failed to ssh", error_call_str)
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.paramiko.SSHClient')
+    def test_check_service_status_multiple_services(self, mock_ssh_client, mock_logging, mock_sleep):
+        """
+        Test check_service_status with multiple services
+        """
+        node = "192.168.1.100"
+        service = ["service1", "service2", "service3"]
+        ssh_private_key = "~/.ssh/id_rsa"
+        timeout = 60
+
+        # Mock SSH client
+        mock_ssh = Mock()
+        mock_ssh_client.return_value = mock_ssh
+        mock_ssh.connect.return_value = None
+
+        # Mock exec_command to return active status
+        mock_stdout = Mock()
+        mock_stdout.readlines.return_value = ["active\n"]
+        mock_ssh.exec_command.return_value = (Mock(), mock_stdout, Mock())
+
+        common_node_functions.check_service_status(node, service, ssh_private_key, timeout)
+
+        # Verify service status was checked for all services
+        self.assertEqual(mock_ssh.exec_command.call_count, 3)
+        mock_ssh.close.assert_called_once()
+
+    @patch('time.sleep')
+    @patch('logging.info')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.paramiko.SSHClient')
+    def test_check_service_status_retry_logic(self, mock_ssh_client, mock_logging, mock_sleep):
+        """
+        Test check_service_status retry logic on connection failure then success
+        """
+        node = "192.168.1.100"
+        service = ["neutron-server"]
+        ssh_private_key = "~/.ssh/id_rsa"
+        timeout = 10
+
+        # Mock SSH client
+        mock_ssh = Mock()
+        mock_ssh_client.return_value = mock_ssh
+        # First two attempts fail, third succeeds
+        mock_ssh.connect.side_effect = [
+            Exception("Timeout"),
+            Exception("Timeout"),
+            None  # Success
+        ]
+
+        # Mock exec_command
+        mock_stdout = Mock()
+        mock_stdout.readlines.return_value = ["active\n"]
+        mock_ssh.exec_command.return_value = (Mock(), mock_stdout, Mock())
+
+        common_node_functions.check_service_status(node, service, ssh_private_key, timeout)
+
+        # Verify multiple connection attempts were made
+        self.assertGreater(mock_ssh.connect.call_count, 1)
+        # Verify service was eventually checked
+        mock_ssh.exec_command.assert_called()
+        mock_ssh.close.assert_called_once()
+
+
+class TestCommonNodeFunctionsIntegration(unittest.TestCase):
+    """Integration-style tests for common_node_functions"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        self.mock_kubecli = Mock(spec=KrknKubernetes)
+
+    @patch('logging.info')
+    def test_get_node_workflow_with_label_filtering(self, mock_logging):
+        """
+        Test complete workflow of getting nodes with label selector and filtering
+        """
+        label_selector = "node-role.kubernetes.io/worker"
+        instance_kill_count = 2
+        available_nodes = ["worker-1", "worker-2", "worker-3", "worker-4", "worker-5"]
+        self.mock_kubecli.list_killable_nodes.return_value = available_nodes
+
+        result = common_node_functions.get_node(label_selector, instance_kill_count, self.mock_kubecli)
+
+        self.assertEqual(len(result), 2)
+        # Verify no duplicates
+        self.assertEqual(len(result), len(set(result)))
+        # Verify all nodes are from the available list
+        self.assertTrue(all(node in available_nodes for node in result))
+
+    @patch('logging.info')
+    def test_get_node_by_name_validation_workflow(self, mock_logging):
+        """
+        Test complete workflow of validating node names
+        """
+        requested_nodes = ["node-a", "node-b"]
+        killable_nodes = ["node-a", "node-b", "node-c", "node-d"]
+        self.mock_kubecli.list_killable_nodes.return_value = killable_nodes
+
+        result = common_node_functions.get_node_by_name(requested_nodes, self.mock_kubecli)
+
+        self.assertEqual(result, requested_nodes)
+        self.mock_kubecli.list_killable_nodes.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_container_scenario_plugin.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for ContainerScenarioPlugin class
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_container_scenario_plugin.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+from unittest.mock import MagicMock
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
+
+from krkn.scenario_plugins.container.container_scenario_plugin import ContainerScenarioPlugin
+
+
+class TestContainerScenarioPlugin(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Set up test fixtures for ContainerScenarioPlugin
+        """
+        self.plugin = ContainerScenarioPlugin()
+
+    def test_get_scenario_types(self):
+        """
+        Test get_scenario_types returns correct scenario type
+        """
+        result = self.plugin.get_scenario_types()
+
+        self.assertEqual(result, ["container_scenarios"])
+        self.assertEqual(len(result), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_gcp_node_scenarios.py

@@ -0,0 +1,781 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for GCP node scenarios
+
+This test suite covers both the GCP class and gcp_node_scenarios class
+using mocks to avoid actual GCP API calls.
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_gcp_node_scenarios.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+import sys
+from unittest.mock import MagicMock, patch
+
+# Mock external dependencies before any imports that use them
+# Create proper nested mock structure for google modules
+mock_google = MagicMock()
+mock_google_auth = MagicMock()
+mock_google_auth_transport = MagicMock()
+mock_google_cloud = MagicMock()
+mock_google_cloud_compute = MagicMock()
+
+sys.modules['google'] = mock_google
+sys.modules['google.auth'] = mock_google_auth
+sys.modules['google.auth.transport'] = mock_google_auth_transport
+sys.modules['google.auth.transport.requests'] = MagicMock()
+sys.modules['google.cloud'] = mock_google_cloud
+sys.modules['google.cloud.compute_v1'] = mock_google_cloud_compute
+sys.modules['paramiko'] = MagicMock()
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
+from krkn.scenario_plugins.node_actions.gcp_node_scenarios import GCP, gcp_node_scenarios
+
+
+class TestGCP(unittest.TestCase):
+    """Test cases for GCP class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Mock google.auth before creating GCP instance
+        self.auth_patcher = patch('google.auth.default')
+        self.compute_patcher = patch('google.cloud.compute_v1.InstancesClient')
+
+        self.mock_auth = self.auth_patcher.start()
+        self.mock_compute_client = self.compute_patcher.start()
+
+        # Configure auth mock to return credentials and project_id
+        self.mock_auth.return_value = (MagicMock(), 'test-project-123')
+
+        # Create GCP instance with mocked dependencies
+        self.gcp = GCP()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.auth_patcher.stop()
+        self.compute_patcher.stop()
+
+    def test_gcp_init_success(self):
+        """Test GCP class initialization success"""
+        self.assertEqual(self.gcp.project_id, 'test-project-123')
+        self.assertIsNotNone(self.gcp.instance_client)
+
+    def test_gcp_init_failure(self):
+        """Test GCP class initialization failure"""
+        with patch('google.auth.default', side_effect=Exception("Auth error")):
+            with self.assertRaises(Exception):
+                GCP()
+
+    def test_get_node_instance_success(self):
+        """Test getting node instance successfully"""
+        # Create mock instance
+        mock_instance = MagicMock()
+        mock_instance.name = 'gke-cluster-node-1'
+
+        # Create mock response
+        mock_response = MagicMock()
+        mock_response.instances = [mock_instance]
+
+        # Mock aggregated_list to return our mock data
+        self.gcp.instance_client.aggregated_list = MagicMock(
+            return_value=[('zones/us-central1-a', mock_response)]
+        )
+
+        result = self.gcp.get_node_instance('gke-cluster-node-1')
+
+        self.assertEqual(result, mock_instance)
+        self.assertEqual(result.name, 'gke-cluster-node-1')
+
+    def test_get_node_instance_partial_match(self):
+        """Test getting node instance with partial name match"""
+        mock_instance = MagicMock()
+        mock_instance.name = 'node-1'
+
+        mock_response = MagicMock()
+        mock_response.instances = [mock_instance]
+
+        self.gcp.instance_client.aggregated_list = MagicMock(
+            return_value=[('zones/us-central1-a', mock_response)]
+        )
+
+        # instance.name ('node-1') in node ('gke-cluster-node-1-abc') == True
+        result = self.gcp.get_node_instance('gke-cluster-node-1-abc')
+
+        self.assertIsNotNone(result)
+        self.assertEqual(result.name, 'node-1')
+
+    def test_get_node_instance_not_found(self):
+        """Test getting node instance when not found"""
+        mock_response = MagicMock()
+        mock_response.instances = None
+
+        self.gcp.instance_client.aggregated_list = MagicMock(
+            return_value=[('zones/us-central1-a', mock_response)]
+        )
+
+        result = self.gcp.get_node_instance('non-existent-node')
+
+        self.assertIsNone(result)
+
+    def test_get_node_instance_failure(self):
+        """Test getting node instance with failure"""
+        self.gcp.instance_client.aggregated_list = MagicMock(
+            side_effect=Exception("GCP error")
+        )
+
+        with self.assertRaises(Exception):
+            self.gcp.get_node_instance('node-1')
+
+    def test_get_instance_name(self):
+        """Test getting instance name"""
+        mock_instance = MagicMock()
+        mock_instance.name = 'gke-cluster-node-1'
+
+        result = self.gcp.get_instance_name(mock_instance)
+
+        self.assertEqual(result, 'gke-cluster-node-1')
+
+    def test_get_instance_name_none(self):
+        """Test getting instance name when name is None"""
+        mock_instance = MagicMock()
+        mock_instance.name = None
+
+        result = self.gcp.get_instance_name(mock_instance)
+
+        self.assertIsNone(result)
+
+    def test_get_instance_zone(self):
+        """Test getting instance zone"""
+        mock_instance = MagicMock()
+        mock_instance.zone = 'https://www.googleapis.com/compute/v1/projects/test-project/zones/us-central1-a'
+
+        result = self.gcp.get_instance_zone(mock_instance)
+
+        self.assertEqual(result, 'us-central1-a')
+
+    def test_get_instance_zone_none(self):
+        """Test getting instance zone when zone is None"""
+        mock_instance = MagicMock()
+        mock_instance.zone = None
+
+        result = self.gcp.get_instance_zone(mock_instance)
+
+        self.assertIsNone(result)
+
+    def test_get_node_instance_zone(self):
+        """Test getting node instance zone"""
+        mock_instance = MagicMock()
+        mock_instance.name = 'gke-cluster-node-1'
+        mock_instance.zone = 'https://www.googleapis.com/compute/v1/projects/test-project/zones/us-west1-b'
+
+        # Patch get_node_instance to return our mock directly
+        with patch.object(self.gcp, 'get_node_instance', return_value=mock_instance):
+            result = self.gcp.get_node_instance_zone('node-1')
+            self.assertEqual(result, 'us-west1-b')
+
+    def test_get_node_instance_name(self):
+        """Test getting node instance name"""
+        mock_instance = MagicMock()
+        mock_instance.name = 'gke-cluster-node-1'
+
+        # Patch get_node_instance to return our mock directly
+        with patch.object(self.gcp, 'get_node_instance', return_value=mock_instance):
+            result = self.gcp.get_node_instance_name('node-1')
+            self.assertEqual(result, 'gke-cluster-node-1')
+
+    def test_get_instance_id(self):
+        """Test getting instance ID (alias for get_node_instance_name)"""
+        # Patch get_node_instance_name since get_instance_id just calls it
+        with patch.object(self.gcp, 'get_node_instance_name', return_value='gke-cluster-node-1'):
+            result = self.gcp.get_instance_id('node-1')
+            self.assertEqual(result, 'gke-cluster-node-1')
+
+    def test_start_instances_success(self):
+        """Test starting instances successfully"""
+        instance_id = 'gke-cluster-node-1'
+
+        # Mock get_node_instance_zone
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.start = MagicMock()
+
+            self.gcp.start_instances(instance_id)
+
+            self.gcp.instance_client.start.assert_called_once()
+
+    def test_start_instances_failure(self):
+        """Test starting instances with failure"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.start = MagicMock(
+                side_effect=Exception("GCP error")
+            )
+
+            with self.assertRaises(RuntimeError):
+                self.gcp.start_instances(instance_id)
+
+    def test_stop_instances_success(self):
+        """Test stopping instances successfully"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.stop = MagicMock()
+
+            self.gcp.stop_instances(instance_id)
+
+            self.gcp.instance_client.stop.assert_called_once()
+
+    def test_stop_instances_failure(self):
+        """Test stopping instances with failure"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.stop = MagicMock(
+                side_effect=Exception("GCP error")
+            )
+
+            with self.assertRaises(RuntimeError):
+                self.gcp.stop_instances(instance_id)
+
+    def test_suspend_instances_success(self):
+        """Test suspending instances successfully"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.suspend = MagicMock()
+
+            self.gcp.suspend_instances(instance_id)
+
+            self.gcp.instance_client.suspend.assert_called_once()
+
+    def test_suspend_instances_failure(self):
+        """Test suspending instances with failure"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.suspend = MagicMock(
+                side_effect=Exception("GCP error")
+            )
+
+            with self.assertRaises(RuntimeError):
+                self.gcp.suspend_instances(instance_id)
+
+    def test_terminate_instances_success(self):
+        """Test terminating instances successfully"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.delete = MagicMock()
+
+            self.gcp.terminate_instances(instance_id)
+
+            self.gcp.instance_client.delete.assert_called_once()
+
+    def test_terminate_instances_failure(self):
+        """Test terminating instances with failure"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.delete = MagicMock(
+                side_effect=Exception("GCP error")
+            )
+
+            with self.assertRaises(RuntimeError):
+                self.gcp.terminate_instances(instance_id)
+
+    def test_reboot_instances_success(self):
+        """Test rebooting instances successfully"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.reset = MagicMock()
+
+            self.gcp.reboot_instances(instance_id)
+
+            self.gcp.instance_client.reset.assert_called_once()
+
+    def test_reboot_instances_failure(self):
+        """Test rebooting instances with failure"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.reset = MagicMock(
+                side_effect=Exception("GCP error")
+            )
+
+            with self.assertRaises(RuntimeError):
+                self.gcp.reboot_instances(instance_id)
+
+    @patch('time.sleep')
+    def test_get_instance_status_success(self, _mock_sleep):
+        """Test getting instance status successfully"""
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.status = 'RUNNING'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.get = MagicMock(return_value=mock_instance)
+
+            result = self.gcp.get_instance_status(instance_id, 'RUNNING', 60)
+
+            self.assertTrue(result)
+
+    @patch('time.sleep')
+    def test_get_instance_status_timeout(self, _mock_sleep):
+        """Test getting instance status with timeout"""
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.status = 'PROVISIONING'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.get = MagicMock(return_value=mock_instance)
+
+            result = self.gcp.get_instance_status(instance_id, 'RUNNING', 5)
+
+            self.assertFalse(result)
+
+    @patch('time.sleep')
+    def test_get_instance_status_failure(self, _mock_sleep):
+        """Test getting instance status with failure"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_node_instance_zone', return_value='us-central1-a'):
+            self.gcp.instance_client.get = MagicMock(
+                side_effect=Exception("GCP error")
+            )
+
+            with self.assertRaises(RuntimeError):
+                self.gcp.get_instance_status(instance_id, 'RUNNING', 60)
+
+    def test_wait_until_suspended_success(self):
+        """Test waiting until instance is suspended"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_instance_status', return_value=True) as mock_get_status:
+            result = self.gcp.wait_until_suspended(instance_id, 60)
+
+            self.assertTrue(result)
+            mock_get_status.assert_called_once_with(instance_id, 'SUSPENDED', 60)
+
+    def test_wait_until_suspended_failure(self):
+        """Test waiting until instance is suspended with failure"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_instance_status', return_value=False):
+            result = self.gcp.wait_until_suspended(instance_id, 60)
+
+            self.assertFalse(result)
+
+    def test_wait_until_running_success(self):
+        """Test waiting until instance is running successfully"""
+        instance_id = 'gke-cluster-node-1'
+        affected_node = MagicMock(spec=AffectedNode)
+
+        with patch('time.time', side_effect=[100, 110]):
+            with patch.object(self.gcp, 'get_instance_status', return_value=True):
+                result = self.gcp.wait_until_running(instance_id, 60, affected_node)
+
+                self.assertTrue(result)
+                affected_node.set_affected_node_status.assert_called_once_with('running', 10)
+
+    def test_wait_until_running_without_affected_node(self):
+        """Test waiting until running without affected node tracking"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_instance_status', return_value=True):
+            result = self.gcp.wait_until_running(instance_id, 60, None)
+
+            self.assertTrue(result)
+
+    def test_wait_until_stopped_success(self):
+        """Test waiting until instance is stopped successfully"""
+        instance_id = 'gke-cluster-node-1'
+        affected_node = MagicMock(spec=AffectedNode)
+
+        with patch('time.time', side_effect=[100, 115]):
+            with patch.object(self.gcp, 'get_instance_status', return_value=True):
+                result = self.gcp.wait_until_stopped(instance_id, 60, affected_node)
+
+                self.assertTrue(result)
+                affected_node.set_affected_node_status.assert_called_once_with('stopped', 15)
+
+    def test_wait_until_stopped_without_affected_node(self):
+        """Test waiting until stopped without affected node tracking"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_instance_status', return_value=True):
+            result = self.gcp.wait_until_stopped(instance_id, 60, None)
+
+            self.assertTrue(result)
+
+    def test_wait_until_terminated_success(self):
+        """Test waiting until instance is terminated successfully"""
+        instance_id = 'gke-cluster-node-1'
+        affected_node = MagicMock(spec=AffectedNode)
+
+        with patch('time.time', side_effect=[100, 120]):
+            with patch.object(self.gcp, 'get_instance_status', return_value=True):
+                result = self.gcp.wait_until_terminated(instance_id, 60, affected_node)
+
+                self.assertTrue(result)
+                affected_node.set_affected_node_status.assert_called_once_with('terminated', 20)
+
+    def test_wait_until_terminated_without_affected_node(self):
+        """Test waiting until terminated without affected node tracking"""
+        instance_id = 'gke-cluster-node-1'
+
+        with patch.object(self.gcp, 'get_instance_status', return_value=True):
+            result = self.gcp.wait_until_terminated(instance_id, 60, None)
+
+            self.assertTrue(result)
+
+
+class TestGCPNodeScenarios(unittest.TestCase):
+    """Test cases for gcp_node_scenarios class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        self.kubecli = MagicMock(spec=KrknKubernetes)
+        self.affected_nodes_status = AffectedNodeStatus()
+
+        # Mock the GCP class
+        with patch('krkn.scenario_plugins.node_actions.gcp_node_scenarios.GCP') as mock_gcp_class:
+            self.mock_gcp = MagicMock()
+            mock_gcp_class.return_value = self.mock_gcp
+            self.scenario = gcp_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=True,
+                affected_nodes_status=self.affected_nodes_status
+            )
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_success(self, mock_wait_ready):
+        """Test node start scenario successfully"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.name = instance_id
+
+        self.mock_gcp.get_node_instance.return_value = mock_instance
+        self.mock_gcp.get_instance_name.return_value = instance_id
+        self.mock_gcp.start_instances.return_value = None
+        self.mock_gcp.wait_until_running.return_value = True
+
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600,
+            poll_interval=15
+        )
+
+        self.mock_gcp.get_node_instance.assert_called_once_with(node)
+        self.mock_gcp.get_instance_name.assert_called_once_with(mock_instance)
+        self.mock_gcp.start_instances.assert_called_once_with(instance_id)
+        self.mock_gcp.wait_until_running.assert_called_once()
+        mock_wait_ready.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+        self.assertEqual(self.affected_nodes_status.affected_nodes[0].node_name, node)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_no_kube_check(self, mock_wait_ready):
+        """Test node start scenario without kube check"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        # Create scenario with node_action_kube_check=False
+        with patch('krkn.scenario_plugins.node_actions.gcp_node_scenarios.GCP') as mock_gcp_class:
+            mock_gcp = MagicMock()
+            mock_gcp_class.return_value = mock_gcp
+            scenario = gcp_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=False,
+                affected_nodes_status=AffectedNodeStatus()
+            )
+
+            mock_instance = MagicMock()
+            mock_instance.name = instance_id
+
+            mock_gcp.get_node_instance.return_value = mock_instance
+            mock_gcp.get_instance_name.return_value = instance_id
+            mock_gcp.start_instances.return_value = None
+            mock_gcp.wait_until_running.return_value = True
+
+            scenario.node_start_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+            # Should not call wait_for_ready_status
+            mock_wait_ready.assert_not_called()
+
+    def test_node_start_scenario_failure(self):
+        """Test node start scenario with failure"""
+        node = 'gke-cluster-node-1'
+
+        self.mock_gcp.get_node_instance.side_effect = Exception("GCP error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_start_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    def test_node_stop_scenario_success(self, mock_wait_unknown):
+        """Test node stop scenario successfully"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.name = instance_id
+
+        self.mock_gcp.get_node_instance.return_value = mock_instance
+        self.mock_gcp.get_instance_name.return_value = instance_id
+        self.mock_gcp.stop_instances.return_value = None
+        self.mock_gcp.wait_until_stopped.return_value = True
+
+        self.scenario.node_stop_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600,
+            poll_interval=15
+        )
+
+        self.mock_gcp.get_node_instance.assert_called_once_with(node)
+        self.mock_gcp.get_instance_name.assert_called_once_with(mock_instance)
+        self.mock_gcp.stop_instances.assert_called_once_with(instance_id)
+        self.mock_gcp.wait_until_stopped.assert_called_once()
+        mock_wait_unknown.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    def test_node_stop_scenario_no_kube_check(self, mock_wait_unknown):
+        """Test node stop scenario without kube check"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        # Create scenario with node_action_kube_check=False
+        with patch('krkn.scenario_plugins.node_actions.gcp_node_scenarios.GCP') as mock_gcp_class:
+            mock_gcp = MagicMock()
+            mock_gcp_class.return_value = mock_gcp
+            scenario = gcp_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=False,
+                affected_nodes_status=AffectedNodeStatus()
+            )
+
+            mock_instance = MagicMock()
+            mock_instance.name = instance_id
+
+            mock_gcp.get_node_instance.return_value = mock_instance
+            mock_gcp.get_instance_name.return_value = instance_id
+            mock_gcp.stop_instances.return_value = None
+            mock_gcp.wait_until_stopped.return_value = True
+
+            scenario.node_stop_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+            # Should not call wait_for_unknown_status
+            mock_wait_unknown.assert_not_called()
+
+    def test_node_stop_scenario_failure(self):
+        """Test node stop scenario with failure"""
+        node = 'gke-cluster-node-1'
+
+        self.mock_gcp.get_node_instance.side_effect = Exception("GCP error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_stop_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+    @patch('time.sleep')
+    def test_node_termination_scenario_success(self, _mock_sleep):
+        """Test node termination scenario successfully"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.name = instance_id
+
+        self.mock_gcp.get_node_instance.return_value = mock_instance
+        self.mock_gcp.get_instance_name.return_value = instance_id
+        self.mock_gcp.terminate_instances.return_value = None
+        self.mock_gcp.wait_until_terminated.return_value = True
+        self.kubecli.list_nodes.return_value = []
+
+        self.scenario.node_termination_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600,
+            poll_interval=15
+        )
+
+        self.mock_gcp.get_node_instance.assert_called_once_with(node)
+        self.mock_gcp.get_instance_name.assert_called_once_with(mock_instance)
+        self.mock_gcp.terminate_instances.assert_called_once_with(instance_id)
+        self.mock_gcp.wait_until_terminated.assert_called_once()
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('time.sleep')
+    def test_node_termination_scenario_node_still_exists(self, _mock_sleep):
+        """Test node termination scenario when node still exists"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.name = instance_id
+
+        self.mock_gcp.get_node_instance.return_value = mock_instance
+        self.mock_gcp.get_instance_name.return_value = instance_id
+        self.mock_gcp.terminate_instances.return_value = None
+        self.mock_gcp.wait_until_terminated.return_value = True
+        # Node still in list after timeout
+        self.kubecli.list_nodes.return_value = [node]
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_termination_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=2,
+                poll_interval=15
+            )
+
+    def test_node_termination_scenario_failure(self):
+        """Test node termination scenario with failure"""
+        node = 'gke-cluster-node-1'
+
+        self.mock_gcp.get_node_instance.side_effect = Exception("GCP error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_termination_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600,
+                poll_interval=15
+            )
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_reboot_scenario_success(self, mock_wait_ready, mock_wait_unknown):
+        """Test node reboot scenario successfully"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.name = instance_id
+
+        self.mock_gcp.get_node_instance.return_value = mock_instance
+        self.mock_gcp.get_instance_name.return_value = instance_id
+        self.mock_gcp.reboot_instances.return_value = None
+        self.mock_gcp.wait_until_running.return_value = True
+
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node=node,
+            timeout=600
+        )
+
+        self.mock_gcp.get_node_instance.assert_called_once_with(node)
+        self.mock_gcp.get_instance_name.assert_called_once_with(mock_instance)
+        self.mock_gcp.reboot_instances.assert_called_once_with(instance_id)
+        self.mock_gcp.wait_until_running.assert_called_once()
+        # Should be called twice in GCP implementation
+        self.assertEqual(mock_wait_unknown.call_count, 1)
+        self.assertEqual(mock_wait_ready.call_count, 1)
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_reboot_scenario_no_kube_check(self, mock_wait_ready, mock_wait_unknown):
+        """Test node reboot scenario without kube check"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        # Create scenario with node_action_kube_check=False
+        with patch('krkn.scenario_plugins.node_actions.gcp_node_scenarios.GCP') as mock_gcp_class:
+            mock_gcp = MagicMock()
+            mock_gcp_class.return_value = mock_gcp
+            scenario = gcp_node_scenarios(
+                kubecli=self.kubecli,
+                node_action_kube_check=False,
+                affected_nodes_status=AffectedNodeStatus()
+            )
+
+            mock_instance = MagicMock()
+            mock_instance.name = instance_id
+
+            mock_gcp.get_node_instance.return_value = mock_instance
+            mock_gcp.get_instance_name.return_value = instance_id
+            mock_gcp.reboot_instances.return_value = None
+            mock_gcp.wait_until_running.return_value = True
+
+            scenario.node_reboot_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600
+            )
+
+            # Should not call wait functions
+            mock_wait_unknown.assert_not_called()
+            mock_wait_ready.assert_not_called()
+
+    def test_node_reboot_scenario_failure(self):
+        """Test node reboot scenario with failure"""
+        node = 'gke-cluster-node-1'
+
+        self.mock_gcp.get_node_instance.side_effect = Exception("GCP error")
+
+        with self.assertRaises(RuntimeError):
+            self.scenario.node_reboot_scenario(
+                instance_kill_count=1,
+                node=node,
+                timeout=600
+            )
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_multiple_kills(self, mock_wait_ready):
+        """Test node start scenario with multiple kill counts"""
+        node = 'gke-cluster-node-1'
+        instance_id = 'gke-cluster-node-1'
+
+        mock_instance = MagicMock()
+        mock_instance.name = instance_id
+
+        self.mock_gcp.get_node_instance.return_value = mock_instance
+        self.mock_gcp.get_instance_name.return_value = instance_id
+        self.mock_gcp.start_instances.return_value = None
+        self.mock_gcp.wait_until_running.return_value = True
+
+        self.scenario.node_start_scenario(
+            instance_kill_count=3,
+            node=node,
+            timeout=600,
+            poll_interval=15
+        )
+
+        self.assertEqual(self.mock_gcp.start_instances.call_count, 3)
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 3)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_health_checker.py

@@ -0,0 +1,503 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for HealthChecker class
+
+This test file provides comprehensive coverage for the main functionality of HealthChecker:
+- HTTP request making with various authentication methods
+- Health check monitoring with status tracking
+- Failure detection and recovery tracking
+- Exit on failure behavior
+- Telemetry collection
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_health_checker.py -v
+
+Assisted By: Claude Code
+"""
+
+import queue
+import unittest
+from datetime import datetime
+from unittest.mock import MagicMock, patch
+
+from krkn_lib.models.telemetry.models import HealthCheck
+
+from krkn.utils.HealthChecker import HealthChecker
+
+
+class TestHealthChecker(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Set up test fixtures for HealthChecker
+        """
+        self.checker = HealthChecker(iterations=5)
+        self.health_check_queue = queue.Queue()
+
+    def tearDown(self):
+        """
+        Clean up after each test
+        """
+        self.checker.current_iterations = 0
+        self.checker.ret_value = 0
+
+    def make_increment_side_effect(self, response_data):
+        """
+        Helper to create a side effect that increments current_iterations
+        """
+        def side_effect(*args, **kwargs):
+            self.checker.current_iterations += 1
+            return response_data
+        return side_effect
+
+    @patch('requests.get')
+    def test_make_request_success(self, mock_get):
+        """
+        Test make_request returns success for 200 status code
+        """
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_get.return_value = mock_response
+
+        result = self.checker.make_request("http://example.com")
+
+        self.assertEqual(result["url"], "http://example.com")
+        self.assertEqual(result["status"], True)
+        self.assertEqual(result["status_code"], 200)
+        mock_get.assert_called_once_with(
+            "http://example.com",
+            auth=None,
+            headers=None,
+            verify=True,
+            timeout=3
+        )
+
+    @patch('requests.get')
+    def test_make_request_with_auth(self, mock_get):
+        """
+        Test make_request with basic authentication
+        """
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_get.return_value = mock_response
+
+        auth = ("user", "pass")
+        result = self.checker.make_request("http://example.com", auth=auth)
+
+        self.assertEqual(result["status"], True)
+        mock_get.assert_called_once_with(
+            "http://example.com",
+            auth=auth,
+            headers=None,
+            verify=True,
+            timeout=3
+        )
+
+    @patch('requests.get')
+    def test_make_request_with_bearer_token(self, mock_get):
+        """
+        Test make_request with bearer token authentication
+        """
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_get.return_value = mock_response
+
+        headers = {"Authorization": "Bearer token123"}
+        result = self.checker.make_request("http://example.com", headers=headers)
+
+        self.assertEqual(result["status"], True)
+        mock_get.assert_called_once_with(
+            "http://example.com",
+            auth=None,
+            headers=headers,
+            verify=True,
+            timeout=3
+        )
+
+    @patch('requests.get')
+    def test_make_request_failure(self, mock_get):
+        """
+        Test make_request returns failure for non-200 status code
+        """
+        mock_response = MagicMock()
+        mock_response.status_code = 500
+        mock_get.return_value = mock_response
+
+        result = self.checker.make_request("http://example.com")
+
+        self.assertEqual(result["status"], False)
+        self.assertEqual(result["status_code"], 500)
+
+    @patch('requests.get')
+    def test_make_request_with_verify_false(self, mock_get):
+        """
+        Test make_request with SSL verification disabled
+        """
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_get.return_value = mock_response
+
+        result = self.checker.make_request("https://example.com", verify=False)
+
+        self.assertEqual(result["status"], True)
+        mock_get.assert_called_once_with(
+            "https://example.com",
+            auth=None,
+            headers=None,
+            verify=False,
+            timeout=3
+        )
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_empty_config(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check with empty config skips checks
+        """
+        config = {
+            "config": [],
+            "interval": 2
+        }
+
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        mock_make_request.assert_not_called()
+        self.assertTrue(self.health_check_queue.empty())
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_successful_requests(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check with all successful requests
+        """
+        mock_make_request.side_effect = self.make_increment_side_effect({
+            "url": "http://example.com",
+            "status": True,
+            "status_code": 200
+        })
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": False
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 2
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # Should have telemetry
+        self.assertFalse(self.health_check_queue.empty())
+        telemetry = self.health_check_queue.get()
+        self.assertEqual(len(telemetry), 1)
+        self.assertEqual(telemetry[0].status, True)
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_failure_then_recovery(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check detects failure and recovery
+        """
+        # Create side effects that increment and return different values
+        call_count = [0]
+        def side_effect(*args, **kwargs):
+            self.checker.current_iterations += 1
+            call_count[0] += 1
+            if call_count[0] == 1:
+                return {"url": "http://example.com", "status": False, "status_code": 500}
+            else:
+                return {"url": "http://example.com", "status": True, "status_code": 200}
+
+        mock_make_request.side_effect = side_effect
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": False
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 3
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # Should have telemetry showing failure period
+        self.assertFalse(self.health_check_queue.empty())
+        telemetry = self.health_check_queue.get()
+
+        # Should have at least 2 entries: one for failure period, one for success period
+        self.assertGreaterEqual(len(telemetry), 1)
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_with_bearer_token(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check correctly handles bearer token
+        """
+        mock_make_request.side_effect = self.make_increment_side_effect({
+            "url": "http://example.com",
+            "status": True,
+            "status_code": 200
+        })
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": "test-token-123",
+                    "auth": None,
+                    "exit_on_failure": False
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 1
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # Verify bearer token was added to headers
+        # make_request is called as: make_request(url, auth, headers, verify_url)
+        call_args = mock_make_request.call_args
+        self.assertEqual(call_args[0][2]['Authorization'], "Bearer test-token-123")
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_with_auth(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check correctly handles basic auth
+        """
+        mock_make_request.side_effect = self.make_increment_side_effect({
+            "url": "http://example.com",
+            "status": True,
+            "status_code": 200
+        })
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": None,
+                    "auth": "user,pass",
+                    "exit_on_failure": False
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 1
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # Verify auth tuple was created correctly
+        # make_request is called as: make_request(url, auth, headers, verify_url)
+        call_args = mock_make_request.call_args
+        self.assertEqual(call_args[0][1], ("user", "pass"))
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_exit_on_failure(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check sets ret_value=2 when exit_on_failure is True
+        """
+        mock_make_request.side_effect = self.make_increment_side_effect({
+            "url": "http://example.com",
+            "status": False,
+            "status_code": 500
+        })
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": True
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 1
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # ret_value should be set to 2 on failure
+        self.assertEqual(self.checker.ret_value, 2)
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_exit_on_failure_not_set_on_success(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check does not set ret_value when request succeeds
+        """
+        mock_make_request.side_effect = self.make_increment_side_effect({
+            "url": "http://example.com",
+            "status": True,
+            "status_code": 200
+        })
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": True
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 1
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # ret_value should remain 0 on success
+        self.assertEqual(self.checker.ret_value, 0)
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_with_verify_url_false(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check respects verify_url setting
+        """
+        mock_make_request.side_effect = self.make_increment_side_effect({
+            "url": "https://example.com",
+            "status": True,
+            "status_code": 200
+        })
+
+        config = {
+            "config": [
+                {
+                    "url": "https://example.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": False,
+                    "verify_url": False
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 1
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # Verify that verify parameter was set to False
+        # make_request is called as: make_request(url, auth, headers, verify_url)
+        call_args = mock_make_request.call_args
+        self.assertEqual(call_args[0][3], False)
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_exception_handling(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check handles exceptions during requests
+        """
+        # Simulate exception during request but also increment to avoid infinite loop
+        def side_effect(*args, **kwargs):
+            self.checker.current_iterations += 1
+            raise Exception("Connection error")
+
+        mock_make_request.side_effect = side_effect
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": False
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 1
+
+        # Should not raise exception
+        self.checker.run_health_check(config, self.health_check_queue)
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_multiple_urls(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check with multiple URLs
+        """
+        call_count = [0]
+        def side_effect(*args, **kwargs):
+            call_count[0] += 1
+            # Increment only after both URLs are called (one iteration)
+            if call_count[0] % 2 == 0:
+                self.checker.current_iterations += 1
+            return {
+                "status": True,
+                "status_code": 200
+            }
+
+        mock_make_request.side_effect = side_effect
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example1.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": False
+                },
+                {
+                    "url": "http://example2.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": False
+                }
+            ],
+            "interval": 0.01
+        }
+
+        self.checker.iterations = 1
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # Should have called make_request for both URLs
+        self.assertEqual(mock_make_request.call_count, 2)
+
+    @patch('krkn.utils.HealthChecker.HealthChecker.make_request')
+    @patch('time.sleep')
+    def test_run_health_check_custom_interval(self, mock_sleep, mock_make_request):
+        """
+        Test run_health_check uses custom interval
+        """
+        mock_make_request.side_effect = self.make_increment_side_effect({
+            "url": "http://example.com",
+            "status": True,
+            "status_code": 200
+        })
+
+        config = {
+            "config": [
+                {
+                    "url": "http://example.com",
+                    "bearer_token": None,
+                    "auth": None,
+                    "exit_on_failure": False
+                }
+            ],
+            "interval": 5
+        }
+
+        self.checker.iterations = 2
+        self.checker.run_health_check(config, self.health_check_queue)
+
+        # Verify sleep was called with custom interval
+        mock_sleep.assert_called_with(5)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_hogs_scenario_plugin.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for HogsScenarioPlugin class
+
+Usage:
+    python -m coverage run -a -m unittest tests/test_hogs_scenario_plugin.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+from unittest.mock import MagicMock
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
+
+from krkn.scenario_plugins.hogs.hogs_scenario_plugin import HogsScenarioPlugin
+
+
+class TestHogsScenarioPlugin(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Set up test fixtures for HogsScenarioPlugin
+        """
+        self.plugin = HogsScenarioPlugin()
+
+    def test_get_scenario_types(self):
+        """
+        Test get_scenario_types returns correct scenario type
+        """
+        result = self.plugin.get_scenario_types()
+
+        self.assertEqual(result, ["hog_scenarios"])
+        self.assertEqual(len(result), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_ibmcloud_node_scenarios.py

@@ -0,0 +1,637 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for IBM Cloud VPC node scenarios
+
+This test suite covers both the IbmCloud class and ibm_node_scenarios class
+using mocks to avoid actual IBM Cloud API calls.
+
+IMPORTANT: These tests use comprehensive mocking and do NOT require any cloud provider
+settings or credentials. No environment variables need to be set. All API clients and
+external dependencies are mocked.
+
+Test Coverage:
+- TestIbmCloud: 30 tests for the IbmCloud VPC API client class
+  - Initialization, SSL configuration, instance operations (start/stop/reboot/delete)
+  - Status checking, wait operations, error handling
+- TestIbmNodeScenarios: 14 tests for node scenario orchestration
+  - Node start/stop/reboot/terminate scenarios
+  - Exception handling, multiple kill counts
+
+Usage:
+    # Run all tests
+    python -m unittest tests.test_ibmcloud_node_scenarios -v
+
+    # Run with coverage
+    python -m coverage run -a -m unittest tests/test_ibmcloud_node_scenarios.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+import sys
+import json
+from unittest.mock import MagicMock, patch, Mock
+
+# Mock paramiko and IBM SDK before importing
+sys.modules['paramiko'] = MagicMock()
+sys.modules['ibm_vpc'] = MagicMock()
+sys.modules['ibm_cloud_sdk_core'] = MagicMock()
+sys.modules['ibm_cloud_sdk_core.authenticators'] = MagicMock()
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
+from krkn.scenario_plugins.node_actions.ibmcloud_node_scenarios import (
+    IbmCloud,
+    ibm_node_scenarios
+)
+
+
+class TestIbmCloud(unittest.TestCase):
+    """Test cases for IbmCloud class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Set up environment variables
+        self.env_patcher = patch.dict('os.environ', {
+            'IBMC_APIKEY': 'test-api-key',
+            'IBMC_URL': 'https://test.cloud.ibm.com'
+        })
+        self.env_patcher.start()
+
+        # Mock IBM VPC client
+        self.mock_vpc = MagicMock()
+        self.vpc_patcher = patch('krkn.scenario_plugins.node_actions.ibmcloud_node_scenarios.VpcV1')
+        self.mock_vpc_class = self.vpc_patcher.start()
+        self.mock_vpc_class.return_value = self.mock_vpc
+
+        # Mock IAMAuthenticator
+        self.auth_patcher = patch('krkn.scenario_plugins.node_actions.ibmcloud_node_scenarios.IAMAuthenticator')
+        self.mock_auth = self.auth_patcher.start()
+
+        # Create IbmCloud instance
+        self.ibm = IbmCloud()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.env_patcher.stop()
+        self.vpc_patcher.stop()
+        self.auth_patcher.stop()
+
+    def test_init_success(self):
+        """Test IbmCloud class initialization"""
+        self.assertIsNotNone(self.ibm.service)
+        self.mock_vpc.set_service_url.assert_called_once_with('https://test.cloud.ibm.com')
+
+    def test_init_missing_api_key(self):
+        """Test initialization fails when IBMC_APIKEY is missing"""
+        with patch.dict('os.environ', {
+            'IBMC_URL': 'https://test.cloud.ibm.com'
+        }, clear=True):
+            with self.assertRaises(Exception) as context:
+                IbmCloud()
+            self.assertIn("IBMC_APIKEY", str(context.exception))
+
+    def test_init_missing_url(self):
+        """Test initialization fails when IBMC_URL is missing"""
+        with patch.dict('os.environ', {
+            'IBMC_APIKEY': 'test-api-key'
+        }, clear=True):
+            with self.assertRaises(Exception) as context:
+                IbmCloud()
+            self.assertIn("IBMC_URL", str(context.exception))
+
+    def test_configure_ssl_verification_disabled(self):
+        """Test disabling SSL verification"""
+        self.ibm.configure_ssl_verification(True)
+        self.mock_vpc.set_disable_ssl_verification.assert_called_with(True)
+
+    def test_configure_ssl_verification_enabled(self):
+        """Test enabling SSL verification"""
+        self.ibm.configure_ssl_verification(False)
+        self.mock_vpc.set_disable_ssl_verification.assert_called_with(False)
+
+    def test_get_instance_id_success(self):
+        """Test getting instance ID by node name"""
+        mock_list = [
+            {'vpc_name': 'test-node-1', 'vpc_id': 'vpc-1'},
+            {'vpc_name': 'test-node-2', 'vpc_id': 'vpc-2'}
+        ]
+
+        with patch.object(self.ibm, 'list_instances', return_value=mock_list):
+            instance_id = self.ibm.get_instance_id('test-node-1')
+            self.assertEqual(instance_id, 'vpc-1')
+
+    def test_get_instance_id_not_found(self):
+        """Test getting instance ID when node not found"""
+        mock_list = [
+            {'vpc_name': 'test-node-1', 'vpc_id': 'vpc-1'}
+        ]
+
+        with patch.object(self.ibm, 'list_instances', return_value=mock_list):
+            with self.assertRaises(SystemExit):
+                self.ibm.get_instance_id('non-existent-node')
+
+    def test_delete_instance_success(self):
+        """Test deleting instance successfully"""
+        self.mock_vpc.delete_instance.return_value = None
+
+        result = self.ibm.delete_instance('vpc-123')
+
+        self.mock_vpc.delete_instance.assert_called_once_with('vpc-123')
+        # Method doesn't explicitly return True, so we just verify no exception
+
+    def test_delete_instance_failure(self):
+        """Test deleting instance with failure"""
+        self.mock_vpc.delete_instance.side_effect = Exception("API Error")
+
+        result = self.ibm.delete_instance('vpc-123')
+
+        self.assertEqual(result, False)
+
+    def test_reboot_instances_success(self):
+        """Test rebooting instance successfully"""
+        self.mock_vpc.create_instance_action.return_value = None
+
+        result = self.ibm.reboot_instances('vpc-123')
+
+        self.assertTrue(result)
+        self.mock_vpc.create_instance_action.assert_called_once_with(
+            'vpc-123',
+            type='reboot'
+        )
+
+    def test_reboot_instances_failure(self):
+        """Test rebooting instance with failure"""
+        self.mock_vpc.create_instance_action.side_effect = Exception("API Error")
+
+        result = self.ibm.reboot_instances('vpc-123')
+
+        self.assertEqual(result, False)
+
+    def test_stop_instances_success(self):
+        """Test stopping instance successfully"""
+        self.mock_vpc.create_instance_action.return_value = None
+
+        result = self.ibm.stop_instances('vpc-123')
+
+        self.assertTrue(result)
+        self.mock_vpc.create_instance_action.assert_called_once_with(
+            'vpc-123',
+            type='stop'
+        )
+
+    def test_stop_instances_failure(self):
+        """Test stopping instance with failure"""
+        self.mock_vpc.create_instance_action.side_effect = Exception("API Error")
+
+        result = self.ibm.stop_instances('vpc-123')
+
+        self.assertEqual(result, False)
+
+    def test_start_instances_success(self):
+        """Test starting instance successfully"""
+        self.mock_vpc.create_instance_action.return_value = None
+
+        result = self.ibm.start_instances('vpc-123')
+
+        self.assertTrue(result)
+        self.mock_vpc.create_instance_action.assert_called_once_with(
+            'vpc-123',
+            type='start'
+        )
+
+    def test_start_instances_failure(self):
+        """Test starting instance with failure"""
+        self.mock_vpc.create_instance_action.side_effect = Exception("API Error")
+
+        result = self.ibm.start_instances('vpc-123')
+
+        self.assertEqual(result, False)
+
+    def test_list_instances_success(self):
+        """Test listing instances successfully"""
+        mock_result = Mock()
+        mock_result.get_result.return_value = {
+            'instances': [
+                {'name': 'node-1', 'id': 'vpc-1'},
+                {'name': 'node-2', 'id': 'vpc-2'}
+            ],
+            'total_count': 2,
+            'limit': 50
+        }
+        self.mock_vpc.list_instances.return_value = mock_result
+
+        instances = self.ibm.list_instances()
+
+        self.assertEqual(len(instances), 2)
+        self.assertEqual(instances[0]['vpc_name'], 'node-1')
+        self.assertEqual(instances[1]['vpc_name'], 'node-2')
+
+    def test_list_instances_with_pagination(self):
+        """Test listing instances with pagination"""
+        # First call returns limit reached
+        mock_result_1 = Mock()
+        mock_result_1.get_result.return_value = {
+            'instances': [
+                {'name': 'node-1', 'id': 'vpc-1'}
+            ],
+            'total_count': 1,
+            'limit': 1
+        }
+
+        # Second call returns remaining
+        mock_result_2 = Mock()
+        mock_vpc_2 = type('obj', (object,), {'name': 'node-2', 'id': 'vpc-2'})
+        mock_result_2.get_result.return_value = {
+            'instances': [mock_vpc_2],
+            'total_count': 1,
+            'limit': 50
+        }
+
+        self.mock_vpc.list_instances.side_effect = [mock_result_1, mock_result_2]
+
+        instances = self.ibm.list_instances()
+
+        self.assertEqual(len(instances), 2)
+        self.assertEqual(self.mock_vpc.list_instances.call_count, 2)
+
+    def test_list_instances_failure(self):
+        """Test listing instances with failure"""
+        self.mock_vpc.list_instances.side_effect = Exception("API Error")
+
+        with self.assertRaises(SystemExit):
+            self.ibm.list_instances()
+
+    def test_find_id_in_list(self):
+        """Test finding ID in VPC list"""
+        vpc_list = [
+            {'vpc_name': 'vpc-1', 'vpc_id': 'id-1'},
+            {'vpc_name': 'vpc-2', 'vpc_id': 'id-2'}
+        ]
+
+        vpc_id = self.ibm.find_id_in_list('vpc-2', vpc_list)
+
+        self.assertEqual(vpc_id, 'id-2')
+
+    def test_find_id_in_list_not_found(self):
+        """Test finding ID in VPC list when not found"""
+        vpc_list = [
+            {'vpc_name': 'vpc-1', 'vpc_id': 'id-1'}
+        ]
+
+        vpc_id = self.ibm.find_id_in_list('vpc-3', vpc_list)
+
+        self.assertIsNone(vpc_id)
+
+    def test_get_instance_status_success(self):
+        """Test getting instance status successfully"""
+        mock_result = Mock()
+        mock_result.get_result.return_value = {'status': 'running'}
+        self.mock_vpc.get_instance.return_value = mock_result
+
+        status = self.ibm.get_instance_status('vpc-123')
+
+        self.assertEqual(status, 'running')
+
+    def test_get_instance_status_failure(self):
+        """Test getting instance status with failure"""
+        self.mock_vpc.get_instance.side_effect = Exception("API Error")
+
+        status = self.ibm.get_instance_status('vpc-123')
+
+        self.assertIsNone(status)
+
+    def test_wait_until_deleted_success(self):
+        """Test waiting until instance is deleted"""
+        # First call returns status, second returns None (deleted)
+        with patch.object(self.ibm, 'get_instance_status', side_effect=['deleting', None]):
+            affected_node = MagicMock(spec=AffectedNode)
+
+            with patch('time.time', side_effect=[100, 105]), \
+                 patch('time.sleep'):
+                result = self.ibm.wait_until_deleted('vpc-123', timeout=60, affected_node=affected_node)
+
+            self.assertTrue(result)
+            affected_node.set_affected_node_status.assert_called_once_with("terminated", 5)
+
+    def test_wait_until_deleted_timeout(self):
+        """Test waiting until deleted with timeout"""
+        with patch.object(self.ibm, 'get_instance_status', return_value='deleting'):
+            with patch('time.sleep'):
+                result = self.ibm.wait_until_deleted('vpc-123', timeout=5)
+
+            self.assertFalse(result)
+
+    def test_wait_until_running_success(self):
+        """Test waiting until instance is running"""
+        with patch.object(self.ibm, 'get_instance_status', side_effect=['starting', 'running']):
+            affected_node = MagicMock(spec=AffectedNode)
+
+            with patch('time.time', side_effect=[100, 105]), \
+                 patch('time.sleep'):
+                result = self.ibm.wait_until_running('vpc-123', timeout=60, affected_node=affected_node)
+
+            self.assertTrue(result)
+            affected_node.set_affected_node_status.assert_called_once_with("running", 5)
+
+    def test_wait_until_running_timeout(self):
+        """Test waiting until running with timeout"""
+        with patch.object(self.ibm, 'get_instance_status', return_value='starting'):
+            with patch('time.sleep'):
+                result = self.ibm.wait_until_running('vpc-123', timeout=5)
+
+            self.assertFalse(result)
+
+    def test_wait_until_stopped_success(self):
+        """Test waiting until instance is stopped"""
+        with patch.object(self.ibm, 'get_instance_status', side_effect=['stopping', 'stopped']):
+            affected_node = MagicMock(spec=AffectedNode)
+
+            with patch('time.time', side_effect=[100, 105]), \
+                 patch('time.sleep'):
+                result = self.ibm.wait_until_stopped('vpc-123', timeout=60, affected_node=affected_node)
+
+            self.assertTrue(result)
+            affected_node.set_affected_node_status.assert_called_once_with("stopped", 5)
+
+    def test_wait_until_stopped_timeout(self):
+        """Test waiting until stopped with timeout"""
+        with patch.object(self.ibm, 'get_instance_status', return_value='stopping'):
+            with patch('time.sleep'):
+                result = self.ibm.wait_until_stopped('vpc-123', timeout=5, affected_node=None)
+
+            self.assertFalse(result)
+
+    def test_wait_until_rebooted_success(self):
+        """Test waiting until instance is rebooted"""
+        # First call checks reboot status (not 'starting'), second call in wait_until_running checks status
+        with patch.object(self.ibm, 'get_instance_status', side_effect=['running', 'running']):
+            affected_node = MagicMock(spec=AffectedNode)
+
+            time_values = [100, 105, 110]
+            with patch('time.time', side_effect=time_values), \
+                 patch('time.sleep'):
+                result = self.ibm.wait_until_rebooted('vpc-123', timeout=60, affected_node=affected_node)
+
+            self.assertTrue(result)
+
+    def test_wait_until_rebooted_timeout(self):
+        """Test waiting until rebooted with timeout"""
+        with patch.object(self.ibm, 'get_instance_status', return_value='starting'):
+            with patch('time.sleep'):
+                result = self.ibm.wait_until_rebooted('vpc-123', timeout=5, affected_node=None)
+
+            self.assertFalse(result)
+
+
+class TestIbmNodeScenarios(unittest.TestCase):
+    """Test cases for ibm_node_scenarios class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Mock KrknKubernetes
+        self.mock_kubecli = MagicMock(spec=KrknKubernetes)
+        self.affected_nodes_status = AffectedNodeStatus()
+
+        # Mock the IbmCloud class entirely to avoid any real API calls
+        self.ibm_cloud_patcher = patch('krkn.scenario_plugins.node_actions.ibmcloud_node_scenarios.IbmCloud')
+        self.mock_ibm_cloud_class = self.ibm_cloud_patcher.start()
+
+        # Create a mock instance that will be returned when IbmCloud() is called
+        self.mock_ibm_cloud_instance = MagicMock()
+        self.mock_ibm_cloud_class.return_value = self.mock_ibm_cloud_instance
+
+        # Create ibm_node_scenarios instance
+        self.scenario = ibm_node_scenarios(
+            kubecli=self.mock_kubecli,
+            node_action_kube_check=True,
+            affected_nodes_status=self.affected_nodes_status,
+            disable_ssl_verification=False
+        )
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.ibm_cloud_patcher.stop()
+
+    def test_init(self):
+        """Test ibm_node_scenarios initialization"""
+        self.assertIsNotNone(self.scenario.ibmcloud)
+        self.assertTrue(self.scenario.node_action_kube_check)
+        self.assertEqual(self.scenario.kubecli, self.mock_kubecli)
+
+    def test_init_with_ssl_disabled(self):
+        """Test initialization with SSL verification disabled"""
+        scenario = ibm_node_scenarios(
+            kubecli=self.mock_kubecli,
+            node_action_kube_check=True,
+            affected_nodes_status=self.affected_nodes_status,
+            disable_ssl_verification=True
+        )
+
+        # Verify configure_ssl_verification was called
+        self.mock_ibm_cloud_instance.configure_ssl_verification.assert_called_with(True)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_success(self, mock_wait_ready):
+        """Test node start scenario successfully"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.start_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_running.return_value = True
+
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+        self.assertEqual(self.affected_nodes_status.affected_nodes[0].node_name, 'test-node')
+        mock_wait_ready.assert_called_once()
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_no_kube_check(self, mock_wait_ready):
+        """Test node start scenario without Kubernetes check"""
+        self.scenario.node_action_kube_check = False
+
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.start_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_running.return_value = True
+
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        mock_wait_ready.assert_not_called()
+
+    def test_node_stop_scenario_success(self):
+        """Test node stop scenario successfully"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.stop_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_stopped.return_value = True
+
+        self.scenario.node_stop_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    def test_node_stop_scenario_failure(self):
+        """Test node stop scenario with stop command failure"""
+        # Configure mock - get_instance_id succeeds but stop_instances fails
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.stop_instances.return_value = False
+
+        # Code raises exception inside try/except, so it should be caught and logged
+        self.scenario.node_stop_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        # Verify that affected nodes were not appended since exception was caught
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 0)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_reboot_scenario_success(self, mock_wait_ready, mock_wait_unknown):
+        """Test node reboot scenario successfully"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.reboot_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_rebooted.return_value = True
+
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            soft_reboot=False
+        )
+
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+        mock_wait_unknown.assert_called_once()
+        mock_wait_ready.assert_called_once()
+
+    def test_node_reboot_scenario_failure(self):
+        """Test node reboot scenario with reboot command failure"""
+        # Configure mock - get_instance_id succeeds but reboot_instances fails
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.reboot_instances.return_value = False
+
+        # Code raises exception inside try/except, so it should be caught and logged
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            soft_reboot=False
+        )
+
+        # Verify that affected nodes were not appended since exception was caught
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 0)
+
+    def test_node_terminate_scenario_success(self):
+        """Test node terminate scenario successfully"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.delete_instance.return_value = None
+        self.mock_ibm_cloud_instance.wait_until_deleted.return_value = True
+
+        self.scenario.node_terminate_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    def test_node_scenario_multiple_kill_count(self):
+        """Test node scenario with multiple kill count"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.stop_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_stopped.return_value = True
+
+        self.scenario.node_stop_scenario(
+            instance_kill_count=2,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        # Should have 2 affected nodes for 2 iterations
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 2)
+
+    def test_node_start_scenario_exception(self):
+        """Test node start scenario with exception during operation"""
+        # Configure mock - get_instance_id succeeds but start_instances fails
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.start_instances.side_effect = Exception("API Error")
+
+        # Should handle exception gracefully
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        # Verify affected node still added even on failure
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    def test_node_stop_scenario_exception(self):
+        """Test node stop scenario with exception"""
+        # Configure mock to raise SystemExit
+        self.mock_ibm_cloud_instance.get_instance_id.side_effect = SystemExit(1)
+
+        # Should handle system exit gracefully
+        with self.assertRaises(SystemExit):
+            self.scenario.node_stop_scenario(
+                instance_kill_count=1,
+                node='test-node',
+                timeout=60,
+                poll_interval=5
+            )
+
+    def test_node_reboot_scenario_exception(self):
+        """Test node reboot scenario with exception during operation"""
+        # Configure mock - get_instance_id succeeds but reboot_instances fails
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.reboot_instances.side_effect = Exception("API Error")
+
+        # Should handle exception gracefully
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            soft_reboot=False
+        )
+
+    def test_node_terminate_scenario_exception(self):
+        """Test node terminate scenario with exception"""
+        # Configure mock - get_instance_id succeeds but delete_instance fails
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'vpc-123'
+        self.mock_ibm_cloud_instance.delete_instance.side_effect = Exception("API Error")
+
+        # Should handle exception gracefully
+        self.scenario.node_terminate_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_ibmcloud_power_node_scenarios.py

@@ -0,0 +1,673 @@
+#!/usr/bin/env python3
+
+"""
+Test suite for IBM Cloud Power node scenarios
+
+This test suite covers both the IbmCloudPower class and ibmcloud_power_node_scenarios class
+using mocks to avoid actual IBM Cloud API calls.
+
+IMPORTANT: These tests use comprehensive mocking and do NOT require any cloud provider
+settings or credentials. No environment variables need to be set. All API clients and
+external dependencies are mocked.
+
+Test Coverage:
+- TestIbmCloudPower: 31 tests for the IbmCloudPower API client class
+  - Authentication, instance operations (start/stop/reboot/delete)
+  - Status checking, wait operations, error handling
+- TestIbmCloudPowerNodeScenarios: 10 tests for node scenario orchestration
+  - Node start/stop/reboot/terminate scenarios
+  - Exception handling, multiple kill counts
+
+Usage:
+    # Run all tests
+    python -m unittest tests.test_ibmcloud_power_node_scenarios -v
+
+    # Run with coverage
+    python -m coverage run -a -m unittest tests/test_ibmcloud_power_node_scenarios.py -v
+
+Assisted By: Claude Code
+"""
+
+import unittest
+import sys
+import json
+from unittest.mock import MagicMock, patch, Mock
+
+# Mock paramiko before importing
+sys.modules['paramiko'] = MagicMock()
+
+from krkn_lib.k8s import KrknKubernetes
+from krkn_lib.models.k8s import AffectedNode, AffectedNodeStatus
+from krkn.scenario_plugins.node_actions.ibmcloud_power_node_scenarios import (
+    IbmCloudPower,
+    ibmcloud_power_node_scenarios
+)
+
+
+class TestIbmCloudPower(unittest.TestCase):
+    """Test cases for IbmCloudPower class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Set up environment variables
+        self.env_patcher = patch.dict('os.environ', {
+            'IBMC_APIKEY': 'test-api-key',
+            'IBMC_POWER_URL': 'https://test.cloud.ibm.com',
+            'IBMC_POWER_CRN': 'crn:v1:bluemix:public:power-iaas:us-south:a/abc123:instance-id::'
+        })
+        self.env_patcher.start()
+
+        # Mock requests
+        self.requests_patcher = patch('krkn.scenario_plugins.node_actions.ibmcloud_power_node_scenarios.requests')
+        self.mock_requests = self.requests_patcher.start()
+
+        # Mock authentication response
+        mock_auth_response = Mock()
+        mock_auth_response.status_code = 200
+        mock_auth_response.json.return_value = {
+            'access_token': 'test-token',
+            'token_type': 'Bearer',
+            'expires_in': 3600
+        }
+
+        self.mock_requests.request.return_value = mock_auth_response
+
+        # Create IbmCloudPower instance
+        self.ibm = IbmCloudPower()
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.env_patcher.stop()
+        self.requests_patcher.stop()
+
+    def test_init_success(self):
+        """Test IbmCloudPower class initialization"""
+        self.assertIsNotNone(self.ibm.api_key)
+        self.assertEqual(self.ibm.api_key, 'test-api-key')
+        self.assertIsNotNone(self.ibm.service_url)
+        self.assertEqual(self.ibm.service_url, 'https://test.cloud.ibm.com')
+        self.assertIsNotNone(self.ibm.CRN)
+        self.assertEqual(self.ibm.cloud_instance_id, 'instance-id')
+        self.assertIsNotNone(self.ibm.token)
+        self.assertIsNotNone(self.ibm.headers)
+
+    def test_init_missing_api_key(self):
+        """Test initialization fails when IBMC_APIKEY is missing"""
+        with patch.dict('os.environ', {
+            'IBMC_POWER_URL': 'https://test.cloud.ibm.com',
+            'IBMC_POWER_CRN': 'crn:v1:bluemix:public:power-iaas:us-south:a/abc123:instance-id::'
+        }, clear=True):
+            with self.assertRaises(Exception) as context:
+                IbmCloudPower()
+            self.assertIn("IBMC_APIKEY", str(context.exception))
+
+    def test_init_missing_power_url(self):
+        """Test initialization fails when IBMC_POWER_URL is missing"""
+        with patch.dict('os.environ', {
+            'IBMC_APIKEY': 'test-api-key',
+            'IBMC_POWER_CRN': 'crn:v1:bluemix:public:power-iaas:us-south:a/abc123:instance-id::'
+        }, clear=True):
+            with self.assertRaises(Exception) as context:
+                IbmCloudPower()
+            self.assertIn("IBMC_POWER_URL", str(context.exception))
+
+    def test_init_missing_crn(self):
+        """Test initialization fails when IBMC_POWER_CRN is missing"""
+        with patch.dict('os.environ', {
+            'IBMC_APIKEY': 'test-api-key',
+            'IBMC_POWER_URL': 'https://test.cloud.ibm.com'
+        }, clear=True):
+            # The code will fail on split() before the IBMC_POWER_CRN check
+            # so we check for either AttributeError or the exception message
+            with self.assertRaises((Exception, AttributeError)):
+                IbmCloudPower()
+
+    def test_authenticate_success(self):
+        """Test successful authentication"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'access_token': 'new-test-token',
+            'token_type': 'Bearer',
+            'expires_in': 3600
+        }
+        self.mock_requests.request.return_value = mock_response
+
+        self.ibm.authenticate()
+
+        self.assertEqual(self.ibm.token['access_token'], 'new-test-token')
+        self.assertIn('Authorization', self.ibm.headers)
+        self.assertEqual(self.ibm.headers['Authorization'], 'Bearer new-test-token')
+
+    def test_authenticate_failure(self):
+        """Test authentication failure"""
+        mock_response = Mock()
+        mock_response.status_code = 401
+        mock_response.raise_for_status.side_effect = Exception("Unauthorized")
+        self.mock_requests.request.return_value = mock_response
+
+        with self.assertRaises(Exception):
+            self.ibm.authenticate()
+
+    def test_get_instance_id_success(self):
+        """Test getting instance ID by node name"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'pvmInstances': [
+                {'serverName': 'test-node-1', 'pvmInstanceID': 'pvm-1'},
+                {'serverName': 'test-node-2', 'pvmInstanceID': 'pvm-2'}
+            ]
+        }
+        self.mock_requests.request.return_value = mock_response
+
+        instance_id = self.ibm.get_instance_id('test-node-1')
+
+        self.assertEqual(instance_id, 'pvm-1')
+
+    def test_get_instance_id_not_found(self):
+        """Test getting instance ID when node not found"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'pvmInstances': [
+                {'serverName': 'test-node-1', 'pvmInstanceID': 'pvm-1'}
+            ]
+        }
+        self.mock_requests.request.return_value = mock_response
+
+        with self.assertRaises(SystemExit):
+            self.ibm.get_instance_id('non-existent-node')
+
+    def test_delete_instance_success(self):
+        """Test deleting instance successfully"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        self.mock_requests.request.return_value = mock_response
+
+        result = self.ibm.delete_instance('pvm-123')
+
+        self.mock_requests.request.assert_called()
+        call_args = self.mock_requests.request.call_args
+        self.assertIn('immediate-shutdown', call_args[1]['data'])
+
+    def test_delete_instance_failure(self):
+        """Test deleting instance with failure"""
+        self.mock_requests.request.side_effect = Exception("API Error")
+
+        result = self.ibm.delete_instance('pvm-123')
+
+        self.assertEqual(result, False)
+
+    def test_reboot_instances_hard_reboot(self):
+        """Test hard reboot of instance"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        self.mock_requests.request.return_value = mock_response
+
+        result = self.ibm.reboot_instances('pvm-123', soft=False)
+
+        self.assertTrue(result)
+        call_args = self.mock_requests.request.call_args
+        self.assertIn('hard-reboot', call_args[1]['data'])
+
+    def test_reboot_instances_soft_reboot(self):
+        """Test soft reboot of instance"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        self.mock_requests.request.return_value = mock_response
+
+        result = self.ibm.reboot_instances('pvm-123', soft=True)
+
+        self.assertTrue(result)
+        call_args = self.mock_requests.request.call_args
+        self.assertIn('soft-reboot', call_args[1]['data'])
+
+    def test_reboot_instances_failure(self):
+        """Test reboot instance with failure"""
+        self.mock_requests.request.side_effect = Exception("API Error")
+
+        result = self.ibm.reboot_instances('pvm-123')
+
+        self.assertEqual(result, False)
+
+    def test_stop_instances_success(self):
+        """Test stopping instance successfully"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        self.mock_requests.request.return_value = mock_response
+
+        result = self.ibm.stop_instances('pvm-123')
+
+        self.assertTrue(result)
+        call_args = self.mock_requests.request.call_args
+        self.assertIn('stop', call_args[1]['data'])
+
+    def test_stop_instances_failure(self):
+        """Test stopping instance with failure"""
+        self.mock_requests.request.side_effect = Exception("API Error")
+
+        result = self.ibm.stop_instances('pvm-123')
+
+        self.assertEqual(result, False)
+
+    def test_start_instances_success(self):
+        """Test starting instance successfully"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        self.mock_requests.request.return_value = mock_response
+
+        result = self.ibm.start_instances('pvm-123')
+
+        self.assertTrue(result)
+        call_args = self.mock_requests.request.call_args
+        self.assertIn('start', call_args[1]['data'])
+
+    def test_start_instances_failure(self):
+        """Test starting instance with failure"""
+        self.mock_requests.request.side_effect = Exception("API Error")
+
+        result = self.ibm.start_instances('pvm-123')
+
+        self.assertEqual(result, False)
+
+    def test_list_instances_success(self):
+        """Test listing instances successfully"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'pvmInstances': [
+                type('obj', (object,), {'serverName': 'node-1', 'pvmInstanceID': 'pvm-1'}),
+                type('obj', (object,), {'serverName': 'node-2', 'pvmInstanceID': 'pvm-2'})
+            ]
+        }
+        self.mock_requests.request.return_value = mock_response
+
+        instances = self.ibm.list_instances()
+
+        self.assertEqual(len(instances), 2)
+        self.assertEqual(instances[0]['serverName'], 'node-1')
+        self.assertEqual(instances[1]['serverName'], 'node-2')
+
+    def test_list_instances_failure(self):
+        """Test listing instances with failure"""
+        self.mock_requests.request.side_effect = Exception("API Error")
+
+        with self.assertRaises(SystemExit):
+            self.ibm.list_instances()
+
+    def test_get_instance_status_success(self):
+        """Test getting instance status successfully"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {'status': 'ACTIVE'}
+        self.mock_requests.request.return_value = mock_response
+
+        status = self.ibm.get_instance_status('pvm-123')
+
+        self.assertEqual(status, 'ACTIVE')
+
+    def test_get_instance_status_failure(self):
+        """Test getting instance status with failure"""
+        self.mock_requests.request.side_effect = Exception("API Error")
+
+        status = self.ibm.get_instance_status('pvm-123')
+
+        self.assertIsNone(status)
+
+    def test_wait_until_deleted_success(self):
+        """Test waiting until instance is deleted"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {'status': None}
+        self.mock_requests.request.side_effect = [
+            mock_response,
+            Exception("Not found")
+        ]
+
+        affected_node = MagicMock(spec=AffectedNode)
+
+        with patch('time.time', side_effect=[100, 105]), \
+             patch('time.sleep'):
+            result = self.ibm.wait_until_deleted('pvm-123', timeout=60, affected_node=affected_node)
+
+        self.assertTrue(result)
+        affected_node.set_affected_node_status.assert_called_once()
+
+    def test_wait_until_deleted_timeout(self):
+        """Test waiting until deleted with timeout"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {'status': 'DELETING'}
+        self.mock_requests.request.return_value = mock_response
+
+        with patch('time.sleep'):
+            result = self.ibm.wait_until_deleted('pvm-123', timeout=5)
+
+        self.assertFalse(result)
+
+    def test_wait_until_running_success(self):
+        """Test waiting until instance is running"""
+        mock_responses = [
+            Mock(status_code=200, json=lambda: {'status': 'BUILD'}),
+            Mock(status_code=200, json=lambda: {'status': 'ACTIVE'})
+        ]
+        self.mock_requests.request.side_effect = mock_responses
+
+        affected_node = MagicMock(spec=AffectedNode)
+
+        with patch('time.time', side_effect=[100, 105]), \
+             patch('time.sleep'):
+            result = self.ibm.wait_until_running('pvm-123', timeout=60, affected_node=affected_node)
+
+        self.assertTrue(result)
+        affected_node.set_affected_node_status.assert_called_once_with("running", 5)
+
+    def test_wait_until_running_timeout(self):
+        """Test waiting until running with timeout"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {'status': 'BUILD'}
+        self.mock_requests.request.return_value = mock_response
+
+        with patch('time.sleep'):
+            result = self.ibm.wait_until_running('pvm-123', timeout=5)
+
+        self.assertFalse(result)
+
+    def test_wait_until_stopped_success(self):
+        """Test waiting until instance is stopped"""
+        mock_responses = [
+            Mock(status_code=200, json=lambda: {'status': 'STOPPING'}),
+            Mock(status_code=200, json=lambda: {'status': 'STOPPED'})
+        ]
+        self.mock_requests.request.side_effect = mock_responses
+
+        affected_node = MagicMock(spec=AffectedNode)
+
+        with patch('time.time', side_effect=[100, 105]), \
+             patch('time.sleep'):
+            result = self.ibm.wait_until_stopped('pvm-123', timeout=60, affected_node=affected_node)
+
+        self.assertTrue(result)
+        affected_node.set_affected_node_status.assert_called_once_with("stopped", 5)
+
+    def test_wait_until_stopped_timeout(self):
+        """Test waiting until stopped with timeout"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {'status': 'STOPPING'}
+        self.mock_requests.request.return_value = mock_response
+
+        with patch('time.sleep'):
+            result = self.ibm.wait_until_stopped('pvm-123', timeout=5, affected_node=None)
+
+        self.assertFalse(result)
+
+    def test_wait_until_rebooted_success(self):
+        """Test waiting until instance is rebooted"""
+        # wait_until_rebooted calls get_instance_status until NOT in reboot state,
+        # then calls wait_until_running which also calls get_instance_status
+        mock_responses = [
+            Mock(status_code=200, json=lambda: {'status': 'HARD_REBOOT'}),  # First check - still rebooting
+            Mock(status_code=200, json=lambda: {'status': 'ACTIVE'}),       # Second check - done rebooting
+            Mock(status_code=200, json=lambda: {'status': 'ACTIVE'})        # wait_until_running check
+        ]
+        self.mock_requests.request.side_effect = mock_responses
+
+        affected_node = MagicMock(spec=AffectedNode)
+
+        # Mock all time() calls - need many values because logging uses time.time() extensively
+        time_values = [100] * 20  # Just provide enough time values
+        with patch('time.time', side_effect=time_values), \
+             patch('time.sleep'):
+            result = self.ibm.wait_until_rebooted('pvm-123', timeout=60, affected_node=affected_node)
+
+        self.assertTrue(result)
+
+    def test_wait_until_rebooted_timeout(self):
+        """Test waiting until rebooted with timeout"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {'status': 'HARD_REBOOT'}
+        self.mock_requests.request.return_value = mock_response
+
+        with patch('time.sleep'):
+            result = self.ibm.wait_until_rebooted('pvm-123', timeout=5, affected_node=None)
+
+        self.assertFalse(result)
+
+    def test_find_id_in_list(self):
+        """Test finding ID in VPC list"""
+        vpc_list = [
+            {'vpc_name': 'vpc-1', 'vpc_id': 'id-1'},
+            {'vpc_name': 'vpc-2', 'vpc_id': 'id-2'}
+        ]
+
+        vpc_id = self.ibm.find_id_in_list('vpc-2', vpc_list)
+
+        self.assertEqual(vpc_id, 'id-2')
+
+    def test_find_id_in_list_not_found(self):
+        """Test finding ID in VPC list when not found"""
+        vpc_list = [
+            {'vpc_name': 'vpc-1', 'vpc_id': 'id-1'}
+        ]
+
+        vpc_id = self.ibm.find_id_in_list('vpc-3', vpc_list)
+
+        self.assertIsNone(vpc_id)
+
+
+class TestIbmCloudPowerNodeScenarios(unittest.TestCase):
+    """Test cases for ibmcloud_power_node_scenarios class"""
+
+    def setUp(self):
+        """Set up test fixtures"""
+        # Mock KrknKubernetes
+        self.mock_kubecli = MagicMock(spec=KrknKubernetes)
+        self.affected_nodes_status = AffectedNodeStatus()
+
+        # Mock the IbmCloudPower class entirely to avoid any real API calls
+        self.ibm_cloud_patcher = patch('krkn.scenario_plugins.node_actions.ibmcloud_power_node_scenarios.IbmCloudPower')
+        self.mock_ibm_cloud_class = self.ibm_cloud_patcher.start()
+
+        # Create a mock instance that will be returned when IbmCloudPower() is called
+        self.mock_ibm_cloud_instance = MagicMock()
+        self.mock_ibm_cloud_class.return_value = self.mock_ibm_cloud_instance
+
+        # Create ibmcloud_power_node_scenarios instance
+        self.scenario = ibmcloud_power_node_scenarios(
+            kubecli=self.mock_kubecli,
+            node_action_kube_check=True,
+            affected_nodes_status=self.affected_nodes_status,
+            disable_ssl_verification=False
+        )
+
+    def tearDown(self):
+        """Clean up after tests"""
+        self.ibm_cloud_patcher.stop()
+
+    def test_init(self):
+        """Test ibmcloud_power_node_scenarios initialization"""
+        self.assertIsNotNone(self.scenario.ibmcloud_power)
+        self.assertTrue(self.scenario.node_action_kube_check)
+        self.assertEqual(self.scenario.kubecli, self.mock_kubecli)
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_success(self, mock_wait_ready):
+        """Test node start scenario successfully"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.start_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_running.return_value = True
+
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+        self.assertEqual(self.affected_nodes_status.affected_nodes[0].node_name, 'test-node')
+        mock_wait_ready.assert_called_once()
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_start_scenario_no_kube_check(self, mock_wait_ready):
+        """Test node start scenario without Kubernetes check"""
+        self.scenario.node_action_kube_check = False
+
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.start_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_running.return_value = True
+
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        mock_wait_ready.assert_not_called()
+
+    def test_node_stop_scenario_success(self):
+        """Test node stop scenario successfully"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.stop_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_stopped.return_value = True
+
+        self.scenario.node_stop_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        # Verify methods were called
+        self.mock_ibm_cloud_instance.get_instance_id.assert_called_once_with('test-node')
+        self.mock_ibm_cloud_instance.stop_instances.assert_called_once_with('pvm-123')
+
+        # Note: affected_nodes are not appended in stop scenario based on the code
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_reboot_scenario_hard_reboot(self, mock_wait_ready, mock_wait_unknown):
+        """Test node hard reboot scenario"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.reboot_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_rebooted.return_value = True
+
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            soft_reboot=False
+        )
+
+        # Verify methods were called
+        self.mock_ibm_cloud_instance.reboot_instances.assert_called_once_with('pvm-123', False)
+        mock_wait_unknown.assert_called_once()
+        mock_wait_ready.assert_called_once()
+
+        # Note: affected_nodes are not appended in reboot scenario based on the code
+
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_unknown_status')
+    @patch('krkn.scenario_plugins.node_actions.common_node_functions.wait_for_ready_status')
+    def test_node_reboot_scenario_soft_reboot(self, mock_wait_ready, mock_wait_unknown):
+        """Test node soft reboot scenario"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.reboot_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_rebooted.return_value = True
+
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            soft_reboot=True
+        )
+
+        # Verify methods were called
+        self.mock_ibm_cloud_instance.reboot_instances.assert_called_once_with('pvm-123', True)
+        mock_wait_unknown.assert_called_once()
+        mock_wait_ready.assert_called_once()
+
+        # Note: affected_nodes are not appended in reboot scenario based on the code
+
+    def test_node_terminate_scenario_success(self):
+        """Test node terminate scenario successfully"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.delete_instance.return_value = None
+        self.mock_ibm_cloud_instance.wait_until_deleted.return_value = True
+
+        self.scenario.node_terminate_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        # Verify methods were called
+        self.mock_ibm_cloud_instance.delete_instance.assert_called_once_with('pvm-123')
+        self.mock_ibm_cloud_instance.wait_until_deleted.assert_called_once()
+
+        # Note: affected_nodes are not appended in terminate scenario based on the code
+
+    def test_node_scenario_multiple_kill_count(self):
+        """Test node scenario with multiple kill count"""
+        # Configure mock methods
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.stop_instances.return_value = True
+        self.mock_ibm_cloud_instance.wait_until_stopped.return_value = True
+
+        self.scenario.node_stop_scenario(
+            instance_kill_count=2,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        # Verify stop was called twice (kill_count=2)
+        self.assertEqual(self.mock_ibm_cloud_instance.stop_instances.call_count, 2)
+
+        # Note: affected_nodes are not appended in stop scenario based on the code
+
+    def test_node_start_scenario_exception(self):
+        """Test node start scenario with exception during operation"""
+        # Configure mock - get_instance_id succeeds but start_instances fails
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.start_instances.side_effect = Exception("API Error")
+
+        # Should handle exception gracefully
+        self.scenario.node_start_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            poll_interval=5
+        )
+
+        # Verify affected node still added even on failure
+        self.assertEqual(len(self.affected_nodes_status.affected_nodes), 1)
+
+    def test_node_reboot_scenario_exception(self):
+        """Test node reboot scenario with exception during operation"""
+        # Configure mock - get_instance_id succeeds but reboot_instances fails
+        self.mock_ibm_cloud_instance.get_instance_id.return_value = 'pvm-123'
+        self.mock_ibm_cloud_instance.reboot_instances.side_effect = Exception("API Error")
+
+        # Should handle exception gracefully
+        self.scenario.node_reboot_scenario(
+            instance_kill_count=1,
+            node='test-node',
+            timeout=60,
+            soft_reboot=False
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_ingress_network_plugin.py

@@ -1,5 +1,5 @@
 import unittest
-import logging
+from unittest.mock import Mock, patch
 from arcaflow_plugin_sdk import plugin
 
 from krkn.scenario_plugins.native.network import ingress_shaping
@@ -8,6 +8,7 @@ from krkn.scenario_plugins.native.network import ingress_shaping
 class NetworkScenariosTest(unittest.TestCase):
 
     def test_serialization(self):
+        """Test serialization of configuration and output objects"""
         plugin.test_object_serialization(
             ingress_shaping.NetworkScenarioConfig(
                 node_interface_name={"foo": ["bar"]},
@@ -39,26 +40,687 @@ class NetworkScenariosTest(unittest.TestCase):
             self.fail,
         )
 
-    def test_network_chaos(self):
-        output_id, output_data = ingress_shaping.network_chaos(
-            params=ingress_shaping.NetworkScenarioConfig(
-                label_selector="node-role.kubernetes.io/control-plane",
-                instance_count=1,
-                network_params={
-                    "latency": "50ms",
-                    "loss": "0.02",
-                    "bandwidth": "100mbit",
-                },
-            ),
-            run_id="network-shaping-test",
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_get_default_interface(self, mock_kube_helper):
+        """Test getting default interface from a node"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_pod_template.render.return_value = "pod_yaml_content"
+
+        mock_kube_helper.create_pod.return_value = None
+        mock_kube_helper.exec_cmd_in_pod.return_value = (
+            "default via 192.168.1.1 dev eth0 proto dhcp metric 100\n"
+            "172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1"
         )
-        if output_id == "error":
-            logging.error(output_data.error)
-            self.fail(
-                "The network chaos scenario did not complete successfully "
-                "because an error/exception occurred"
+        mock_kube_helper.delete_pod.return_value = None
+
+        # Test
+        result = ingress_shaping.get_default_interface(
+            node="test-node",
+            pod_template=mock_pod_template,
+            cli=mock_cli,
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        self.assertEqual(result, ["eth0"])
+        mock_kube_helper.create_pod.assert_called_once()
+        mock_kube_helper.exec_cmd_in_pod.assert_called_once()
+        mock_kube_helper.delete_pod.assert_called_once()
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_verify_interface_with_empty_list(self, mock_kube_helper):
+        """Test verifying interface when input list is empty"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_pod_template.render.return_value = "pod_yaml_content"
+
+        mock_kube_helper.create_pod.return_value = None
+        mock_kube_helper.exec_cmd_in_pod.return_value = (
+            "default via 192.168.1.1 dev eth0 proto dhcp metric 100\n"
+        )
+        mock_kube_helper.delete_pod.return_value = None
+
+        # Test
+        result = ingress_shaping.verify_interface(
+            input_interface_list=[],
+            node="test-node",
+            pod_template=mock_pod_template,
+            cli=mock_cli,
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        self.assertEqual(result, ["eth0"])
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_verify_interface_with_valid_interfaces(self, mock_kube_helper):
+        """Test verifying interface with valid interface list"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_pod_template.render.return_value = "pod_yaml_content"
+
+        mock_kube_helper.create_pod.return_value = None
+        mock_kube_helper.exec_cmd_in_pod.return_value = (
+            "eth0             UP             192.168.1.10/24\n"
+            "eth1             UP             10.0.0.5/24\n"
+            "lo               UNKNOWN        127.0.0.1/8\n"
+        )
+        mock_kube_helper.delete_pod.return_value = None
+
+        # Test
+        result = ingress_shaping.verify_interface(
+            input_interface_list=["eth0", "eth1"],
+            node="test-node",
+            pod_template=mock_pod_template,
+            cli=mock_cli,
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        self.assertEqual(result, ["eth0", "eth1"])
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_verify_interface_with_invalid_interface(self, mock_kube_helper):
+        """Test verifying interface with an interface that doesn't exist"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_pod_template.render.return_value = "pod_yaml_content"
+
+        mock_kube_helper.create_pod.return_value = None
+        mock_kube_helper.exec_cmd_in_pod.return_value = (
+            "eth0             UP             192.168.1.10/24\n"
+            "lo               UNKNOWN        127.0.0.1/8\n"
+        )
+        mock_kube_helper.delete_pod.return_value = None
+
+        # Test - should raise exception
+        with self.assertRaises(Exception) as context:
+            ingress_shaping.verify_interface(
+                input_interface_list=["eth0", "eth99"],
+                node="test-node",
+                pod_template=mock_pod_template,
+                cli=mock_cli,
+                image="quay.io/krkn-chaos/krkn:tools"
             )
 
+        self.assertIn("Interface eth99 not found", str(context.exception))
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_default_interface')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_get_node_interfaces_with_label_selector(self, mock_kube_helper, mock_get_default_interface):
+        """Test getting node interfaces using label selector"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_kube_helper.get_node.return_value = ["node1", "node2"]
+        mock_get_default_interface.return_value = ["eth0"]
+
+        # Test
+        result = ingress_shaping.get_node_interfaces(
+            node_interface_dict=None,
+            label_selector="node-role.kubernetes.io/worker",
+            instance_count=2,
+            pod_template=mock_pod_template,
+            cli=mock_cli,
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        self.assertEqual(result, {"node1": ["eth0"], "node2": ["eth0"]})
+        self.assertEqual(mock_get_default_interface.call_count, 2)
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.verify_interface')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_get_node_interfaces_with_node_dict(self, mock_kube_helper, mock_verify_interface):
+        """Test getting node interfaces with provided node interface dictionary"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_kube_helper.get_node.return_value = ["node1"]
+        mock_verify_interface.return_value = ["eth0", "eth1"]
+
+        # Test
+        result = ingress_shaping.get_node_interfaces(
+            node_interface_dict={"node1": ["eth0", "eth1"]},
+            label_selector=None,
+            instance_count=1,
+            pod_template=mock_pod_template,
+            cli=mock_cli,
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        self.assertEqual(result, {"node1": ["eth0", "eth1"]})
+        mock_verify_interface.assert_called_once()
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_get_node_interfaces_no_selector_no_dict(self, mock_kube_helper):
+        """Test that exception is raised when both node dict and label selector are missing"""
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+
+        with self.assertRaises(Exception) as context:
+            ingress_shaping.get_node_interfaces(
+                node_interface_dict=None,
+                label_selector=None,
+                instance_count=1,
+                pod_template=mock_pod_template,
+                cli=mock_cli,
+                image="quay.io/krkn-chaos/krkn:tools"
+            )
+
+        self.assertIn("label selector must be provided", str(context.exception))
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_create_ifb(self, mock_kube_helper):
+        """Test creating virtual interfaces"""
+        mock_cli = Mock()
+        mock_kube_helper.exec_cmd_in_pod.return_value = None
+
+        # Test
+        ingress_shaping.create_ifb(cli=mock_cli, number=2, pod_name="test-pod")
+
+        # Assertions
+        # Should call modprobe once and ip link set for each interface
+        self.assertEqual(mock_kube_helper.exec_cmd_in_pod.call_count, 3)
+
+        # Verify modprobe call
+        first_call = mock_kube_helper.exec_cmd_in_pod.call_args_list[0]
+        self.assertIn("modprobe", first_call[0][1])
+        self.assertIn("numifbs=2", first_call[0][1])
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_delete_ifb(self, mock_kube_helper):
+        """Test deleting virtual interfaces"""
+        mock_cli = Mock()
+        mock_kube_helper.exec_cmd_in_pod.return_value = None
+
+        # Test
+        ingress_shaping.delete_ifb(cli=mock_cli, pod_name="test-pod")
+
+        # Assertions
+        mock_kube_helper.exec_cmd_in_pod.assert_called_once()
+        call_args = mock_kube_helper.exec_cmd_in_pod.call_args[0][1]
+        self.assertIn("modprobe", call_args)
+        self.assertIn("-r", call_args)
+        self.assertIn("ifb", call_args)
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_get_job_pods(self, mock_kube_helper):
+        """Test getting pods associated with a job"""
+        mock_cli = Mock()
+        mock_api_response = Mock()
+        mock_api_response.metadata.labels = {"controller-uid": "test-uid-123"}
+
+        mock_kube_helper.list_pods.return_value = ["pod1", "pod2"]
+
+        # Test
+        result = ingress_shaping.get_job_pods(cli=mock_cli, api_response=mock_api_response)
+
+        # Assertions
+        self.assertEqual(result, "pod1")
+        mock_kube_helper.list_pods.assert_called_once_with(
+            mock_cli,
+            label_selector="controller-uid=test-uid-123",
+            namespace="default"
+        )
+
+    @patch('time.sleep', return_value=None)
+    @patch('time.time')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_wait_for_job_success(self, mock_kube_helper, mock_time, mock_sleep):
+        """Test waiting for jobs to complete successfully"""
+        mock_batch_cli = Mock()
+        mock_time.side_effect = [0, 10, 20]  # Simulate time progression
+
+        # First job succeeds
+        mock_response1 = Mock()
+        mock_response1.status.succeeded = 1
+        mock_response1.status.failed = None
+
+        # Second job succeeds
+        mock_response2 = Mock()
+        mock_response2.status.succeeded = 1
+        mock_response2.status.failed = None
+
+        mock_kube_helper.get_job_status.side_effect = [mock_response1, mock_response2]
+
+        # Test
+        ingress_shaping.wait_for_job(
+            batch_cli=mock_batch_cli,
+            job_list=["job1", "job2"],
+            timeout=300
+        )
+
+        # Assertions
+        self.assertEqual(mock_kube_helper.get_job_status.call_count, 2)
+
+    @patch('time.sleep', return_value=None)
+    @patch('time.time')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_wait_for_job_timeout(self, mock_kube_helper, mock_time, mock_sleep):
+        """Test waiting for jobs times out"""
+        mock_batch_cli = Mock()
+        mock_time.side_effect = [0, 350]  # Simulate timeout
+
+        mock_response = Mock()
+        mock_response.status.succeeded = None
+        mock_response.status.failed = None
+
+        mock_kube_helper.get_job_status.return_value = mock_response
+
+        # Test - should raise exception
+        with self.assertRaises(Exception) as context:
+            ingress_shaping.wait_for_job(
+                batch_cli=mock_batch_cli,
+                job_list=["job1"],
+                timeout=300
+            )
+
+        self.assertIn("timeout", str(context.exception))
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_delete_jobs(self, mock_kube_helper):
+        """Test deleting jobs"""
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+
+        mock_response = Mock()
+        mock_response.status.failed = None
+        mock_kube_helper.get_job_status.return_value = mock_response
+        mock_kube_helper.delete_job.return_value = None
+
+        # Test
+        ingress_shaping.delete_jobs(
+            cli=mock_cli,
+            batch_cli=mock_batch_cli,
+            job_list=["job1", "job2"]
+        )
+
+        # Assertions
+        self.assertEqual(mock_kube_helper.get_job_status.call_count, 2)
+        self.assertEqual(mock_kube_helper.delete_job.call_count, 2)
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_job_pods')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_delete_jobs_with_failed_job(self, mock_kube_helper, mock_get_job_pods):
+        """Test deleting jobs when one has failed"""
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+
+        mock_response = Mock()
+        mock_response.status.failed = 1
+
+        mock_pod_status = Mock()
+        mock_pod_status.status.container_statuses = []
+
+        mock_log_response = Mock()
+        mock_log_response.data.decode.return_value = "Error log content"
+
+        mock_kube_helper.get_job_status.return_value = mock_response
+        mock_get_job_pods.return_value = "failed-pod"
+        mock_kube_helper.read_pod.return_value = mock_pod_status
+        mock_kube_helper.get_pod_log.return_value = mock_log_response
+        mock_kube_helper.delete_job.return_value = None
+
+        # Test
+        ingress_shaping.delete_jobs(
+            cli=mock_cli,
+            batch_cli=mock_batch_cli,
+            job_list=["failed-job"]
+        )
+
+        # Assertions
+        mock_kube_helper.read_pod.assert_called_once()
+        mock_kube_helper.get_pod_log.assert_called_once()
+
+    def test_get_ingress_cmd_basic(self):
+        """Test generating ingress traffic shaping commands"""
+        result = ingress_shaping.get_ingress_cmd(
+            interface_list=["eth0"],
+            network_parameters={"latency": "50ms"},
+            duration=120
+        )
+
+        # Assertions
+        self.assertIn("tc qdisc add dev eth0 handle ffff: ingress", result)
+        self.assertIn("tc filter add dev eth0", result)
+        self.assertIn("ifb0", result)
+        self.assertIn("delay 50ms", result)
+        self.assertIn("sleep 120", result)
+        self.assertIn("tc qdisc del", result)
+
+    def test_get_ingress_cmd_multiple_interfaces(self):
+        """Test generating commands for multiple interfaces"""
+        result = ingress_shaping.get_ingress_cmd(
+            interface_list=["eth0", "eth1"],
+            network_parameters={"latency": "50ms", "bandwidth": "100mbit"},
+            duration=120
+        )
+
+        # Assertions
+        self.assertIn("eth0", result)
+        self.assertIn("eth1", result)
+        self.assertIn("ifb0", result)
+        self.assertIn("ifb1", result)
+        self.assertIn("delay 50ms", result)
+        self.assertIn("rate 100mbit", result)
+
+    def test_get_ingress_cmd_all_parameters(self):
+        """Test generating commands with all network parameters"""
+        result = ingress_shaping.get_ingress_cmd(
+            interface_list=["eth0"],
+            network_parameters={
+                "latency": "50ms",
+                "loss": "0.02",
+                "bandwidth": "100mbit"
+            },
+            duration=120
+        )
+
+        # Assertions
+        self.assertIn("delay 50ms", result)
+        self.assertIn("loss 0.02", result)
+        self.assertIn("rate 100mbit", result)
+
+    def test_get_ingress_cmd_invalid_interface(self):
+        """Test that invalid interface names raise an exception"""
+        with self.assertRaises(Exception) as context:
+            ingress_shaping.get_ingress_cmd(
+                interface_list=["eth0; rm -rf /"],
+                network_parameters={"latency": "50ms"},
+                duration=120
+            )
+
+        self.assertIn("does not match the required regex pattern", str(context.exception))
+
+    @patch('yaml.safe_load')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.create_virtual_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_ingress_cmd')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_apply_ingress_filter(self, mock_kube_helper, mock_get_cmd, mock_create_virtual, mock_yaml):
+        """Test applying ingress filters to a node"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+        mock_pod_template = Mock()
+        mock_job_template = Mock()
+        mock_job_template.render.return_value = "job_yaml"
+
+        mock_cfg = ingress_shaping.NetworkScenarioConfig(
+            node_interface_name={"node1": ["eth0"]},
+            network_params={"latency": "50ms"},
+            test_duration=120
+        )
+
+        mock_yaml.return_value = {"metadata": {"name": "test-job"}}
+        mock_get_cmd.return_value = "tc commands"
+        mock_kube_helper.create_job.return_value = Mock()
+
+        # Test
+        result = ingress_shaping.apply_ingress_filter(
+            cfg=mock_cfg,
+            interface_list=["eth0"],
+            node="node1",
+            pod_template=mock_pod_template,
+            job_template=mock_job_template,
+            batch_cli=mock_batch_cli,
+            cli=mock_cli,
+            create_interfaces=True,
+            param_selector="all",
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        mock_create_virtual.assert_called_once()
+        mock_get_cmd.assert_called_once()
+        mock_kube_helper.create_job.assert_called_once()
+        self.assertEqual(result, "test-job")
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_create_virtual_interfaces(self, mock_kube_helper):
+        """Test creating virtual interfaces on a node"""
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_pod_template.render.return_value = "pod_yaml"
+
+        mock_kube_helper.create_pod.return_value = None
+        mock_kube_helper.exec_cmd_in_pod.return_value = None
+        mock_kube_helper.delete_pod.return_value = None
+
+        # Test
+        ingress_shaping.create_virtual_interfaces(
+            cli=mock_cli,
+            interface_list=["eth0", "eth1"],
+            node="test-node",
+            pod_template=mock_pod_template,
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        mock_kube_helper.create_pod.assert_called_once()
+        mock_kube_helper.delete_pod.assert_called_once()
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_ifb')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_delete_virtual_interfaces(self, mock_kube_helper, mock_delete_ifb):
+        """Test deleting virtual interfaces from nodes"""
+        mock_cli = Mock()
+        mock_pod_template = Mock()
+        mock_pod_template.render.return_value = "pod_yaml"
+
+        mock_kube_helper.create_pod.return_value = None
+        mock_kube_helper.delete_pod.return_value = None
+
+        # Test
+        ingress_shaping.delete_virtual_interfaces(
+            cli=mock_cli,
+            node_list=["node1", "node2"],
+            pod_template=mock_pod_template,
+            image="quay.io/krkn-chaos/krkn:tools"
+        )
+
+        # Assertions
+        self.assertEqual(mock_kube_helper.create_pod.call_count, 2)
+        self.assertEqual(mock_delete_ifb.call_count, 2)
+        self.assertEqual(mock_kube_helper.delete_pod.call_count, 2)
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.Environment')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.FileSystemLoader')
+    @patch('yaml.safe_load')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_jobs')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_virtual_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.wait_for_job')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.apply_ingress_filter')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_node_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_network_chaos_parallel_execution(
+        self, mock_kube_helper, mock_get_nodes, mock_apply_filter,
+        mock_wait_job, mock_delete_virtual, mock_delete_jobs, mock_yaml,
+        mock_file_loader, mock_env
+    ):
+        """Test network chaos with parallel execution"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+        mock_yaml.return_value = {"metadata": {"name": "test-pod"}}
+        mock_kube_helper.setup_kubernetes.return_value = (mock_cli, mock_batch_cli)
+        mock_get_nodes.return_value = {"node1": ["eth0"], "node2": ["eth1"]}
+        mock_apply_filter.side_effect = ["job1", "job2"]
+
+        # Test
+        cfg = ingress_shaping.NetworkScenarioConfig(
+            label_selector="node-role.kubernetes.io/worker",
+            instance_count=2,
+            network_params={"latency": "50ms"},
+            execution_type="parallel",
+            test_duration=120,
+            wait_duration=30
+        )
+
+        output_id, output_data = ingress_shaping.network_chaos(params=cfg, run_id="test-run")
+
+        # Assertions
+        self.assertEqual(output_id, "success")
+        self.assertEqual(output_data.filter_direction, "ingress")
+        self.assertEqual(output_data.execution_type, "parallel")
+        self.assertEqual(mock_apply_filter.call_count, 2)
+        mock_wait_job.assert_called_once()
+        mock_delete_virtual.assert_called_once()
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.Environment')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.FileSystemLoader')
+    @patch('yaml.safe_load')
+    @patch('time.sleep', return_value=None)
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_jobs')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_virtual_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.wait_for_job')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.apply_ingress_filter')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_node_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_network_chaos_serial_execution(
+        self, mock_kube_helper, mock_get_nodes, mock_apply_filter,
+        mock_wait_job, mock_delete_virtual, mock_delete_jobs, mock_sleep, mock_yaml,
+        mock_file_loader, mock_env
+    ):
+        """Test network chaos with serial execution"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+        mock_yaml.return_value = {"metadata": {"name": "test-pod"}}
+        mock_kube_helper.setup_kubernetes.return_value = (mock_cli, mock_batch_cli)
+        mock_get_nodes.return_value = {"node1": ["eth0"]}
+        mock_apply_filter.return_value = "job1"
+
+        # Test
+        cfg = ingress_shaping.NetworkScenarioConfig(
+            label_selector="node-role.kubernetes.io/worker",
+            instance_count=1,
+            network_params={"latency": "50ms", "bandwidth": "100mbit"},
+            execution_type="serial",
+            test_duration=120,
+            wait_duration=30
+        )
+
+        output_id, output_data = ingress_shaping.network_chaos(params=cfg, run_id="test-run")
+
+        # Assertions
+        self.assertEqual(output_id, "success")
+        self.assertEqual(output_data.execution_type, "serial")
+        # Should be called once per parameter per node
+        self.assertEqual(mock_apply_filter.call_count, 2)
+        # Should wait for jobs twice (once per parameter)
+        self.assertEqual(mock_wait_job.call_count, 2)
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.Environment')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.FileSystemLoader')
+    @patch('yaml.safe_load')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_jobs')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_virtual_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_node_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_network_chaos_invalid_execution_type(
+        self, mock_kube_helper, mock_get_nodes, mock_delete_virtual, mock_delete_jobs, mock_yaml,
+        mock_file_loader, mock_env
+    ):
+        """Test network chaos with invalid execution type"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+        mock_yaml.return_value = {"metadata": {"name": "test-pod"}}
+        mock_kube_helper.setup_kubernetes.return_value = (mock_cli, mock_batch_cli)
+        mock_get_nodes.return_value = {"node1": ["eth0"]}
+
+        # Test
+        cfg = ingress_shaping.NetworkScenarioConfig(
+            label_selector="node-role.kubernetes.io/worker",
+            instance_count=1,
+            network_params={"latency": "50ms"},
+            execution_type="invalid_type",
+            test_duration=120
+        )
+
+        output_id, output_data = ingress_shaping.network_chaos(params=cfg, run_id="test-run")
+
+        # Assertions
+        self.assertEqual(output_id, "error")
+        self.assertIn("Invalid execution type", output_data.error)
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.Environment')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.FileSystemLoader')
+    @patch('yaml.safe_load')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_jobs')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_virtual_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_node_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_network_chaos_get_nodes_error(
+        self, mock_kube_helper, mock_get_nodes, mock_delete_virtual, mock_delete_jobs, mock_yaml,
+        mock_file_loader, mock_env
+    ):
+        """Test network chaos when getting nodes fails"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+        mock_yaml.return_value = {"metadata": {"name": "test-pod"}}
+        mock_kube_helper.setup_kubernetes.return_value = (mock_cli, mock_batch_cli)
+        mock_get_nodes.side_effect = Exception("Failed to get nodes")
+
+        # Test
+        cfg = ingress_shaping.NetworkScenarioConfig(
+            label_selector="node-role.kubernetes.io/worker",
+            instance_count=1,
+            network_params={"latency": "50ms"}
+        )
+
+        output_id, output_data = ingress_shaping.network_chaos(params=cfg, run_id="test-run")
+
+        # Assertions
+        self.assertEqual(output_id, "error")
+        self.assertIn("Failed to get nodes", output_data.error)
+
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.Environment')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.FileSystemLoader')
+    @patch('yaml.safe_load')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_jobs')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.delete_virtual_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.apply_ingress_filter')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.get_node_interfaces')
+    @patch('krkn.scenario_plugins.native.network.ingress_shaping.kube_helper')
+    def test_network_chaos_apply_filter_error(
+        self, mock_kube_helper, mock_get_nodes, mock_apply_filter,
+        mock_delete_virtual, mock_delete_jobs, mock_yaml,
+        mock_file_loader, mock_env
+    ):
+        """Test network chaos when applying filter fails"""
+        # Setup mocks
+        mock_cli = Mock()
+        mock_batch_cli = Mock()
+        mock_yaml.return_value = {"metadata": {"name": "test-pod"}}
+        mock_kube_helper.setup_kubernetes.return_value = (mock_cli, mock_batch_cli)
+        mock_get_nodes.return_value = {"node1": ["eth0"]}
+        mock_apply_filter.side_effect = Exception("Failed to apply filter")
+
+        # Test
+        cfg = ingress_shaping.NetworkScenarioConfig(
+            label_selector="node-role.kubernetes.io/worker",
+            instance_count=1,
+            network_params={"latency": "50ms"},
+            execution_type="parallel"
+        )
+
+        output_id, output_data = ingress_shaping.network_chaos(params=cfg, run_id="test-run")
+
+        # Assertions
+        self.assertEqual(output_id, "error")
+        self.assertIn("Failed to apply filter", output_data.error)
+        # Cleanup should still be called
+        mock_delete_virtual.assert_called_once()
+
 
 if __name__ == "__main__":
     unittest.main()