Merge pull request #30 from paigerube14/retryPowerfulseal

Retry powerfulseal for post action checks

config/config.yaml

@@ -1,13 +1,11 @@
+
 kraken:
     kubeconfig_path: /root/.kube/config                    # Path to kubeconfig
     exit_on_failure: False                                 # Exit when a post action scenario fails
     scenarios:                                             # List of policies/chaos scenarios to load
         - -    scenarios/etcd.yml
-          -    scenarios/post_action_etcd_example.sh
         - -    scenarios/openshift-kube-apiserver.yml
-          -    scenarios/post_action_openshift-kube-apiserver.yml
         - -    scenarios/openshift-apiserver.yml
-          -    scenarios/post_action_openshift-apiserver.yml
         - -    scenarios/regex_openshift_pod_kill.yml
           -    scenarios/post_action_regex.py
     node_scenarios:                                        # List of chaos node scenarios to load
@@ -20,4 +18,4 @@ cerberus:
 tunings:
     wait_duration: 60                                      # Duration to wait between each chaos scenario
     iterations: 1                                          # Number of times to execute the scenarios
-    daemon_mode: False                                     # Iterations are set to infinity which means that the cerberus will monitor the resources forever
+    daemon_mode: False                                     # Iterations are set to infinity which means that the cerberus will monitor the resources forever

requirements.txt

@@ -1,5 +1,5 @@
 datetime
 pyfiglet
-powerfulseal==3.0.0rc11
+git+https://github.com/powerfulseal/powerfulseal.git
 requests
 boto3

run_kraken.py

@@ -119,7 +119,7 @@ def run_post_action(kubeconfig_path, scenario, pre_action_output=""):
             else:
                 logging.info(scenario + ' post action response did not match pre check output')
                 return False
-    else:
+    elif scenario != "":
         # invoke custom bash script
         action_output = runcommand.invoke(scenario).strip()
         if pre_action_output:
@@ -208,7 +208,11 @@ def main(cfg):
                 try:
                     # Loop to run the scenarios starts here
                     for scenario in scenarios:
-                        pre_action_output = run_post_action(kubeconfig_path, scenario[1])
+
+                        if len(scenario) > 1:
+                            pre_action_output = run_post_action(kubeconfig_path, scenario[1])
+                        else:
+                            pre_action_output = ''
                         runcommand.invoke("powerfulseal autonomous --use-pod-delete-instead-of-ssh-kill"  # noqa
                                           " --policy-file %s --kubeconfig %s --no-cloud"
                                           " --inventory-kubernetes --headless"
@@ -217,6 +221,7 @@ def main(cfg):
                         logging.info("Scenario: %s has been successfully injected!" % (scenario[0]))
                         logging.info("Waiting for the specified duration: %s" % (wait_duration))
                         time.sleep(wait_duration)
+
                         failed_post_scenarios = post_actions(kubeconfig_path, scenario,
                                                              failed_post_scenarios,
                                                              pre_action_output)

scenarios/etcd.yml

@@ -18,3 +18,15 @@ scenarios:
           - kill:
               probability: 1
               force: true
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-etcd"
+              selector: "k8s-app=etcd"
+        retries:
+          retriesTimeout:
+            timeout: 180
+
+        actions:
+          - checkPodCount:
+              count: 3

scenarios/openshift-apiserver.yml

@@ -21,3 +21,15 @@ scenarios:
           - kill:
               probability: 1
               force: true
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-apiserver"
+              selector: "app=openshift-apiserver"
+        retries:
+          retriesTimeout:
+            timeout: 180
+
+        actions:
+          - checkPodCount:
+              count: 3

scenarios/openshift-kube-apiserver.yml

@@ -20,3 +20,14 @@ scenarios:
           - kill:
               probability: 1
               force: true
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-kube-apiserver"
+              selector: "app=openshift-kube-apiserver"
+        retries:
+          retriesTimeout:
+            timeout: 180
+        actions:
+          - checkPodCount:
+              count: 3

scenarios/post_action_etcd.yml

@@ -1,21 +0,0 @@
-config:
-  runStrategy:
-    runs: 1
-    maxSecondsBetweenRuns: 10
-    minSecondsBetweenRuns: 1
-scenarios:
-  - name: "check 3 pods are in namespace with selector: etcd"
-    steps:
-    - podAction:
-        matches:
-          - labels:
-              namespace: "openshift-etcd"
-              selector: "k8s-app=etcd"
-        filters:
-          - property:
-              name: "state"
-              value: "Running"
-        # The actions will be executed in the order specified
-        actions:
-          - checkPodCount:
-              count: 3

scenarios/post_action_etcd_example.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-pods="$(oc get pods -n openshift-etcd | grep -c Running)"
-echo "$pods"

scenarios/post_action_openshift-apiserver.yml

@@ -1,23 +0,0 @@
-config:
-  runStrategy:
-    runs: 1
-    maxSecondsBetweenRuns: 30
-    minSecondsBetweenRuns: 1
-scenarios:
-  - name: "check 3 pods are in namespace with selector: openshift-apiserver"
-    steps:
-    - podAction:
-        matches:
-          - labels:
-              namespace: "openshift-apiserver"
-              selector: "app=openshift-apiserver"
-
-        filters:
-          - property:
-              name: "state"
-              value: "Running"
-
-        # The actions will be executed in the order specified
-        actions:
-          - checkPodCount:
-              count: 3

scenarios/post_action_openshift-kube-apiserver.yml

@@ -1,21 +0,0 @@
-config:
-  runStrategy:
-    runs: 1
-    maxSecondsBetweenRuns: 30
-    minSecondsBetweenRuns: 1
-scenarios:
-  - name: "check 3 pods are in namespace with selector: openshift-kube-apiserver"
-    steps:
-    - podAction:
-        matches:
-          - labels:
-              namespace: "openshift-kube-apiserver"
-              selector: "app=openshift-kube-apiserver"
-        filters:
-          - property:
-              name: "state"
-              value: "Running"
-        # The actions will be executed in the order specified
-        actions:
-          - checkPodCount:
-              count: 3

scenarios/post_action_regex_openshift_pod_kill.yml

@@ -1,18 +0,0 @@
-config:
-  runStrategy:
-    runs: 1
-    maxSecondsBetweenRuns: 30
-    minSecondsBetweenRuns: 1
-scenarios:
-  - name: kill up to 3 pods in any openshift namespace
-    steps:
-    - podAction:
-        matches:
-          - namespace: "openshift-.*"
-        filters:
-          - property:
-             name: "state"
-             value: "Running"
-        actions:
-          - checkPodCount:
-              count: 146

scenarios/prometheus.yml

@@ -21,3 +21,15 @@ scenarios:
           - kill:
               probability: 1
               force: true
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-monitoring"
+              selector: "app=prometheus"
+        retries:
+          retriesTimeout:
+            timeout: 180
+
+        actions:
+          - checkPodCount:
+              count: 2