From 8f5b688fba7fba59b23c5f07b2bf6de00d5a4fd2 Mon Sep 17 00:00:00 2001 From: prubenda Date: Tue, 18 Aug 2020 09:16:44 -0400 Subject: [PATCH 1/2] working on powerfulseal retry logic --- config/config.yaml | 6 ++---- requirements.txt | 2 +- run_kraken.py | 9 +++++++-- scenarios/etcd.yml | 12 ++++++++++++ scenarios/openshift-apiserver.yml | 12 ++++++++++++ scenarios/openshift-kube-apiserver.yml | 11 +++++++++++ scenarios/prometheus.yml | 12 ++++++++++++ 7 files changed, 57 insertions(+), 7 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index d9005fd9..dc821eed 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,13 +1,11 @@ + kraken: kubeconfig_path: /root/.kube/config # Path to kubeconfig exit_on_failure: False # Exit when a post action scenario fails scenarios: # List of policies/chaos scenarios to load - - scenarios/etcd.yml - - scenarios/post_action_etcd_example.sh - - scenarios/openshift-kube-apiserver.yml - - scenarios/post_action_openshift-kube-apiserver.yml - - scenarios/openshift-apiserver.yml - - scenarios/post_action_openshift-apiserver.yml - - scenarios/regex_openshift_pod_kill.yml - scenarios/post_action_regex.py node_scenarios: # List of chaos node scenarios to load @@ -20,4 +18,4 @@ cerberus: tunings: wait_duration: 60 # Duration to wait between each chaos scenario iterations: 1 # Number of times to execute the scenarios - daemon_mode: False # Iterations are set to infinity which means that the cerberus will monitor the resources forever + daemon_mode: False # Iterations are set to infinity which means that the cerberus will monitor the resources forever \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c7e263f6..806e49e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ datetime pyfiglet -powerfulseal==3.0.0rc11 +git+https://github.com/powerfulseal/powerfulseal.git requests boto3 diff --git a/run_kraken.py b/run_kraken.py index 9cb55b00..54c6a59d 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -119,7 +119,7 @@ def run_post_action(kubeconfig_path, scenario, pre_action_output=""): else: logging.info(scenario + ' post action response did not match pre check output') return False - else: + elif scenario != "": # invoke custom bash script action_output = runcommand.invoke(scenario).strip() if pre_action_output: @@ -208,7 +208,11 @@ def main(cfg): try: # Loop to run the scenarios starts here for scenario in scenarios: - pre_action_output = run_post_action(kubeconfig_path, scenario[1]) + + if len(scenario) > 1: + pre_action_output = run_post_action(kubeconfig_path, scenario[1]) + else: + pre_action_output = '' runcommand.invoke("powerfulseal autonomous --use-pod-delete-instead-of-ssh-kill" # noqa " --policy-file %s --kubeconfig %s --no-cloud" " --inventory-kubernetes --headless" @@ -217,6 +221,7 @@ def main(cfg): logging.info("Scenario: %s has been successfully injected!" % (scenario[0])) logging.info("Waiting for the specified duration: %s" % (wait_duration)) time.sleep(wait_duration) + failed_post_scenarios = post_actions(kubeconfig_path, scenario, failed_post_scenarios, pre_action_output) diff --git a/scenarios/etcd.yml b/scenarios/etcd.yml index a0e579b4..6a5f1152 100755 --- a/scenarios/etcd.yml +++ b/scenarios/etcd.yml @@ -18,3 +18,15 @@ scenarios: - kill: probability: 1 force: true + - podAction: + matches: + - labels: + namespace: "openshift-etcd" + selector: "k8s-app=etcd" + retries: + retriesTimeout: + timeout: 180 + + actions: + - checkPodCount: + count: 3 \ No newline at end of file diff --git a/scenarios/openshift-apiserver.yml b/scenarios/openshift-apiserver.yml index dbf329d6..0104b114 100755 --- a/scenarios/openshift-apiserver.yml +++ b/scenarios/openshift-apiserver.yml @@ -21,3 +21,15 @@ scenarios: - kill: probability: 1 force: true + - podAction: + matches: + - labels: + namespace: "openshift-apiserver" + selector: "app=openshift-apiserver" + retries: + retriesTimeout: + timeout: 180 + + actions: + - checkPodCount: + count: 3 \ No newline at end of file diff --git a/scenarios/openshift-kube-apiserver.yml b/scenarios/openshift-kube-apiserver.yml index 94c72c5d..2112557f 100755 --- a/scenarios/openshift-kube-apiserver.yml +++ b/scenarios/openshift-kube-apiserver.yml @@ -20,3 +20,14 @@ scenarios: - kill: probability: 1 force: true + - podAction: + matches: + - labels: + namespace: "openshift-kube-apiserver" + selector: "app=openshift-kube-apiserver" + retries: + retriesTimeout: + timeout: 180 + actions: + - checkPodCount: + count: 3 \ No newline at end of file diff --git a/scenarios/prometheus.yml b/scenarios/prometheus.yml index 086764fc..6fc6daa4 100644 --- a/scenarios/prometheus.yml +++ b/scenarios/prometheus.yml @@ -21,3 +21,15 @@ scenarios: - kill: probability: 1 force: true + - podAction: + matches: + - labels: + namespace: "openshift-monitoring" + selector: "app=prometheus" + retries: + retriesTimeout: + timeout: 180 + + actions: + - checkPodCount: + count: 2 \ No newline at end of file From a704c77a396fd8ecd59e6cba2b9d1b1577c36aa5 Mon Sep 17 00:00:00 2001 From: prubenda Date: Fri, 11 Sep 2020 17:17:59 -0400 Subject: [PATCH 2/2] deleting post action files that are no longer needed --- scenarios/post_action_etcd.yml | 21 ----------------- scenarios/post_action_etcd_example.sh | 3 --- scenarios/post_action_openshift-apiserver.yml | 23 ------------------- .../post_action_openshift-kube-apiserver.yml | 21 ----------------- .../post_action_regex_openshift_pod_kill.yml | 18 --------------- 5 files changed, 86 deletions(-) delete mode 100755 scenarios/post_action_etcd.yml delete mode 100755 scenarios/post_action_etcd_example.sh delete mode 100755 scenarios/post_action_openshift-apiserver.yml delete mode 100755 scenarios/post_action_openshift-kube-apiserver.yml delete mode 100755 scenarios/post_action_regex_openshift_pod_kill.yml diff --git a/scenarios/post_action_etcd.yml b/scenarios/post_action_etcd.yml deleted file mode 100755 index 6a472e26..00000000 --- a/scenarios/post_action_etcd.yml +++ /dev/null @@ -1,21 +0,0 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 10 - minSecondsBetweenRuns: 1 -scenarios: - - name: "check 3 pods are in namespace with selector: etcd" - steps: - - podAction: - matches: - - labels: - namespace: "openshift-etcd" - selector: "k8s-app=etcd" - filters: - - property: - name: "state" - value: "Running" - # The actions will be executed in the order specified - actions: - - checkPodCount: - count: 3 \ No newline at end of file diff --git a/scenarios/post_action_etcd_example.sh b/scenarios/post_action_etcd_example.sh deleted file mode 100755 index f122291b..00000000 --- a/scenarios/post_action_etcd_example.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -pods="$(oc get pods -n openshift-etcd | grep -c Running)" -echo "$pods" diff --git a/scenarios/post_action_openshift-apiserver.yml b/scenarios/post_action_openshift-apiserver.yml deleted file mode 100755 index 938b74c2..00000000 --- a/scenarios/post_action_openshift-apiserver.yml +++ /dev/null @@ -1,23 +0,0 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "check 3 pods are in namespace with selector: openshift-apiserver" - steps: - - podAction: - matches: - - labels: - namespace: "openshift-apiserver" - selector: "app=openshift-apiserver" - - filters: - - property: - name: "state" - value: "Running" - - # The actions will be executed in the order specified - actions: - - checkPodCount: - count: 3 diff --git a/scenarios/post_action_openshift-kube-apiserver.yml b/scenarios/post_action_openshift-kube-apiserver.yml deleted file mode 100755 index 7487661b..00000000 --- a/scenarios/post_action_openshift-kube-apiserver.yml +++ /dev/null @@ -1,21 +0,0 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: "check 3 pods are in namespace with selector: openshift-kube-apiserver" - steps: - - podAction: - matches: - - labels: - namespace: "openshift-kube-apiserver" - selector: "app=openshift-kube-apiserver" - filters: - - property: - name: "state" - value: "Running" - # The actions will be executed in the order specified - actions: - - checkPodCount: - count: 3 diff --git a/scenarios/post_action_regex_openshift_pod_kill.yml b/scenarios/post_action_regex_openshift_pod_kill.yml deleted file mode 100755 index ba011b75..00000000 --- a/scenarios/post_action_regex_openshift_pod_kill.yml +++ /dev/null @@ -1,18 +0,0 @@ -config: - runStrategy: - runs: 1 - maxSecondsBetweenRuns: 30 - minSecondsBetweenRuns: 1 -scenarios: - - name: kill up to 3 pods in any openshift namespace - steps: - - podAction: - matches: - - namespace: "openshift-.*" - filters: - - property: - name: "state" - value: "Running" - actions: - - checkPodCount: - count: 146