diff --git a/config/config.yaml b/config/config.yaml index ac7ffb9b..08d6a527 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -20,11 +20,11 @@ kraken: - time_scenarios: # List of chaos time scenarios to load - scenarios/time_scenarios_example.yml - litmus_scenarios: # List of litmus scenarios to load - - - https://raw.githubusercontent.com/cloud-bulldozer/kraken/master/scenarios/templates/litmus-rbac.yaml + - - scenarios/templates/litmus-rbac.yaml - scenarios/node_cpu_hog_engine.yaml - - - https://raw.githubusercontent.com/cloud-bulldozer/kraken/master/scenarios/templates/litmus-rbac.yaml + - - scenarios/templates/litmus-rbac.yaml - scenarios/node_mem_engine.yaml - - - https://raw.githubusercontent.com/cloud-bulldozer/kraken/master/scenarios/templates/litmus-rbac.yaml + - - scenarios/templates/litmus-rbac.yaml - scenarios/node_io_engine.yaml - cluster_shut_down_scenarios: - - scenarios/cluster_shut_down_scenario.yml diff --git a/kraken/invoke/command.py b/kraken/invoke/command.py index 5fe26fee..c367671a 100644 --- a/kraken/invoke/command.py +++ b/kraken/invoke/command.py @@ -1,15 +1,14 @@ import subprocess import logging -import sys # Invokes a given command and returns the stdout def invoke(command, timeout=None): + output = "" try: output = subprocess.check_output(command, shell=True, universal_newlines=True, timeout=timeout) except Exception as e: logging.error("Failed to run %s, error: %s" % (command, e)) - sys.exit(1) return output diff --git a/kraken/litmus/common_litmus.py b/kraken/litmus/common_litmus.py index faff7aa6..a924fa74 100644 --- a/kraken/litmus/common_litmus.py +++ b/kraken/litmus/common_litmus.py @@ -20,7 +20,6 @@ def run(scenarios_list, config, litmus_uninstall, wait_duration, litmus_namespac yaml_item = list(yaml.safe_load_all(f.content))[0] else: with open(item, "r") as f: - logging.info("opened yaml" + str(item)) yaml_item = list(yaml.safe_load_all(f))[0] if yaml_item["kind"] == "ChaosEngine": @@ -34,14 +33,10 @@ def run(scenarios_list, config, litmus_uninstall, wait_duration, litmus_namespac else: logging.info("Scenario: %s was not successfully injected, please check" % item) if litmus_uninstall: - for l_item in l_scenario: - logging.info("item " + str(l_item)) - runcommand.invoke("kubectl delete -f %s" % l_item) + delete_chaos(litmus_namespace) sys.exit(1) if litmus_uninstall: - for item in l_scenario: - logging.info("item " + str(item)) - runcommand.invoke("kubectl delete -f %s" % item) + delete_chaos(litmus_namespace) logging.info("Waiting for the specified duration: %s" % wait_duration) time.sleep(wait_duration) end_time = int(time.time()) @@ -53,6 +48,7 @@ def run(scenarios_list, config, litmus_uninstall, wait_duration, litmus_namespac # Install litmus and wait until pod is running def install_litmus(version, namespace): + logging.info("Installing version %s of litmus in namespace %s" % (version, namespace)) litmus_install = runcommand.invoke( "kubectl -n %s apply -f " "https://litmuschaos.github.io/litmus/litmus-operator-%s.yaml" % (namespace, version) ) @@ -65,7 +61,7 @@ def install_litmus(version, namespace): '[ { "op": "add", "path": "/spec/template/spec/containers/0/env/-", ' '"value": { "name": "ANALYTICS", "value": "FALSE" } } ]\'' % namespace ) - + logging.info("Waiting for litmus operator to become available") runcommand.invoke("oc wait deploy -n %s chaos-operator-ce --for=condition=Available" % namespace) @@ -75,68 +71,86 @@ def deploy_all_experiments(version_string, namespace): logging.error("Incorrect version string for litmus, needs to start with 'v' " "followed by a number") sys.exit(1) version = version_string[1:] - + logging.info("Installing all litmus experiments") runcommand.invoke( "kubectl -n %s apply -f " "https://hub.litmuschaos.io/api/chaos/%s?file=charts/generic/experiments.yaml" % (namespace, version) ) -def delete_experiments(namespace): - runcommand.invoke("kubectl -n %s delete chaosengine --all" % namespace) - - -# Check status of experiment -def check_experiment(engine_name, experiment_name, namespace): +def wait_for_initialized(engine_name, experiment_name, namespace): chaos_engine = runcommand.invoke( - "kubectl get chaosengines/%s -n %s -o jsonpath=" "'{.status.engineStatus}'" % (engine_name, namespace) + "kubectl get chaosengines/%s -n %s -o jsonpath='{.status.engineStatus}'" % (engine_name, namespace) ) engine_status = chaos_engine.strip() max_tries = 30 engine_counter = 0 - while engine_status.lower() != "running" and engine_status.lower() != "completed": + while engine_status.lower() != "initialized": time.sleep(10) - logging.info("Waiting for engine to start running.") + logging.info("Waiting for " + experiment_name + " to be initialized") chaos_engine = runcommand.invoke( - "kubectl get chaosengines/%s -n %s -o jsonpath=" "'{.status.engineStatus}'" % (engine_name, namespace) + "kubectl get chaosengines/%s -n %s -o jsonpath='{.status.engineStatus}'" % (engine_name, namespace) ) engine_status = chaos_engine.strip() if engine_counter >= max_tries: - logging.error("Chaos engine took longer than 5 minutes to be running or complete") + logging.error("Chaos engine " + experiment_name + " took longer than 5 minutes to be initialized") return False engine_counter += 1 # need to see if error in run if "notfound" in engine_status.lower(): logging.info("Chaos engine was not found") return False + return True + + +def wait_for_status(engine_name, expected_status, experiment_name, namespace): + + if expected_status == "running": + response = wait_for_initialized(engine_name, experiment_name, namespace) + if not response: + logging.info("Chaos engine never initialized, exiting") + return False + chaos_engine = runcommand.invoke( + "kubectl get chaosengines/%s -n %s -o jsonpath='{.status.experiments[0].status}'" % (engine_name, namespace) + ) + engine_status = chaos_engine.strip() + max_tries = 30 + engine_counter = 0 + while engine_status.lower() != expected_status: + time.sleep(10) + logging.info("Waiting for " + experiment_name + " to be " + expected_status) + chaos_engine = runcommand.invoke( + "kubectl get chaosengines/%s -n %s -o jsonpath='{.status.experiments[0].status}'" % (engine_name, namespace) + ) + engine_status = chaos_engine.strip() + if engine_counter >= max_tries: + logging.error("Chaos engine " + experiment_name + " took longer than 5 minutes to be " + expected_status) + return False + engine_counter += 1 + # need to see if error in run + if "notfound" in engine_status.lower(): + logging.info("Chaos engine was not found") + return False + return True + + +# Check status of experiment +def check_experiment(engine_name, experiment_name, namespace): + + wait_response = wait_for_status(engine_name, "running", experiment_name, namespace) + + if wait_response: + wait_for_status(engine_name, "completed", experiment_name, namespace) + else: + sys.exit(1) - if not chaos_engine: - return False chaos_result = runcommand.invoke( "kubectl get chaosresult %s" "-%s -n %s -o " "jsonpath='{.status.experimentStatus.verdict}'" % (engine_name, experiment_name, namespace) ) - result_counter = 0 - status = chaos_result.strip() - while status == "Awaited": - logging.info("Waiting for chaos result to finish, sleeping 10 seconds") - time.sleep(10) - chaos_result = runcommand.invoke( - "kubectl get chaosresult %s" - "-%s -n %s -o " - "jsonpath='{.status.experimentStatus.verdict}'" % (engine_name, experiment_name, namespace) - ) - status = chaos_result.strip() - if result_counter >= max_tries: - logging.error("Chaos results took longer than 5 minutes to get a final result") - return False - result_counter += 1 - if "notfound" in status.lower(): - logging.info("Chaos result was not found") - return False - - if status == "Pass": + if chaos_result == "Pass": + logging.info("Engine " + str(engine_name) + " finished with status " + str(chaos_result)) return True else: chaos_result = runcommand.invoke( @@ -144,17 +158,40 @@ def check_experiment(engine_name, experiment_name, namespace): "-%s -n %s -o jsonpath=" "'{.status.experimentStatus.failStep}'" % (engine_name, experiment_name, namespace) ) - logging.info("Chaos result failed information: " + str(chaos_result)) + logging.info("Chaos scenario:" + engine_name + " failed with error: " + str(chaos_result)) + logging.info( + "See 'kubectl get chaosresult %s" + "-%s -n %s -o yaml' for full results" % (engine_name, experiment_name, namespace) + ) return False +# Delete all chaos engines in a given namespace +def delete_chaos_experiments(namespace): + + namespace_exists = runcommand.invoke("oc get project -o name | grep -c " + namespace + " | xargs") + if namespace_exists.strip() != "0": + logging.info("Deleting all litmus experiments") + runcommand.invoke("kubectl delete chaosexperiment --all -n " + str(namespace)) + + # Delete all chaos engines in a given namespace def delete_chaos(namespace): - runcommand.invoke("kubectl delete chaosengine --all -n " + str(namespace)) - runcommand.invoke("kubectl delete chaosexperiment --all -n " + str(namespace)) - runcommand.invoke("kubectl delete chaosresult --all -n " + str(namespace)) + + namespace_exists = runcommand.invoke("oc get project -o name | grep -c " + namespace + " | xargs") + if namespace_exists.strip() != "0": + logging.info("Deleting all litmus run objects") + runcommand.invoke("kubectl delete chaosengine --all -n " + str(namespace)) + runcommand.invoke("kubectl delete chaosresult --all -n " + str(namespace)) + else: + logging.info(namespace + " namespace doesn't exist") -# Uninstall litmus operator -def uninstall_litmus(version): - runcommand.invoke("kubectl delete -f " "https://litmuschaos.github.io/litmus/litmus-operator-%s.yaml" % version) +def uninstall_litmus(version, litmus_namespace): + namespace_exists = runcommand.invoke("oc get project -o name | grep -c " + litmus_namespace + " | xargs") + if namespace_exists.strip() != "0": + logging.info("Uninstallting Litmus operator") + runcommand.invoke( + "kubectl delete -n %s -f " + "https://litmuschaos.github.io/litmus/litmus-operator-%s.yaml" % (litmus_namespace, version) + ) diff --git a/run_kraken.py b/run_kraken.py index 1a000915..ca5ee524 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -142,6 +142,10 @@ def main(cfg): elif scenario_type == "litmus_scenarios": logging.info("Running litmus scenarios") if not litmus_installed: + # Will always uninstall first + common_litmus.delete_chaos(litmus_namespace) + common_litmus.delete_chaos_experiments(litmus_namespace) + common_litmus.uninstall_litmus(litmus_version, litmus_namespace) common_litmus.install_litmus(litmus_version, litmus_namespace) common_litmus.deploy_all_experiments(litmus_version, litmus_namespace) litmus_installed = True @@ -198,8 +202,8 @@ def main(cfg): if litmus_uninstall and litmus_installed: common_litmus.delete_chaos(litmus_namespace) - common_litmus.delete_experiments(litmus_namespace) - common_litmus.uninstall_litmus(litmus_version) + common_litmus.delete_chaos_experiments(litmus_namespace) + common_litmus.uninstall_litmus(litmus_version, litmus_namespace) if failed_post_scenarios: logging.error("Post scenarios are still failing at the end of all iterations")