diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 00000000..5fad664a --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,9 @@ +kraken: + kubeconfig_path: /root/.kube/config # Path to kubeconfig + scenarios: # List of policies/chaos scenarios to load + - scenarios/etcd.yml + - scenarios/openshift-kube-apiserver.yml + - scenarios/openshift-apiserver.yml + +tunings: + wait_duration: 60 # Duration to wait between each chaos scenario diff --git a/kraken/invoke/__init__.py b/kraken/invoke/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/kraken/invoke/command.py b/kraken/invoke/command.py new file mode 100644 index 00000000..152875ff --- /dev/null +++ b/kraken/invoke/command.py @@ -0,0 +1,12 @@ +import subprocess +import logging + + +# Invokes a given command and returns the stdout +def invoke(command): + try: + output = subprocess.check_output(command, shell=True, + universal_newlines=True) + except Exception: + logging.error("Failed to run %s" % (command)) + return output diff --git a/kraken/kubernetes/__init__.py b/kraken/kubernetes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/kraken/kubernetes/client.py b/kraken/kubernetes/client.py new file mode 100644 index 00000000..4a29ce1f --- /dev/null +++ b/kraken/kubernetes/client.py @@ -0,0 +1,97 @@ +from kubernetes import client, config +from kubernetes.client.rest import ApiException +import logging + +# Load kubeconfig and initialize kubernetes python client +def initialize_clients(kubeconfig_path): + global cli + config.load_kube_config(kubeconfig_path) + cli = client.CoreV1Api() + + +# List nodes in the cluster +def list_nodes(): + nodes = [] + try: + ret = cli.list_node(pretty=True) + except ApiException as e: + logging.error("Exception when calling CoreV1Api->list_node: %s\n" % e) + for node in ret.items: + nodes.append(node.metadata.name) + return nodes + + +# List pods in the given namespace +def list_pods(namespace): + pods = [] + try: + ret = cli.list_namespaced_pod(namespace, pretty=True) + except ApiException as e: + logging.error("Exception when calling \ + CoreV1Api->list_namespaced_pod: %s\n" % e) + for pod in ret.items: + pods.append(pod.metadata.name) + return pods + + +# Monitor the status of the cluster nodes and set the status to true or false +def monitor_nodes(): + nodes = list_nodes() + notready_nodes = [] + node_kerneldeadlock_status = "False" + for node in nodes: + try: + node_info = cli.read_node_status(node, pretty=True) + except ApiException as e: + logging.error("Exception when calling \ + CoreV1Api->read_node_status: %s\n" % e) + for condition in node_info.status.conditions: + if condition.type == "KernelDeadlock": + node_kerneldeadlock_status = condition.status + elif condition.type == "Ready": + node_ready_status = condition.status + else: + continue + if ( + node_kerneldeadlock_status != "False" # noqa + or node_ready_status != "True" # noqa + ): + notready_nodes.append(node) + if len(notready_nodes) != 0: + status = False + else: + status = True + return status, notready_nodes + + +# Monitor the status of the pods in the specified namespace +# and set the status to true or false +def monitor_namespace(namespace): + pods = list_pods(namespace) + notready_pods = [] + for pod in pods: + try: + pod_info = cli.read_namespaced_pod_status(pod, namespace, + pretty=True) + except ApiException as e: + logging.error("Exception when calling \ + CoreV1Api->read_namespaced_pod_status: %s\n" % e) + pod_status = pod_info.status.phase + if pod_status != "Running" \ + and pod_status != "Completed" \ + and pod_status != "Succeeded": + notready_pods.append(pod) + if len(notready_pods) != 0: + status = False + else: + status = True + return status, notready_pods + + +# Monitor component namespace +def monitor_component(iteration, component_namespace): + watch_component_status, failed_component_pods = \ + monitor_namespace(component_namespace) + logging.info("Iteration %s: %s: %s" + % (iteration, component_namespace, watch_component_status)) + return watch_component_status, failed_component_pods diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..54abded1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +datetime +pyfiglet +powerfulseal diff --git a/run_kraken.py b/run_kraken.py new file mode 100644 index 00000000..c55d1aa6 --- /dev/null +++ b/run_kraken.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +import sys +import os +import time +import optparse +import logging +import yaml +import kraken.kubernetes.client as kubecli +import kraken.invoke.command as runcommand +import pyfiglet + + +# Main function +def main(cfg): + # Start kraken + print(pyfiglet.figlet_format("kraken")) + logging.info("Starting kraken") + + # Parse and read the config + if os.path.isfile(cfg): + with open(cfg, 'r') as f: + config = yaml.full_load(f) + kubeconfig_path = config["kraken"]["kubeconfig_path"] + scenarios = config["kraken"]["scenarios"] + wait_duration = config["tunings"]["wait_duration"] + + # Initialize clients + if not os.path.isfile(kubeconfig_path): + kubeconfig_path = None + logging.info("Initializing client to talk to the Kubernetes cluster") + kubecli.initialize_clients(kubeconfig_path) + + # Cluster info + logging.info("Fetching cluster info") + cluster_version = runcommand.invoke("kubectl get clusterversion") + cluster_info = runcommand.invoke("kubectl cluster-info | awk 'NR==1' | sed -r " + "'s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g'") # noqa + logging.info("\n%s%s" % (cluster_version, cluster_info)) + + # Inject chaos scenarios specified in the config + try: + for scenario in scenarios: + logging.info("Injecting scenario: %s" %(scenario)) + runcommand.invoke("powerfulseal autonomous --use-pod-delete-instead-of-ssh-kill --policy-file %s --kubeconfig %s --no-cloud --inventory-kubernetes --headless" % (scenario,kubeconfig_path)) + logging.info("Scenario: %s has been successfully injected!" %(scenario)) + logging.info("Waiting for the specified duration: %s" %(wait_duration)) + time.sleep(wait_duration) + except: + logging.error("Failed to run scenario: %s, please check" %(scenario)) + else: + logging.error("Cannot find a config at %s, please check" % (cfg)) + sys.exit(1) + + +if __name__ == "__main__": + # Initialize the parser to read the config + parser = optparse.OptionParser() + parser.add_option( + "-c", "--config", + dest="cfg", + help="config location", + default="config/config.yaml", + ) + (options, args) = parser.parse_args() + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler("kraken.report", mode='w'), + logging.StreamHandler() + ] + ) + if (options.cfg is None): + logging.error("Please check if you have passed the config") + sys.exit(1) + else: + main(options.cfg) diff --git a/scenarios/etcd.yml b/scenarios/etcd.yml new file mode 100644 index 00000000..c9468ab3 --- /dev/null +++ b/scenarios/etcd.yml @@ -0,0 +1,23 @@ +config: + loopsNumber: 1 + minSecondsBetweenRuns: 1 + maxSecondsBetweenRuns: 30 + +# the scenarios describing actions on kubernetes pods +podScenarios: + - name: "delete etcd pods" + + match: + - labels: + namespace: "openshift-etcd" + selector: "k8s-app=etcd" + + filters: + - randomSample: + size: 1 + + # The actions will be executed in the order specified + actions: + - kill: + probability: 1 + force: true diff --git a/scenarios/openshift-apiserver.yml b/scenarios/openshift-apiserver.yml new file mode 100644 index 00000000..b3b41ca0 --- /dev/null +++ b/scenarios/openshift-apiserver.yml @@ -0,0 +1,23 @@ +config: + loopsNumber: 1 + minSecondsBetweenRuns: 1 + maxSecondsBetweenRuns: 30 + +# the scenarios describing actions on kubernetes pods +podScenarios: + - name: "delete openshift-apiserver pods" + + match: + - labels: + namespace: "openshift-apiserver" + selector: "app=openshift-apiserver" + + filters: + - randomSample: + size: 1 + + # The actions will be executed in the order specified + actions: + - kill: + probability: 1 + force: true diff --git a/scenarios/openshift-kube-apiserver.yml b/scenarios/openshift-kube-apiserver.yml new file mode 100644 index 00000000..05030dc3 --- /dev/null +++ b/scenarios/openshift-kube-apiserver.yml @@ -0,0 +1,23 @@ +config: + loopsNumber: 1 + minSecondsBetweenRuns: 1 + maxSecondsBetweenRuns: 30 + +# the scenarios describing actions on kubernetes pods +podScenarios: + - name: "delete openshift-kube-apiserver pods" + + match: + - labels: + namespace: "openshift-kube-apiserver" + selector: "app=openshift-kube-apiserver" + + filters: + - randomSample: + size: 1 + + # The actions will be executed in the order specified + actions: + - kill: + probability: 1 + force: true