Add initial version of kraken

This commit:
- Adds support to run pod chaos scenarios including killing an etcd,
  ApiServer and kube-apiserver using powerfulseal tool.
- Adds support to create a report with the details about each chaos
  injection along with timestamps. The report is generated in the
  run directory.
- Adds kubernetes package with a bunch of functions which can be
  used later to talk to the kubernetes API to be able to know the
  status of the targeted components/nodes.

config/config.yaml

@@ -0,0 +1,9 @@
+kraken:
+    kubeconfig_path: /root/.kube/config                    # Path to kubeconfig
+    scenarios:                                             # List of policies/chaos scenarios to load
+        -    scenarios/etcd.yml
+        -    scenarios/openshift-kube-apiserver.yml                           
+        -    scenarios/openshift-apiserver.yml
+
+tunings:
+    wait_duration: 60                                      # Duration to wait between each chaos scenario 

kraken/invoke/command.py

@@ -0,0 +1,12 @@
+import subprocess
+import logging
+
+
+# Invokes a given command and returns the stdout
+def invoke(command):
+    try:
+        output = subprocess.check_output(command, shell=True,
+                                         universal_newlines=True)
+    except Exception:
+        logging.error("Failed to run %s" % (command))
+    return output

kraken/kubernetes/client.py

@@ -0,0 +1,97 @@
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+import logging
+
+# Load kubeconfig and initialize kubernetes python client
+def initialize_clients(kubeconfig_path):
+    global cli
+    config.load_kube_config(kubeconfig_path)
+    cli = client.CoreV1Api()
+
+
+# List nodes in the cluster
+def list_nodes():
+    nodes = []
+    try:
+        ret = cli.list_node(pretty=True)
+    except ApiException as e:
+        logging.error("Exception when calling CoreV1Api->list_node: %s\n" % e)
+    for node in ret.items:
+        nodes.append(node.metadata.name)
+    return nodes
+
+
+# List pods in the given namespace
+def list_pods(namespace):
+    pods = []
+    try:
+        ret = cli.list_namespaced_pod(namespace, pretty=True)
+    except ApiException as e:
+        logging.error("Exception when calling \
+                       CoreV1Api->list_namespaced_pod: %s\n" % e)
+    for pod in ret.items:
+        pods.append(pod.metadata.name)
+    return pods
+
+
+# Monitor the status of the cluster nodes and set the status to true or false
+def monitor_nodes():
+    nodes = list_nodes()
+    notready_nodes = []
+    node_kerneldeadlock_status = "False"
+    for node in nodes:
+        try:
+            node_info = cli.read_node_status(node, pretty=True)
+        except ApiException as e:
+            logging.error("Exception when calling \
+                           CoreV1Api->read_node_status: %s\n" % e)
+        for condition in node_info.status.conditions:
+            if condition.type == "KernelDeadlock":
+                node_kerneldeadlock_status = condition.status
+            elif condition.type == "Ready":
+                node_ready_status = condition.status
+            else:
+                continue
+        if (
+            node_kerneldeadlock_status != "False"       # noqa
+            or node_ready_status != "True"               # noqa
+        ):
+            notready_nodes.append(node)
+    if len(notready_nodes) != 0:
+        status = False
+    else:
+        status = True
+    return status, notready_nodes
+
+
+# Monitor the status of the pods in the specified namespace
+# and set the status to true or false
+def monitor_namespace(namespace):
+    pods = list_pods(namespace)
+    notready_pods = []
+    for pod in pods:
+        try:
+            pod_info = cli.read_namespaced_pod_status(pod, namespace,
+                                                      pretty=True)
+        except ApiException as e:
+            logging.error("Exception when calling \
+                           CoreV1Api->read_namespaced_pod_status: %s\n" % e)
+        pod_status = pod_info.status.phase
+        if pod_status != "Running" \
+            and pod_status != "Completed" \
+            and pod_status != "Succeeded":
+            notready_pods.append(pod)
+    if len(notready_pods) != 0:
+        status = False
+    else:
+        status = True
+    return status, notready_pods
+
+
+# Monitor component namespace
+def monitor_component(iteration, component_namespace):
+    watch_component_status, failed_component_pods = \
+        monitor_namespace(component_namespace)
+    logging.info("Iteration %s: %s: %s"
+                 % (iteration, component_namespace, watch_component_status))
+    return watch_component_status, failed_component_pods

requirements.txt

@@ -0,0 +1,3 @@
+datetime
+pyfiglet
+powerfulseal

run_kraken.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import time
+import optparse
+import logging
+import yaml
+import kraken.kubernetes.client as kubecli
+import kraken.invoke.command as runcommand
+import pyfiglet
+
+
+# Main function
+def main(cfg):
+    # Start kraken
+    print(pyfiglet.figlet_format("kraken"))
+    logging.info("Starting kraken")
+
+    # Parse and read the config
+    if os.path.isfile(cfg):
+        with open(cfg, 'r') as f:
+            config = yaml.full_load(f)
+        kubeconfig_path = config["kraken"]["kubeconfig_path"]
+        scenarios = config["kraken"]["scenarios"]
+        wait_duration = config["tunings"]["wait_duration"]
+
+        # Initialize clients
+        if not os.path.isfile(kubeconfig_path):
+            kubeconfig_path = None
+        logging.info("Initializing client to talk to the Kubernetes cluster")
+        kubecli.initialize_clients(kubeconfig_path)
+
+        # Cluster info
+        logging.info("Fetching cluster info")
+        cluster_version = runcommand.invoke("kubectl get clusterversion")
+        cluster_info = runcommand.invoke("kubectl cluster-info | awk 'NR==1' | sed -r "
+                                         "'s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g'") # noqa
+        logging.info("\n%s%s" % (cluster_version, cluster_info))
+
+        # Inject chaos scenarios specified in the config
+        try:
+            for scenario in scenarios:
+                logging.info("Injecting scenario: %s" %(scenario))
+                runcommand.invoke("powerfulseal autonomous --use-pod-delete-instead-of-ssh-kill --policy-file %s --kubeconfig %s --no-cloud --inventory-kubernetes --headless" % (scenario,kubeconfig_path))
+                logging.info("Scenario: %s has been successfully injected!" %(scenario))
+                logging.info("Waiting for the specified duration: %s" %(wait_duration))
+                time.sleep(wait_duration) 
+        except:
+            logging.error("Failed to run scenario: %s, please check" %(scenario))
+    else:
+        logging.error("Cannot find a config at %s, please check" % (cfg))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    # Initialize the parser to read the config
+    parser = optparse.OptionParser()
+    parser.add_option(
+        "-c", "--config",
+        dest="cfg",
+        help="config location",
+        default="config/config.yaml",
+    )
+    (options, args) = parser.parse_args()
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(message)s",
+        handlers=[
+            logging.FileHandler("kraken.report", mode='w'),
+            logging.StreamHandler()
+        ]
+    )
+    if (options.cfg is None):
+        logging.error("Please check if you have passed the config")
+        sys.exit(1)
+    else:
+        main(options.cfg)

scenarios/etcd.yml

@@ -0,0 +1,23 @@
+config:
+  loopsNumber: 1
+  minSecondsBetweenRuns: 1
+  maxSecondsBetweenRuns: 30
+
+# the scenarios describing actions on kubernetes pods
+podScenarios:
+  - name: "delete etcd pods"
+
+    match:
+      - labels:
+          namespace: "openshift-etcd"
+          selector: "k8s-app=etcd"
+
+    filters:
+      - randomSample:
+          size: 1
+
+    # The actions will be executed in the order specified
+    actions:
+      - kill:
+          probability: 1
+          force: true

scenarios/openshift-apiserver.yml

@@ -0,0 +1,23 @@
+config:
+  loopsNumber: 1
+  minSecondsBetweenRuns: 1
+  maxSecondsBetweenRuns: 30
+
+# the scenarios describing actions on kubernetes pods
+podScenarios:
+  - name: "delete openshift-apiserver pods"
+
+    match:
+      - labels:
+          namespace: "openshift-apiserver"
+          selector: "app=openshift-apiserver" 
+
+    filters:
+      - randomSample:
+          size: 1
+
+    # The actions will be executed in the order specified
+    actions:
+      - kill:
+          probability: 1
+          force: true

scenarios/openshift-kube-apiserver.yml

@@ -0,0 +1,23 @@
+config:
+  loopsNumber: 1
+  minSecondsBetweenRuns: 1
+  maxSecondsBetweenRuns: 30
+
+# the scenarios describing actions on kubernetes pods
+podScenarios:
+  - name: "delete openshift-kube-apiserver pods"
+
+    match:
+      - labels:
+          namespace: "openshift-kube-apiserver"
+          selector: "app=openshift-kube-apiserver" 
+
+    filters:
+      - randomSample:
+          size: 1
+
+    # The actions will be executed in the order specified
+    actions:
+      - kill:
+          probability: 1
+          force: true