Adding Kraken to PerfScale Pipeline

This commit adds kraken to CI pipeline and thereby enabling chaos
scenarios to be injected on specified jump host.

CI/scenarios/etcd.yml

@@ -0,0 +1,20 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "delete etcd pods"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-etcd"
+              selector: "k8s-app=etcd"
+        filters:
+          - randomSample:
+              size: 1
+        actions:
+          - kill:
+              probability: 1
+              force: true

CI/scenarios/openshift-apiserver.yml

@@ -0,0 +1,23 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "delete openshift-apiserver pods"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-apiserver"
+              selector: "app=openshift-apiserver"
+
+        filters:
+          - randomSample:
+              size: 1
+
+        # The actions will be executed in the order specified
+        actions:
+          - kill:
+              probability: 1
+              force: true

CI/scenarios/openshift-kube-apiserver.yml

@@ -0,0 +1,22 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "delete openshift-kube-apiserver pods"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-kube-apiserver"
+              selector: "app=openshift-kube-apiserver"
+        filters:
+          - randomSample:
+              size: 1
+
+        # The actions will be executed in the order specified
+        actions:
+          - kill:
+              probability: 1
+              force: true

CI/scenarios/post_action_etcd.yml

@@ -0,0 +1,21 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 10
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "check 3 pods are in namespace with selector: etcd"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-etcd"
+              selector: "k8s-app=etcd"
+        filters:
+          - property:
+              name: "state"
+              value: "Running"
+        # The actions will be executed in the order specified
+        actions:
+          - checkPodCount:
+              count: 3

CI/scenarios/post_action_etcd_example.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pods="$(oc get pods -n openshift-etcd | grep -c Running)"
+echo "$pods"

CI/scenarios/post_action_etcd_example_py.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import subprocess
+import logging
+
+
+def run(cmd):
+    try:
+        output = subprocess.Popen(cmd, shell=True,
+                                  universal_newlines=True, stdout=subprocess.PIPE,
+                                  stderr=subprocess.STDOUT)
+        (out, err) = output.communicate()
+        logging.info("out " + str(out))
+    except Exception as e:
+        logging.error("Failed to run %s, error: %s" % (cmd, e))
+    return out
+
+
+pods_running = run("oc get pods -n openshift-etcd | grep -c Running").rstrip()
+
+if pods_running == str(3):
+    print("There were 3 pods running properly")
+else:
+    print("ERROR there were " + str(pods_running) + " pods running instead of 3")

CI/scenarios/post_action_openshift-apiserver.yml

@@ -0,0 +1,23 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "check 3 pods are in namespace with selector: openshift-apiserver"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-apiserver"
+              selector: "app=openshift-apiserver"
+
+        filters:
+          - property:
+              name: "state"
+              value: "Running"
+
+        # The actions will be executed in the order specified
+        actions:
+          - checkPodCount:
+              count: 3

CI/scenarios/post_action_openshift-kube-apiserver.yml

@@ -0,0 +1,21 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "check 3 pods are in namespace with selector: openshift-kube-apiserver"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-kube-apiserver"
+              selector: "app=openshift-kube-apiserver"
+        filters:
+          - property:
+              name: "state"
+              value: "Running"
+        # The actions will be executed in the order specified
+        actions:
+          - checkPodCount:
+              count: 3

CI/scenarios/post_action_prometheus.yml

@@ -0,0 +1,22 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 10
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "check 2 pods are in namespace with selector: prometheus"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-monitoring"
+              selector: "app=prometheus"
+        filters:
+          - property:
+              name: "state"
+              value: "Running"
+        # The actions will be executed in the order specified
+        actions:
+          - checkPodCount:
+              count: 2
+ 

CI/scenarios/post_action_regex.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+import subprocess
+import re
+import sys
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+import logging
+
+
+# List all namespaces
+def list_namespaces():
+    namespaces = []
+    try:
+        config.load_kube_config()
+        cli = client.CoreV1Api()
+        ret = cli.list_namespace(pretty=True)
+    except ApiException as e:
+        logging.error("Exception when calling \
+                       CoreV1Api->list_namespaced_pod: %s\n" % e)
+    for namespace in ret.items:
+        namespaces.append(namespace.metadata.name)
+    return namespaces
+
+
+# Check if all the watch_namespaces are valid
+def check_namespaces(namespaces):
+    try:
+        valid_namespaces = list_namespaces()
+        regex_namespaces = set(namespaces) - set(valid_namespaces)
+        final_namespaces = set(namespaces) - set(regex_namespaces)
+        valid_regex = set()
+        if regex_namespaces:
+            for namespace in valid_namespaces:
+                for regex_namespace in regex_namespaces:
+                    if re.search(regex_namespace, namespace):
+                        final_namespaces.add(namespace)
+                        valid_regex.add(regex_namespace)
+                        break
+        invalid_namespaces = regex_namespaces - valid_regex
+        if invalid_namespaces:
+            raise Exception("There exists no namespaces matching: %s" % (invalid_namespaces))
+        return list(final_namespaces)
+    except Exception as e:
+        logging.error("%s" % (e))
+        sys.exit(1)
+
+
+def run(cmd):
+    try:
+        output = subprocess.Popen(cmd, shell=True,
+                                  universal_newlines=True, stdout=subprocess.PIPE,
+                                  stderr=subprocess.STDOUT)
+        (out, err) = output.communicate()
+    except Exception as e:
+        logging.error("Failed to run %s, error: %s" % (cmd, e))
+    return out
+
+
+regex_namespace = ["openshift-.*"]
+namespaces = check_namespaces(regex_namespace)
+pods_running = 0
+for namespace in namespaces:
+    new_pods_running = run("oc get pods -n " + namespace + " | grep -c Running").rstrip()
+    try:
+        pods_running += int(new_pods_running)
+    except Exception:
+        continue
+print(pods_running)

CI/scenarios/post_action_regex.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+pods="$(oc get pods -n openshift-etcd | grep -c Running)"
+echo "$pods"
+
+if [ "$pods" -eq 3 ]
+then
+  echo "Pods Pass"
+else
+  # need capital error for proper error catching in run_kraken
+  echo "ERROR pod count $pods doesnt match 3 expected pods"
+fi

CI/scenarios/post_action_regex_openshift_pod_kill.yml

@@ -0,0 +1,18 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: kill up to 3 pods in any openshift namespace
+    steps:
+    - podAction:
+        matches:
+          - namespace: "openshift-.*"
+        filters:
+          - property:
+             name: "state"
+             value: "Running"
+        actions:
+          - checkPodCount:
+              count: 146

CI/scenarios/prometheus.yml

@@ -0,0 +1,23 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: "delete prometheus pods"
+    steps:
+    - podAction:
+        matches:
+          - labels:
+              namespace: "openshift-monitoring"
+              selector: "app=prometheus"
+
+        filters:
+          - randomSample:
+              size: 1
+
+        # The actions will be executed in the order specified
+        actions:
+          - kill:
+              probability: 1
+              force: true

CI/scenarios/regex_openshift_pod_kill.yml

@@ -0,0 +1,20 @@
+config:
+  runStrategy:
+    runs: 1
+    maxSecondsBetweenRuns: 30
+    minSecondsBetweenRuns: 1
+scenarios:
+  - name: kill up to 3 pods in any openshift namespace
+    steps:
+    - podAction:
+        matches:
+          - namespace: "openshift-.*"
+        filters:
+          - property:
+             name: "state"
+             value: "Running"
+          - randomSample:
+              size: 3
+        actions:
+          - kill:
+              probability: .7

ansible/ansible.cfg

@@ -0,0 +1,11 @@
+[defaults]
+callback_whitelist = profile_tasks
+host_key_checking = False
+log_path = ~/ansible.log
+retry_files_enabled = False
+# work around privilege escalation timeouts in ansible:
+timeout = 30
+
+[callback_profile_tasks]
+task_output_limit = 10000
+sort_order = none

ansible/inventory

@@ -0,0 +1 @@
+[orchestration]

ansible/kraken.yml

@@ -0,0 +1,26 @@
+---
+- hosts: orchestration
+  gather_facts: true
+  remote_user: "{{ orchestration_user }}"
+  vars_files:
+   - vars/kraken_vars.yml
+
+  tasks:
+  - name: Git clone kraken repository
+    git:
+      repo: "{{ kraken_repository }}"
+      dest: "{{ kraken_dir }}"
+      force: yes
+
+  - name: Generate kraken config file
+    template:
+      src: kraken.j2
+      dest: "{{ kraken_config }}"
+
+  - name: Start injecting failures
+    shell: |
+      cd "{{ kraken_dir }}"
+      cp -r "{{ scenarios_folder_path }}"* scenarios/
+      unset CONFIG
+      python3 run_kraken.py
+    ignore_errors: yes

ansible/templates/kraken.j2

@@ -0,0 +1,13 @@
+kraken:
+    kubeconfig_path: {{ kubeconfig_path }}                 # Path to kubeconfig
+    exit_on_failure: {{ exit_on_failure }}                 # Exit when a post action scenario fails
+    scenarios: {{ scenarios }}                             # List of policies/chaos scenarios to load
+
+cerberus:
+    cerberus_enabled: {{ cerberus_enabled }}               # Enable it when cerberus is previously installed
+    cerberus_url: {{ cerberus_url }}                       # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal
+
+tunings:
+    wait_duration: {{ wait_duration }}                     # Duration to wait between each chaos scenario
+    iterations: {{ iterations }}                           # Number of times to execute the scenarios
+    daemon_mode: {{ daemon_mode }}                         # Iterations are set to infinity which means that the cerberus will monitor the resources forever

ansible/vars/kraken_vars.yml

@@ -0,0 +1,35 @@
+###############################################################################
+# Ansible SSH variables.
+###############################################################################
+ansible_public_key_file: "{{ lookup('env', 'PUBLIC_KEY')|default('~/.ssh/id_rsa.pub', true) }}"
+ansible_private_key_file: "{{ lookup('env', 'PRIVATE_KEY')|default('~/.ssh/id_rsa', true) }}"
+
+orchestration_user: "{{ lookup('env', 'ORCHESTRATION_USER')|default('root', true) }}"
+###############################################################################
+
+# kube config location
+kubeconfig_path: "{{ lookup('env', 'KUBECONFIG_PATH')|default('/root/.kube/config', true) }}"
+
+# kraken dir location on jump host
+kraken_dir: "{{ lookup('env', 'KRAKEN_DIR')|default('/root/kraken', true) }}"
+
+# kraken config path location
+kraken_config: "{{ lookup('env', 'KRAKEN_CONFIG')|default('/root/kraken/config/config.yaml', true) }}"
+
+# kraken repository location
+kraken_repository: "{{ lookup('env', 'KRAKEN_REPOSITORY')|default('https://github.com/openshift-scale/kraken.git', true) }}"
+
+# scenarios to inject
+scenarios_folder_path: "{{ lookup('env', 'SCENARIOS_FOLDER_PATH')|default('CI/scenarios/', true) }}"
+scenarios: "{{ lookup('env', 'SCENARIOS')|default('[[scenarios/etcd.yml, scenarios/post_action_etcd_example.sh], [scenarios/openshift-apiserver.yml, scenarios/post_action_openshift-kube-apiserver.yml], [scenarios/openshift-kube-apiserver.yml, scenarios/post_action_openshift-apiserver.yml], [scenarios/regex_openshift_pod_kill.yml, scenarios/post_action_regex.py]]', true) }}"
+
+exit_on_failure: "{{ lookup('env', 'EXIT_ON_FAILURE')|default(false, true) }}"
+
+# Cerberus enabled by user 
+cerberus_enabled: "{{ lookup('env', 'CERBERUS_ENABLED')|default(false, true) }}"
+cerberus_url: "{{ lookup('env', 'CERBERUS_URL')|default('', true) }}"
+
+# Kraken configurations
+wait_duration: "{{ lookup('env', 'WAIT_DURATION')|default(60, true) }}"
+iterations: "{{ lookup('env', 'ITERATIONS')|default(1, true) }}"
+daemon_mode: "{{ lookup('env', 'DAEMON_MODE')|default(false, true) }}"