From aac254ce453643ad50ef3c0ef02dec6c1b9f53b6 Mon Sep 17 00:00:00 2001 From: Yashashree Suresh Date: Wed, 19 Aug 2020 00:41:06 +0530 Subject: [PATCH] Adding Kraken to PerfScale Pipeline This commit adds kraken to CI pipeline and thereby enabling chaos scenarios to be injected on specified jump host. --- CI/scenarios/etcd.yml | 20 ++++++ CI/scenarios/openshift-apiserver.yml | 23 +++++++ CI/scenarios/openshift-kube-apiserver.yml | 22 ++++++ CI/scenarios/post_action_etcd.yml | 21 ++++++ CI/scenarios/post_action_etcd_example.sh | 3 + CI/scenarios/post_action_etcd_example_py.py | 23 +++++++ .../post_action_openshift-apiserver.yml | 23 +++++++ .../post_action_openshift-kube-apiserver.yml | 21 ++++++ CI/scenarios/post_action_prometheus.yml | 22 ++++++ CI/scenarios/post_action_regex.py | 68 +++++++++++++++++++ CI/scenarios/post_action_regex.sh | 11 +++ .../post_action_regex_openshift_pod_kill.yml | 18 +++++ CI/scenarios/prometheus.yml | 23 +++++++ CI/scenarios/regex_openshift_pod_kill.yml | 20 ++++++ ansible/ansible.cfg | 11 +++ ansible/inventory | 1 + ansible/kraken.yml | 26 +++++++ ansible/templates/kraken.j2 | 13 ++++ ansible/vars/kraken_vars.yml | 35 ++++++++++ 19 files changed, 404 insertions(+) create mode 100755 CI/scenarios/etcd.yml create mode 100755 CI/scenarios/openshift-apiserver.yml create mode 100755 CI/scenarios/openshift-kube-apiserver.yml create mode 100755 CI/scenarios/post_action_etcd.yml create mode 100755 CI/scenarios/post_action_etcd_example.sh create mode 100755 CI/scenarios/post_action_etcd_example_py.py create mode 100755 CI/scenarios/post_action_openshift-apiserver.yml create mode 100755 CI/scenarios/post_action_openshift-kube-apiserver.yml create mode 100644 CI/scenarios/post_action_prometheus.yml create mode 100755 CI/scenarios/post_action_regex.py create mode 100755 CI/scenarios/post_action_regex.sh create mode 100755 CI/scenarios/post_action_regex_openshift_pod_kill.yml create mode 100644 CI/scenarios/prometheus.yml create mode 100755 CI/scenarios/regex_openshift_pod_kill.yml create mode 100644 ansible/ansible.cfg create mode 100644 ansible/inventory create mode 100644 ansible/kraken.yml create mode 100644 ansible/templates/kraken.j2 create mode 100644 ansible/vars/kraken_vars.yml diff --git a/CI/scenarios/etcd.yml b/CI/scenarios/etcd.yml new file mode 100755 index 00000000..a0e579b4 --- /dev/null +++ b/CI/scenarios/etcd.yml @@ -0,0 +1,20 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: "delete etcd pods" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-etcd" + selector: "k8s-app=etcd" + filters: + - randomSample: + size: 1 + actions: + - kill: + probability: 1 + force: true diff --git a/CI/scenarios/openshift-apiserver.yml b/CI/scenarios/openshift-apiserver.yml new file mode 100755 index 00000000..dbf329d6 --- /dev/null +++ b/CI/scenarios/openshift-apiserver.yml @@ -0,0 +1,23 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: "delete openshift-apiserver pods" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-apiserver" + selector: "app=openshift-apiserver" + + filters: + - randomSample: + size: 1 + + # The actions will be executed in the order specified + actions: + - kill: + probability: 1 + force: true diff --git a/CI/scenarios/openshift-kube-apiserver.yml b/CI/scenarios/openshift-kube-apiserver.yml new file mode 100755 index 00000000..94c72c5d --- /dev/null +++ b/CI/scenarios/openshift-kube-apiserver.yml @@ -0,0 +1,22 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: "delete openshift-kube-apiserver pods" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-kube-apiserver" + selector: "app=openshift-kube-apiserver" + filters: + - randomSample: + size: 1 + + # The actions will be executed in the order specified + actions: + - kill: + probability: 1 + force: true diff --git a/CI/scenarios/post_action_etcd.yml b/CI/scenarios/post_action_etcd.yml new file mode 100755 index 00000000..6a472e26 --- /dev/null +++ b/CI/scenarios/post_action_etcd.yml @@ -0,0 +1,21 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 10 + minSecondsBetweenRuns: 1 +scenarios: + - name: "check 3 pods are in namespace with selector: etcd" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-etcd" + selector: "k8s-app=etcd" + filters: + - property: + name: "state" + value: "Running" + # The actions will be executed in the order specified + actions: + - checkPodCount: + count: 3 \ No newline at end of file diff --git a/CI/scenarios/post_action_etcd_example.sh b/CI/scenarios/post_action_etcd_example.sh new file mode 100755 index 00000000..f122291b --- /dev/null +++ b/CI/scenarios/post_action_etcd_example.sh @@ -0,0 +1,3 @@ +#!/bin/bash +pods="$(oc get pods -n openshift-etcd | grep -c Running)" +echo "$pods" diff --git a/CI/scenarios/post_action_etcd_example_py.py b/CI/scenarios/post_action_etcd_example_py.py new file mode 100755 index 00000000..1d840625 --- /dev/null +++ b/CI/scenarios/post_action_etcd_example_py.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import subprocess +import logging + + +def run(cmd): + try: + output = subprocess.Popen(cmd, shell=True, + universal_newlines=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + (out, err) = output.communicate() + logging.info("out " + str(out)) + except Exception as e: + logging.error("Failed to run %s, error: %s" % (cmd, e)) + return out + + +pods_running = run("oc get pods -n openshift-etcd | grep -c Running").rstrip() + +if pods_running == str(3): + print("There were 3 pods running properly") +else: + print("ERROR there were " + str(pods_running) + " pods running instead of 3") diff --git a/CI/scenarios/post_action_openshift-apiserver.yml b/CI/scenarios/post_action_openshift-apiserver.yml new file mode 100755 index 00000000..938b74c2 --- /dev/null +++ b/CI/scenarios/post_action_openshift-apiserver.yml @@ -0,0 +1,23 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: "check 3 pods are in namespace with selector: openshift-apiserver" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-apiserver" + selector: "app=openshift-apiserver" + + filters: + - property: + name: "state" + value: "Running" + + # The actions will be executed in the order specified + actions: + - checkPodCount: + count: 3 diff --git a/CI/scenarios/post_action_openshift-kube-apiserver.yml b/CI/scenarios/post_action_openshift-kube-apiserver.yml new file mode 100755 index 00000000..7487661b --- /dev/null +++ b/CI/scenarios/post_action_openshift-kube-apiserver.yml @@ -0,0 +1,21 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: "check 3 pods are in namespace with selector: openshift-kube-apiserver" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-kube-apiserver" + selector: "app=openshift-kube-apiserver" + filters: + - property: + name: "state" + value: "Running" + # The actions will be executed in the order specified + actions: + - checkPodCount: + count: 3 diff --git a/CI/scenarios/post_action_prometheus.yml b/CI/scenarios/post_action_prometheus.yml new file mode 100644 index 00000000..ebae7879 --- /dev/null +++ b/CI/scenarios/post_action_prometheus.yml @@ -0,0 +1,22 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 10 + minSecondsBetweenRuns: 1 +scenarios: + - name: "check 2 pods are in namespace with selector: prometheus" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-monitoring" + selector: "app=prometheus" + filters: + - property: + name: "state" + value: "Running" + # The actions will be executed in the order specified + actions: + - checkPodCount: + count: 2 + \ No newline at end of file diff --git a/CI/scenarios/post_action_regex.py b/CI/scenarios/post_action_regex.py new file mode 100755 index 00000000..ce12ab8c --- /dev/null +++ b/CI/scenarios/post_action_regex.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import subprocess +import re +import sys +from kubernetes import client, config +from kubernetes.client.rest import ApiException +import logging + + +# List all namespaces +def list_namespaces(): + namespaces = [] + try: + config.load_kube_config() + cli = client.CoreV1Api() + ret = cli.list_namespace(pretty=True) + except ApiException as e: + logging.error("Exception when calling \ + CoreV1Api->list_namespaced_pod: %s\n" % e) + for namespace in ret.items: + namespaces.append(namespace.metadata.name) + return namespaces + + +# Check if all the watch_namespaces are valid +def check_namespaces(namespaces): + try: + valid_namespaces = list_namespaces() + regex_namespaces = set(namespaces) - set(valid_namespaces) + final_namespaces = set(namespaces) - set(regex_namespaces) + valid_regex = set() + if regex_namespaces: + for namespace in valid_namespaces: + for regex_namespace in regex_namespaces: + if re.search(regex_namespace, namespace): + final_namespaces.add(namespace) + valid_regex.add(regex_namespace) + break + invalid_namespaces = regex_namespaces - valid_regex + if invalid_namespaces: + raise Exception("There exists no namespaces matching: %s" % (invalid_namespaces)) + return list(final_namespaces) + except Exception as e: + logging.error("%s" % (e)) + sys.exit(1) + + +def run(cmd): + try: + output = subprocess.Popen(cmd, shell=True, + universal_newlines=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + (out, err) = output.communicate() + except Exception as e: + logging.error("Failed to run %s, error: %s" % (cmd, e)) + return out + + +regex_namespace = ["openshift-.*"] +namespaces = check_namespaces(regex_namespace) +pods_running = 0 +for namespace in namespaces: + new_pods_running = run("oc get pods -n " + namespace + " | grep -c Running").rstrip() + try: + pods_running += int(new_pods_running) + except Exception: + continue +print(pods_running) diff --git a/CI/scenarios/post_action_regex.sh b/CI/scenarios/post_action_regex.sh new file mode 100755 index 00000000..10626cc0 --- /dev/null +++ b/CI/scenarios/post_action_regex.sh @@ -0,0 +1,11 @@ +#!/bin/bash +pods="$(oc get pods -n openshift-etcd | grep -c Running)" +echo "$pods" + +if [ "$pods" -eq 3 ] +then + echo "Pods Pass" +else + # need capital error for proper error catching in run_kraken + echo "ERROR pod count $pods doesnt match 3 expected pods" +fi diff --git a/CI/scenarios/post_action_regex_openshift_pod_kill.yml b/CI/scenarios/post_action_regex_openshift_pod_kill.yml new file mode 100755 index 00000000..ba011b75 --- /dev/null +++ b/CI/scenarios/post_action_regex_openshift_pod_kill.yml @@ -0,0 +1,18 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: kill up to 3 pods in any openshift namespace + steps: + - podAction: + matches: + - namespace: "openshift-.*" + filters: + - property: + name: "state" + value: "Running" + actions: + - checkPodCount: + count: 146 diff --git a/CI/scenarios/prometheus.yml b/CI/scenarios/prometheus.yml new file mode 100644 index 00000000..086764fc --- /dev/null +++ b/CI/scenarios/prometheus.yml @@ -0,0 +1,23 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: "delete prometheus pods" + steps: + - podAction: + matches: + - labels: + namespace: "openshift-monitoring" + selector: "app=prometheus" + + filters: + - randomSample: + size: 1 + + # The actions will be executed in the order specified + actions: + - kill: + probability: 1 + force: true diff --git a/CI/scenarios/regex_openshift_pod_kill.yml b/CI/scenarios/regex_openshift_pod_kill.yml new file mode 100755 index 00000000..9bcce66b --- /dev/null +++ b/CI/scenarios/regex_openshift_pod_kill.yml @@ -0,0 +1,20 @@ +config: + runStrategy: + runs: 1 + maxSecondsBetweenRuns: 30 + minSecondsBetweenRuns: 1 +scenarios: + - name: kill up to 3 pods in any openshift namespace + steps: + - podAction: + matches: + - namespace: "openshift-.*" + filters: + - property: + name: "state" + value: "Running" + - randomSample: + size: 3 + actions: + - kill: + probability: .7 diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 00000000..5949187e --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +callback_whitelist = profile_tasks +host_key_checking = False +log_path = ~/ansible.log +retry_files_enabled = False +# work around privilege escalation timeouts in ansible: +timeout = 30 + +[callback_profile_tasks] +task_output_limit = 10000 +sort_order = none diff --git a/ansible/inventory b/ansible/inventory new file mode 100644 index 00000000..2b8139de --- /dev/null +++ b/ansible/inventory @@ -0,0 +1 @@ +[orchestration] diff --git a/ansible/kraken.yml b/ansible/kraken.yml new file mode 100644 index 00000000..3bae962b --- /dev/null +++ b/ansible/kraken.yml @@ -0,0 +1,26 @@ +--- +- hosts: orchestration + gather_facts: true + remote_user: "{{ orchestration_user }}" + vars_files: + - vars/kraken_vars.yml + + tasks: + - name: Git clone kraken repository + git: + repo: "{{ kraken_repository }}" + dest: "{{ kraken_dir }}" + force: yes + + - name: Generate kraken config file + template: + src: kraken.j2 + dest: "{{ kraken_config }}" + + - name: Start injecting failures + shell: | + cd "{{ kraken_dir }}" + cp -r "{{ scenarios_folder_path }}"* scenarios/ + unset CONFIG + python3 run_kraken.py + ignore_errors: yes diff --git a/ansible/templates/kraken.j2 b/ansible/templates/kraken.j2 new file mode 100644 index 00000000..798e2aa7 --- /dev/null +++ b/ansible/templates/kraken.j2 @@ -0,0 +1,13 @@ +kraken: + kubeconfig_path: {{ kubeconfig_path }} # Path to kubeconfig + exit_on_failure: {{ exit_on_failure }} # Exit when a post action scenario fails + scenarios: {{ scenarios }} # List of policies/chaos scenarios to load + +cerberus: + cerberus_enabled: {{ cerberus_enabled }} # Enable it when cerberus is previously installed + cerberus_url: {{ cerberus_url }} # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal + +tunings: + wait_duration: {{ wait_duration }} # Duration to wait between each chaos scenario + iterations: {{ iterations }} # Number of times to execute the scenarios + daemon_mode: {{ daemon_mode }} # Iterations are set to infinity which means that the cerberus will monitor the resources forever diff --git a/ansible/vars/kraken_vars.yml b/ansible/vars/kraken_vars.yml new file mode 100644 index 00000000..f35de533 --- /dev/null +++ b/ansible/vars/kraken_vars.yml @@ -0,0 +1,35 @@ +############################################################################### +# Ansible SSH variables. +############################################################################### +ansible_public_key_file: "{{ lookup('env', 'PUBLIC_KEY')|default('~/.ssh/id_rsa.pub', true) }}" +ansible_private_key_file: "{{ lookup('env', 'PRIVATE_KEY')|default('~/.ssh/id_rsa', true) }}" + +orchestration_user: "{{ lookup('env', 'ORCHESTRATION_USER')|default('root', true) }}" +############################################################################### + +# kube config location +kubeconfig_path: "{{ lookup('env', 'KUBECONFIG_PATH')|default('/root/.kube/config', true) }}" + +# kraken dir location on jump host +kraken_dir: "{{ lookup('env', 'KRAKEN_DIR')|default('/root/kraken', true) }}" + +# kraken config path location +kraken_config: "{{ lookup('env', 'KRAKEN_CONFIG')|default('/root/kraken/config/config.yaml', true) }}" + +# kraken repository location +kraken_repository: "{{ lookup('env', 'KRAKEN_REPOSITORY')|default('https://github.com/openshift-scale/kraken.git', true) }}" + +# scenarios to inject +scenarios_folder_path: "{{ lookup('env', 'SCENARIOS_FOLDER_PATH')|default('CI/scenarios/', true) }}" +scenarios: "{{ lookup('env', 'SCENARIOS')|default('[[scenarios/etcd.yml, scenarios/post_action_etcd_example.sh], [scenarios/openshift-apiserver.yml, scenarios/post_action_openshift-kube-apiserver.yml], [scenarios/openshift-kube-apiserver.yml, scenarios/post_action_openshift-apiserver.yml], [scenarios/regex_openshift_pod_kill.yml, scenarios/post_action_regex.py]]', true) }}" + +exit_on_failure: "{{ lookup('env', 'EXIT_ON_FAILURE')|default(false, true) }}" + +# Cerberus enabled by user +cerberus_enabled: "{{ lookup('env', 'CERBERUS_ENABLED')|default(false, true) }}" +cerberus_url: "{{ lookup('env', 'CERBERUS_URL')|default('', true) }}" + +# Kraken configurations +wait_duration: "{{ lookup('env', 'WAIT_DURATION')|default(60, true) }}" +iterations: "{{ lookup('env', 'ITERATIONS')|default(1, true) }}" +daemon_mode: "{{ lookup('env', 'DAEMON_MODE')|default(false, true) }}"