Adding delete of namespaces

README.md

@@ -44,6 +44,7 @@ Kraken supports pod, node, time/date and [litmus](https://github.com/litmuschaos
 
 - [Cluster Shut Down Scenarios](docs/cluster_shut_down_scenarios.md)
 
+- [Namespace Scenarios](docs/namespace_scenarios.md)
 
 ### Kraken scenario pass/fail criteria and report
 It's important to make sure to check if the targeted component recovered from the chaos injection and also if the Kubernetes/OpenShift cluster is healthy as failures in one component can have an adverse impact on other components. Kraken does this by:

config/config.yaml

@@ -22,6 +22,9 @@ kraken:
         -   cluster_shut_down_scenarios:
             - - scenarios/cluster_shut_down_scenario.yml
               - scenarios/post_action_shut_down.py
+        -   namespace_scenarios:
+            - scenarios/regex_namespace.yaml
+            - scenarios/ingress_namespace.yaml
 cerberus:
     cerberus_enabled: False                                # Enable it when cerberus is previously installed
     cerberus_url:                                          # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal

config/config_performance.yaml

@@ -22,6 +22,9 @@ kraken:
         -   cluster_shut_down_scenarios:
             - - scenarios/cluster_shut_down_scenario.yml
               - scenarios/post_action_shut_down.py
+        - namespace_scenarios:
+            - scenarios/regex_namespace.yaml
+            - scenarios/ingress_namespace.yaml
 cerberus:
     cerberus_enabled: True                                # Enable it when cerberus is previously installed
     cerberus_url: http://0.0.0.0:8080                     # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal

docs/namespace_scenarios.md

@@ -0,0 +1,28 @@
+###  Delete Namespace Scenarios
+
+Using this type of scenario configuration, one is able to delete specific namespace or namespace matching a certain regex string
+
+Configuration Options:
+
+**action:** default is `delete`
+
+**namespace:** specific namespace or regex style namespace of what you want to delete, gets all namespaces if not specified
+
+**label_selector:** label on the namespace you want to delete
+
+**runs:** number of runs to kill namespaces, based on matching namespace and label specified, default is 1
+
+**sleep:** number of seconds to wait between each iteration/count of killing namespaces. Defaults to 10 seconds if not set
+
+Refer to [namespace_scenarios_example](https://github.com/openshift-scale/kraken/blob/master/scenarios/regex_namespace) config file.
+
+```
+scenarios:
+- action: delete
+  namespace: "^.*$"
+  runs: 1
+- action: delete
+  namespace: "^.*ingress.*$"
+  runs: 1
+  sleep: 15
+```

kraken/kubernetes/client.py

@@ -4,7 +4,8 @@ from kubernetes.client.rest import ApiException
 import logging
 import kraken.invoke.command as runcommand
 import json
-
+import sys
+import re
 
 kraken_node_name = ""
 
@@ -16,6 +17,44 @@ def initialize_clients(kubeconfig_path):
     cli = client.CoreV1Api()
 
 
+# List all namespaces
+def list_namespaces(label_selector=None):
+    namespaces = []
+    try:
+        if label_selector:
+            ret = cli.list_namespace(pretty=True, label_selector=label_selector)
+        else:
+            ret = cli.list_namespace(pretty=True)
+    except ApiException as e:
+        logging.error("Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e)
+    for namespace in ret.items:
+        namespaces.append(namespace.metadata.name)
+    return namespaces
+
+
+# Check if all the watch_namespaces are valid
+def check_namespaces(namespaces, label_selectors=None):
+    try:
+        valid_namespaces = list_namespaces(label_selectors)
+        regex_namespaces = set(namespaces) - set(valid_namespaces)
+        final_namespaces = set(namespaces) - set(regex_namespaces)
+        valid_regex = set()
+        if regex_namespaces:
+            for namespace in valid_namespaces:
+                for regex_namespace in regex_namespaces:
+                    if re.search(regex_namespace, namespace):
+                        final_namespaces.add(namespace)
+                        valid_regex.add(regex_namespace)
+                        break
+        invalid_namespaces = regex_namespaces - valid_regex
+        if invalid_namespaces:
+            raise Exception("There exists no namespaces matching: %s" % (invalid_namespaces))
+        return list(final_namespaces)
+    except Exception as e:
+        logging.info("%s" % (e))
+        sys.exit(1)
+
+
 # List nodes in the cluster
 def list_nodes(label_selector=None):
     nodes = []

kraken/namespace_actions/common_namespace_functions.py

@@ -0,0 +1,44 @@
+import time
+import random
+import logging
+import kraken.invoke.command as runcommand
+import kraken.kubernetes.client as kubecli
+import kraken.cerberus.setup as cerberus
+import yaml
+import sys
+
+
+def run(scenarios_list, config, wait_duration):
+    for scenario_config in scenarios_list:
+        with open(scenario_config, "r") as f:
+            scenario_config = yaml.full_load(f)
+            for scenario in scenario_config["scenarios"]:
+                scenario_namespace = scenario.get("namespace", "^.*$")
+                scenario_label = scenario.get("label_selector", None)
+                run_count = scenario.get("runs", 1)
+                namespace_action = scenario.get("action", "delete")
+                run_sleep = scenario.get("sleep", 10)
+                namespaces = kubecli.check_namespaces([scenario_namespace], scenario_label)
+                for i in range(run_count):
+                    if len(namespaces) == 0:
+                        logging.error(
+                            "Couldn't %s %s namespaces, not enough namespaces matching %s with label %s"
+                            % (namespace_action, str(run_count), scenario_namespace, str(scenario_label))
+                        )
+                        sys.exit(1)
+                    selected_namespace = namespaces[random.randint(0, len(namespaces) - 1)]
+                    try:
+                        runcommand.invoke("oc %s project %s" % (namespace_action, selected_namespace))
+                        logging.info(namespace_action + " on namespace " + str(selected_namespace) + " was successful")
+                    except Exception as e:
+                        logging.info(
+                            namespace_action + " on namespace " + str(selected_namespace) + " was unsuccessful"
+                        )
+                        logging.info("Namespace action error: " + str(e))
+                    namespaces.remove(selected_namespace)
+                    logging.info("Waiting %s seconds between namespace deletions" % str(run_sleep))
+                    time.sleep(run_sleep)
+
+                logging.info("Waiting for the specified duration: %s" % wait_duration)
+                time.sleep(wait_duration)
+                cerberus.get_status(config)

run_kraken.py

@@ -14,6 +14,7 @@ import kraken.litmus.common_litmus as common_litmus
 import kraken.time_actions.common_time_functions as time_actions
 import kraken.performance_dashboards.setup as performance_dashboards
 import kraken.pod_scenarios.setup as pod_scenarios
+import kraken.namespace_actions.common_namespace_functions as namespace_actions
 import kraken.shut_down.common_shut_down_func as shut_down
 import kraken.node_actions.run as nodeaction
 import kraken.kube_burner.client as kube_burner
@@ -140,6 +141,9 @@ def main(cfg):
                         elif scenario_type == "cluster_shut_down_scenarios":
                             shut_down.run(scenarios_list, config, wait_duration)
 
+                        elif scenario_type == "namespace_scenarios":
+                            logging.info("Running namespace scenarios")
+                            namespace_actions.run(scenarios_list, config, wait_duration)
             iteration += 1
             logging.info("")
 

scenarios/ingress_namespace.yaml

@@ -0,0 +1,5 @@
+scenarios:
+- action: delete
+  namespace: "^.*ingress.*$"
+  runs: 1
+  sleep: 15

scenarios/regex_namespace.yaml

@@ -0,0 +1,5 @@
+scenarios:
+- action: delete
+  namespace: "^.*$"
+  runs: 2
+  sleep: 15