Adding server set up for kraken

README.md

@@ -65,6 +65,13 @@ It's important to make sure to check if the targeted component recovered from th
 - Leveraging [Cerberus](https://github.com/openshift-scale/cerberus) to monitor the cluster under test and consuming the aggregated go/no-go signal to determine pass/fail post chaos. It is highly recommended to turn on the Cerberus health check feature avaliable in Kraken. Instructions on installing and setting up Cerberus can be found [here](https://github.com/openshift-scale/cerberus#installation) or can be installed from Kraken using the [instructions](https://github.com/cloud-bulldozer/kraken#setting-up-infrastructure-dependencies). Once Cerberus is up and running, set cerberus_enabled to True and cerberus_url to the url where Cerberus publishes go/no-go signal in the Kraken config file. Cerberus can monitor [application routes](https://github.com/cloud-bulldozer/cerberus/blob/master/docs/config.md#watch-routes) during the chaos and fails the run if it encounters downtime as it's a potential downtime in customer, users environment as well. It is especially important during the control plane chaos scenarios including the API server, Etcd, Ingress etc. It can be enabled by setting `check_applicaton_routes: True` in the [Kraken config](https://github.com/cloud-bulldozer/kraken/blob/master/config/config.yaml) provided application routes are being monitored in the [cerberus config](https://github.com/cloud-bulldozer/kraken/blob/master/config/cerberus.yaml)
 - Leveraging [kube-burner](docs/alerts.md) alerting feature to fail the runs in case of critical alerts.
 
+### Signaling
+In CI runs or any external job it is useful to stop Kraken once a certain test or state gets reached. We created a way to signal to kraken to pause the chaos or stop it completely using a signal posted to a port of your choice
+
+For example if we have a test run loading the cluster running and kraken separately running; we want to be able to know when to start/stop the kraken run based on when the test run completes or gets to a certain loaded state
+
+More detailed information on enabling and leveraging this feature can be found [here](docs/signal.md).
+
 
 ### Performance monitoring
 Monitoring the Kubernetes/OpenShift cluster to observe the impact of Kraken chaos scenarios on various components is key to find out the bottlenecks as it's important to make sure the cluster is healthy in terms if both recovery as well as performance during/after the failure has been injected. Instructions on enabling it can be found [here](docs/performance_dashboards.md).

config/config.yaml

@@ -2,6 +2,8 @@ kraken:
     distribution: openshift                                # Distribution can be kubernetes or openshift
     kubeconfig_path: /root/.kube/config                    # Path to kubeconfig
     exit_on_failure: False                                 # Exit when a post action scenario fails
+    port: 8081
+    publish_kraken_status: True
     litmus_version: v1.13.6                                # Litmus version to install
     litmus_uninstall: False                                # If you want to uninstall litmus if failure
     chaos_scenarios:                                       # List of policies/chaos scenarios to load

docs/signal.md

@@ -0,0 +1,53 @@
+### Signaling to Kraken
+This functionality allows a user to be able to pause or stop the kraken run at any time no matter the number of iterations or dameon_mode set in the config
+
+If publish_kraken_status is set to True in the config, kraken will start up a connection to a url at a certain port to decide if it should continue running
+
+By default it will get posted to http://0.0.0.0:8081/
+
+
+#### States
+There are 3 states in the kraken status
+
+```PAUSE```: When the Kraken signal is 'PAUSE', this will pause the kraken test and wait for the wait_duration until the signal returns to RUN
+
+```STOP```: When the Kraken signal is 'STOP', end the kraken run and print out report
+
+```RUN```: When the Kraken signal is 'RUN', continue kraken run based on iterations
+
+
+
+#### Configuration
+
+In the config you need to set these 2 parameters to tell kraken which port to post the kraken run status to
+As well if you want to publish and stop running based on the kraken status or not
+```
+    port: 8081
+    publish_kraken_status: True
+```
+
+
+#### Setting Signal
+
+See [set_stop_signal.py](https://github.com/cloud-bulldozer/kraken/tree/master/set_stop_signal.py) for an example of how to reset the kraken status during kraken execution
+
+Make sure to set the correct port number in your set_stop_signal script
+
+
+
+##### Url Examples
+To stop run:
+
+```
+curl -X POST http:/0.0.0.0:8081/STOP
+```
+
+To pause run:
+```
+curl -X POST http:/0.0.0.0:8081/PAUSE
+```
+
+To start running again:
+```
+curl -X POST http:/0.0.0.0:8081/RUN
+```

run_kraken.py

@@ -20,6 +20,12 @@ import kraken.node_actions.run as nodeaction
 import kraken.kube_burner.client as kube_burner
 import kraken.zone_outage.actions as zone_outages
 import kraken.application_outage.actions as application_outage
+import server as server
+
+
+def publish_kraken_status(status):
+    with open("/tmp/kraken_status", "w+") as file:
+        file.write(str(status))
 
 
 # Main function
@@ -36,6 +42,8 @@ def main(cfg):
         distribution = config["kraken"].get("distribution", "openshift")
         kubeconfig_path = config["kraken"].get("kubeconfig_path", "")
         chaos_scenarios = config["kraken"].get("chaos_scenarios", [])
+        publish_running_status = config["kraken"].get("publish_kraken_status", False)
+        port = config["kraken"].get("port", "8081")
         litmus_version = config["kraken"].get("litmus_version", "v1.9.1")
         litmus_uninstall = config["kraken"].get("litmus_uninstall", False)
         wait_duration = config["tunings"].get("wait_duration", 60)
@@ -69,6 +77,22 @@ def main(cfg):
         # find node kraken might be running on
         kubecli.find_kraken_node()
 
+        # Set up kraken url to track signal
+        if not 0 <= int(port) <= 65535:
+            logging.info("Using port 8081 as %s isn't a valid port number" % (port))
+            port = 8081
+        address = ("0.0.0.0", port)
+
+        # Setting the first signal to RUN
+        # If publish_running_status is False this should keep us going in our loop below
+        run_signal = "RUN"
+        if publish_running_status:
+            server_address = address[0]
+            port = address[1]
+            logging.info("Publishing kraken go status at http://%s:%s" % (server_address, port))
+            server.start_server(address)
+            publish_kraken_status(run_signal)
+
         # Cluster info
         logging.info("Fetching cluster info")
         cluster_version = runcommand.invoke("kubectl get clusterversion", 60)
@@ -108,11 +132,24 @@ def main(cfg):
         start_time = int(time.time())
 
         # Loop to run the chaos starts here
-        while int(iteration) < iterations:
+        while int(iteration) < iterations and run_signal != "STOP":
             # Inject chaos scenarios specified in the config
             logging.info("Executing scenarios for iteration " + str(iteration))
             if chaos_scenarios:
                 for scenario in chaos_scenarios:
+                    if publish_running_status:
+                        run_signal = server.get_status(address)
+                    if run_signal == "PAUSE":
+                        while publish_running_status and run_signal == "PAUSE":
+                            logging.info(
+                                "Pausing Kraken run, waiting for %s seconds and will re-poll signal"
+                                % str(wait_duration)
+                            )
+                            time.sleep(wait_duration)
+                            run_signal = server.get_status(address)
+                    if run_signal == "STOP":
+                        logging.info("Received STOP signal; ending Kraken run")
+                        break
                     scenario_type = list(scenario.keys())[0]
                     scenarios_list = scenario[scenario_type]
                     if scenarios_list:

server.py

@@ -0,0 +1,76 @@
+import sys
+import logging
+import _thread
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from http.client import HTTPConnection
+
+
+# Start a simple http server to publish the cerberus status file content
+class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
+    requests_served = 0
+
+    def do_GET(self):
+        if self.path == "/":
+            self.do_status()
+
+    def do_status(self):
+        self.send_response(200)
+        self.end_headers()
+        f = open("/tmp/kraken_status", "rb")
+        self.wfile.write(f.read())
+        SimpleHTTPRequestHandler.requests_served = SimpleHTTPRequestHandler.requests_served + 1
+
+    def do_POST(self):
+        if self.path == "/STOP":
+            self.set_stop()
+        elif self.path == "/RUN":
+            self.set_run()
+        elif self.path == "/PAUSE":
+            self.set_pause()
+
+    def set_run(self):
+        self.send_response(200)
+        self.end_headers()
+        with open("/tmp/kraken_status", "w+") as file:
+            file.write(str("STOP"))
+
+    def set_stop(self):
+        self.send_response(200)
+        self.end_headers()
+        with open("/tmp/kraken_status", "w+") as file:
+            file.write(str("STOP"))
+
+    def set_pause(self):
+        self.send_response(200)
+        self.end_headers()
+        with open("/tmp/kraken_status", "w+") as file:
+            file.write(str("PAUSE"))
+
+
+def start_server(address):
+    server = address[0]
+    port = address[1]
+    global httpd
+    httpd = HTTPServer(address, SimpleHTTPRequestHandler)
+    logging.info("Starting http server at http://%s:%s\n" % (server, port))
+    try:
+        _thread.start_new_thread(httpd.serve_forever, ())
+    except Exception:
+        logging.error(
+            "Failed to start the http server \
+                      at http://%s:%s"
+            % (server, port)
+        )
+        sys.exit(1)
+
+
+def get_status(address):
+    server = address[0]
+    port = address[1]
+    httpc = HTTPConnection(server, port)
+    logging.info("connection set up")
+    httpc.request("GET", "/")
+    response = httpc.getresponse()
+    status = response.read()
+    logging.info("response " + str(status.decode()))
+    return status.decode()

set_stop_signal.py

@@ -0,0 +1,12 @@
+import http
+
+conn = http.client.HTTPConnection("0.0.0.0", "<port>")
+
+conn.request("POST", "/STOP", {})
+
+# conn.request('POST', '/PAUSE', {})
+
+# conn.request('POST', '/RUN', {})
+
+response = conn.getresponse()
+print(response.read().decode())