From bc863fa01ff65ba7432debd16047c1c5e055adb1 Mon Sep 17 00:00:00 2001 From: Naga Ravi Chaitanya Elluri Date: Tue, 25 Apr 2023 22:50:37 -0400 Subject: [PATCH] Add support to check for critical alerts This commit enables users to opt in to check for critical alerts firing in the cluster post chaos at the end of each scenario. Chaos scenario is considered as failed if the cluster is unhealthy in which case user can start debugging to fix and harden respective areas. Fixes https://github.com/redhat-chaos/krkn/issues/410 --- config/config.yaml | 2 +- config/config_kubernetes.yaml | 2 +- docs/alerts.md | 15 +++++++++++++-- kraken/prometheus/client.py | 36 +++++++++++++++++++++++++++++++++-- requirements.txt | 3 ++- run_kraken.py | 18 +++++++++++++++++- 6 files changed, 68 insertions(+), 8 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 1a1986c8..7ec63789 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -65,7 +65,7 @@ performance_monitoring: uuid: # uuid for the run is generated by default if not set enable_alerts: False # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error alert_profile: config/alerts # Path to alert profile with the prometheus queries - + check_critical_alerts: False # When enabled will check prometheus for critical alerts firing post chaos tunings: wait_duration: 60 # Duration to wait between each chaos scenario iterations: 1 # Number of times to execute the scenarios diff --git a/config/config_kubernetes.yaml b/config/config_kubernetes.yaml index df21fb46..524c48f5 100644 --- a/config/config_kubernetes.yaml +++ b/config/config_kubernetes.yaml @@ -32,7 +32,7 @@ performance_monitoring: uuid: # uuid for the run is generated by default if not set enable_alerts: False # Runs the queries specified in the alert profile and displays the info or exits 1 when severity=error alert_profile: config/alerts # Path to alert profile with the prometheus queries - + check_critical_alerts: False # When enabled will check prometheus for critical alerts firing post chaos after soak time for the cluster to settle down tunings: wait_duration: 60 # Duration to wait between each chaos scenario iterations: 1 # Number of times to execute the scenarios diff --git a/docs/alerts.md b/docs/alerts.md index c05a1b0d..6ef76ef6 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -1,6 +1,17 @@ ## Alerts -Pass/fail based on metrics captured from the cluster is important in addition to checking the health status and recovery. Kraken supports alerting based on the queries defined by the user and modifies the return code of the run to determine pass/fail. It's especially useful in case of automated runs in CI where user won't be able to monitor the system. It uses [Kube-burner](https://kube-burner.readthedocs.io/en/latest/) under the hood. This feature can be enabled in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) by setting the following: +Pass/fail based on metrics captured from the cluster is important in addition to checking the health status and recovery. Kraken supports: + +### Checking for critical alerts +If enabled, the check runs at the end of each scenario and Kraken exits in case critical alerts are firing to allow user to debug. You can enable it in the config: + +``` +performance_monitoring: + check_critical_alerts: False # When enabled will check prometheus for critical alerts firing post chaos +``` + +### Alerting based on the queries defined by the user +Takes PromQL queries as input and modifies the return code of the run to determine pass/fail. It's especially useful in case of automated runs in CI where user won't be able to monitor the system. It uses [Kube-burner](https://kube-burner.readthedocs.io/en/latest/) under the hood. This feature can be enabled in the [config](https://github.com/redhat-chaos/krkn/blob/main/config/config.yaml) by setting the following: ``` performance_monitoring: @@ -11,7 +22,7 @@ performance_monitoring: alert_profile: config/alerts # Path to alert profile with the prometheus queries. ``` -### Alert profile +#### Alert profile A couple of [alert profiles](https://github.com/redhat-chaos/krkn/tree/main/config) [alerts](https://github.com/redhat-chaos/krkn/blob/main/config/alerts) are shipped by default and can be tweaked to add more queries to alert on. The following are a few alerts examples: ``` diff --git a/kraken/prometheus/client.py b/kraken/prometheus/client.py index 13dfa93f..798e2aa0 100644 --- a/kraken/prometheus/client.py +++ b/kraken/prometheus/client.py @@ -1,5 +1,37 @@ +import urllib3 +import logging +import prometheus_api_client +import sys import kraken.invoke.command as runcommand +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# Initialize the client +def initialize_prom_client(distribution, prometheus_url, prometheus_bearer_token): + global prom_cli + prometheus_url, prometheus_bearer_token = instance(distribution, prometheus_url, prometheus_bearer_token) + if prometheus_url and prometheus_bearer_token: + bearer = "Bearer " + prometheus_bearer_token + headers = {"Authorization": bearer} + try: + prom_cli = prometheus_api_client.PrometheusConnect(url=prometheus_url, headers=headers, disable_ssl=True) + except Exception as e: + logging.error("Not able to initialize the client %s" % e) + sys.exit(1) + else: + prom_cli = None + + +# Process custom prometheus query +def process_prom_query(query): + if prom_cli: + try: + return prom_cli.custom_query(query=query, params=None) + except Exception as e: + logging.error("Failed to get the metrics: %s" % e) + sys.exit(1) + else: + logging.info("Skipping the prometheus query as the prometheus client couldn't " "be initilized\n") # Get prometheus details def instance(distribution, prometheus_url, prometheus_bearer_token): @@ -10,8 +42,8 @@ def instance(distribution, prometheus_url, prometheus_bearer_token): prometheus_url = "https://" + url if distribution == "openshift" and not prometheus_bearer_token: prometheus_bearer_token = runcommand.invoke( - "oc -n openshift-monitoring sa get-token prometheus-k8s " - "|| oc create token -n openshift-monitoring prometheus-k8s --duration=12h " + "oc create token -n openshift-monitoring prometheus-k8s --duration=12h " + "|| oc -n openshift-monitoring sa get-token prometheus-k8s " "|| oc sa new-token -n openshift-monitoring prometheus-k8s" ) return prometheus_url, prometheus_bearer_token diff --git a/requirements.txt b/requirements.txt index 2761c520..67f4cea3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,4 +32,5 @@ wheel service_identity git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.0.0 git+https://github.com/redhat-chaos/arcaflow-plugin-kill-pod.git@c406307329b07d4077e85e4e615b3bb133f20144 -arcaflow >= 0.3.0 \ No newline at end of file +arcaflow >= 0.3.0 +prometheus_api_client diff --git a/run_kraken.py b/run_kraken.py index 7ff9f104..b02dfe5d 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -24,6 +24,7 @@ import kraken.pvc.pvc_scenario as pvc_scenario import kraken.network_chaos.actions as network_chaos import kraken.arcaflow_plugin as arcaflow_plugin import server as server +import kraken.prometheus.client as promcli from kraken import plugins KUBE_BURNER_URL = ( @@ -86,6 +87,7 @@ def main(cfg): run_uuid = config["performance_monitoring"].get("uuid", "") enable_alerts = config["performance_monitoring"].get("enable_alerts", False) alert_profile = config["performance_monitoring"].get("alert_profile", "") + check_critical_alerts = config["performance_monitoring"].get("check_critical_alerts", False) # Initialize clients if not os.path.isfile(kubeconfig_path): @@ -318,6 +320,20 @@ def main(cfg): logging.info("Running Network Chaos") network_chaos.run(scenarios_list, config, wait_duration) + # Check for critical alerts when enabled + if check_critical_alerts: + logging.info("Checking for critical alerts firing post choas") + promcli.initialize_prom_client(distribution, prometheus_url, prometheus_bearer_token) + query = r"""ALERTS{severity="critical"}""" + critical_alerts = promcli.process_prom_query(query) + critical_alerts_count = len(critical_alerts) + if critical_alerts_count > 0: + logging.error("Critical alerts are firing: %s", critical_alerts) + logging.error("Please check, exiting") + sys.exit(1) + else: + logging.info("No critical alerts are firing!!") + iteration += 1 logging.info("") @@ -355,7 +371,7 @@ def main(cfg): else: logging.error("Alert profile is not defined") sys.exit(1) - + if litmus_uninstall and litmus_installed: common_litmus.delete_chaos(litmus_namespace) common_litmus.delete_chaos_experiments(litmus_namespace)