From ab951adb7877ce8b7310e925fd3794ff51b3757b Mon Sep 17 00:00:00 2001 From: Naga Ravi Chaitanya Elluri Date: Mon, 26 Feb 2024 09:43:34 -0500 Subject: [PATCH] Expose thresholds config options (#574) This commit allows users to edit the thresholds in the chaos-recommender config to be able to identify outliers based on their use case. Fixes https://github.com/krkn-chaos/krkn/issues/509 Signed-off-by: Naga Ravi Chaitanya Elluri --- kraken/chaos_recommender/analysis.py | 10 +++------- utils/chaos_recommender/README.md | 14 ++++++++++++-- utils/chaos_recommender/chaos_recommender.py | 20 +++++++++++++++----- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/kraken/chaos_recommender/analysis.py b/kraken/chaos_recommender/analysis.py index 40db4f5c..5a6267da 100644 --- a/kraken/chaos_recommender/analysis.py +++ b/kraken/chaos_recommender/analysis.py @@ -4,10 +4,6 @@ import pandas as pd import kraken.chaos_recommender.kraken_tests as kraken_tests import time -threshold = .7 # Adjust the threshold as needed -heatmap_cpu_threshold = .5 -heatmap_mem_threshold = .5 - KRAKEN_TESTS_PATH = "./kraken_chaos_tests.txt" #Placeholder, this should be done with topology @@ -27,7 +23,7 @@ def calculate_zscores(data): zscores["Network"] = (data["NETWORK"] - data["NETWORK"].mean()) / data["NETWORK"].std() return zscores -def identify_outliers(data): +def identify_outliers(data, threshold): outliers_cpu = data[data["CPU"] > threshold]["Service"].tolist() outliers_memory = data[data["Memory"] > threshold]["Service"].tolist() outliers_network = data[data["Network"] > threshold]["Service"].tolist() @@ -47,7 +43,7 @@ def get_services_above_heatmap_threshold(dataframe, cpu_threshold, mem_threshold return cpu_services, mem_services -def analysis(file_path, chaos_tests_config): +def analysis(file_path, chaos_tests_config, threshold, heatmap_cpu_threshold, heatmap_mem_threshold): # Load the telemetry data from file data = load_telemetry_data(file_path) @@ -55,7 +51,7 @@ def analysis(file_path, chaos_tests_config): zscores = calculate_zscores(data) # Identify outliers - outliers_cpu, outliers_memory, outliers_network = identify_outliers(zscores) + outliers_cpu, outliers_memory, outliers_network = identify_outliers(zscores, threshold) cpu_services, mem_services = get_services_above_heatmap_threshold(data, heatmap_cpu_threshold, heatmap_mem_threshold) # Display the identified outliers diff --git a/utils/chaos_recommender/README.md b/utils/chaos_recommender/README.md index e9aefc2a..58b38cba 100644 --- a/utils/chaos_recommender/README.md +++ b/utils/chaos_recommender/README.md @@ -20,6 +20,8 @@ This tool profiles an application and gathers telemetry data such as CPU, Memory $ git clone https://github.com/krkn-chaos/krkn.git $ cd krkn $ pip3 install -r requirements.txt + Edit configuration file: + $ vi config/recommender_config.yaml $ python3.9 utils/chaos_recommender/chaos_recommender.py ``` @@ -42,6 +44,9 @@ You can customize the default values by editing the `krkn/config/recommender_con - `MEM`: list of memory related tests available in Krkn - `NETWORK`: list of network related tests available in Krkn - `CPU`: list of memory related tests available in Krkn + - `threshold`: Specify the threshold to use for comparison and identifying outliers + - `cpu_threshold`: Specify the cpu threshold to compare with the cpu limits set on the pods and identify outliers + - `mem_threshold`: Specify the memory threshold to compare with the memory limits set on the pods and identify outliers *TIP:* to collect prometheus endpoint and token from your OpenShift cluster you can run the following commands: ``` @@ -82,7 +87,12 @@ You can also provide the input values through command-line arguments launching t Network related chaos tests (space separated list) -G GENERIC [GENERIC ...], --GENERIC GENERIC [GENERIC ...] Memory related chaos tests (space separated list) - + --threshold THRESHOLD + Threshold + --cpu_threshold CPU_THRESHOLD + CPU threshold to compare with the cpu limits + --mem_threshold MEM_THRESHOLD + Memory threshold to compare with the memory limits ``` If you provide the input values through command-line arguments, the corresponding config file inputs would be ignored. @@ -97,7 +107,7 @@ After obtaining telemetry data, sourced either locally or from Prometheus, the t ## Customizing Thresholds and Options -You can customize the thresholds and options used for data analysis by modifying the `krkn/kraken/chaos_recommender/analysis.py` file. For example, you can adjust the threshold for identifying outliers by changing the value of the `threshold` variable in the `identify_outliers` function. +You can customize the thresholds and options used for data analysis and identifying the outliers by setting the threshold, cpu_threshold and mem_threshold parameters in the config. ## Additional Files diff --git a/utils/chaos_recommender/chaos_recommender.py b/utils/chaos_recommender/chaos_recommender.py index 23629cc8..d7565e51 100644 --- a/utils/chaos_recommender/chaos_recommender.py +++ b/utils/chaos_recommender/chaos_recommender.py @@ -26,7 +26,6 @@ def parse_arguments(parser): parser.add_argument("-t", "--token", action="store", default="", help="Kubernetes authentication token") parser.add_argument("-s", "--scrape-duration", action="store", default="10m", help="Prometheus scrape duration") parser.add_argument("-L", "--log-level", action="store", default="INFO", help="log level (DEBUG, INFO, WARNING, ERROR, CRITICAL") - parser.add_argument("-M", "--MEM", nargs='+', action="store", default=[], help="Memory related chaos tests (space separated list)") parser.add_argument("-C", "--CPU", nargs='+', action="store", default=[], @@ -35,7 +34,9 @@ def parse_arguments(parser): help="Network related chaos tests (space separated list)") parser.add_argument("-G", "--GENERIC", nargs='+', action="store", default=[], help="Memory related chaos tests (space separated list)") - + parser.add_argument("--threshold", action="store", default="", help="Threshold") + parser.add_argument("--cpu_threshold", action="store", default="", help="CPU threshold") + parser.add_argument("--mem_threshold", action="store", default="", help="Memory threshold") return parser.parse_args() @@ -55,8 +56,11 @@ def read_configuration(config_file_path): auth_token = config.get("auth_token", "") scrape_duration = config.get("scrape_duration", "10m") chaos_tests = config.get("chaos_tests" , {}) + threshold = config.get("threshold", ".7") + heatmap_cpu_threshold = config.get("cpu_threshold", ".5") + heatmap_mem_threshold = config.get("mem_threshold", ".5") return (namespace, kubeconfig, prometheus_endpoint, auth_token, scrape_duration, - chaos_tests, log_level) + chaos_tests, log_level, threshold, heatmap_cpu_threshold, heatmap_mem_threshold) def prompt_input(prompt, default_value): user_input = input(f"{prompt} [{default_value}]: ") @@ -81,7 +85,10 @@ def main(): auth_token, scrape_duration, chaos_tests, - log_level + log_level, + threshold, + heatmap_cpu_threshold, + heatmap_mem_threshold ) = read_configuration(args.config_file) if args.options: @@ -92,6 +99,9 @@ def main(): log_level = args.log_level prometheus_endpoint = args.prometheus_endpoint chaos_tests = {"MEM": args.MEM, "GENERIC": args.GENERIC, "CPU": args.CPU, "NETWORK": args.NETWORK} + threshold = args.threshold + heatmap_mem_threshold = args.heatmap_mem_threshold + heatmap_cpu_threshold = args.heatmap_cpu_threshold if log_level not in ["DEBUG","INFO", "WARNING", "ERROR","CRITICAL"]: logging.error(f"{log_level} not a valid log level") @@ -111,7 +121,7 @@ def main(): logging.info("Fetching the Telemetry data") file_path = prometheus.fetch_utilization_from_prometheus(prometheus_endpoint, auth_token, namespace, scrape_duration) - analysis(file_path, chaos_tests) + analysis(file_path, chaos_tests, threshold, heatmap_cpu_threshold, heatmap_mem_threshold) if __name__ == "__main__": main()