Expose thresholds config options (#574)

This commit allows users to edit the thresholds in the chaos-recommender
config to be able to identify outliers based on their use case.

Fixes https://github.com/krkn-chaos/krkn/issues/509
Signed-off-by: Naga Ravi Chaitanya Elluri <nelluri@redhat.com>

kraken/chaos_recommender/analysis.py

@@ -4,10 +4,6 @@ import pandas as pd
 import kraken.chaos_recommender.kraken_tests as kraken_tests
 import time
 
-threshold = .7  # Adjust the threshold as needed
-heatmap_cpu_threshold = .5
-heatmap_mem_threshold = .5
-
 KRAKEN_TESTS_PATH = "./kraken_chaos_tests.txt"
 
 #Placeholder, this should be done with topology
@@ -27,7 +23,7 @@ def calculate_zscores(data):
     zscores["Network"] = (data["NETWORK"] - data["NETWORK"].mean()) / data["NETWORK"].std()
     return zscores
 
-def identify_outliers(data):
+def identify_outliers(data, threshold):
     outliers_cpu = data[data["CPU"] > threshold]["Service"].tolist()
     outliers_memory = data[data["Memory"] > threshold]["Service"].tolist()
     outliers_network = data[data["Network"] > threshold]["Service"].tolist()
@@ -47,7 +43,7 @@ def get_services_above_heatmap_threshold(dataframe, cpu_threshold, mem_threshold
     return cpu_services, mem_services
 
 
-def analysis(file_path, chaos_tests_config):
+def analysis(file_path, chaos_tests_config, threshold, heatmap_cpu_threshold, heatmap_mem_threshold):
     # Load the telemetry data from file
     data = load_telemetry_data(file_path)
 
@@ -55,7 +51,7 @@ def analysis(file_path, chaos_tests_config):
     zscores = calculate_zscores(data)
 
     # Identify outliers
-    outliers_cpu, outliers_memory, outliers_network = identify_outliers(zscores)
+    outliers_cpu, outliers_memory, outliers_network = identify_outliers(zscores, threshold)
     cpu_services, mem_services = get_services_above_heatmap_threshold(data, heatmap_cpu_threshold, heatmap_mem_threshold)
 
     # Display the identified outliers

utils/chaos_recommender/README.md

@@ -20,6 +20,8 @@ This tool profiles an application and gathers telemetry data such as CPU, Memory
     $ git clone https://github.com/krkn-chaos/krkn.git 
     $ cd krkn
     $ pip3 install -r requirements.txt
+    Edit configuration file:
+    $ vi config/recommender_config.yaml 
     $ python3.9 utils/chaos_recommender/chaos_recommender.py
     ```
 
@@ -42,6 +44,9 @@ You can customize the default values by editing the `krkn/config/recommender_con
     - `MEM`: list of memory related tests available in Krkn
     - `NETWORK`: list of network related tests available in Krkn
     - `CPU`: list of memory related tests available in Krkn
+  - `threshold`: Specify the threshold to use for comparison and identifying outliers
+  - `cpu_threshold`: Specify the cpu threshold to compare with the cpu limits set on the pods and identify outliers
+  - `mem_threshold`: Specify the memory threshold to compare with the memory limits set on the pods and identify outliers
 
 *TIP:* to collect prometheus endpoint and token from your OpenShift cluster you can run the following commands:
         ```
@@ -82,7 +87,12 @@ You can also provide the input values through command-line arguments launching t
                         Network related chaos tests (space separated list)
   -G GENERIC [GENERIC ...], --GENERIC GENERIC [GENERIC ...]
                         Memory related chaos tests (space separated list)
-
+  --threshold THRESHOLD
+                        Threshold
+  --cpu_threshold CPU_THRESHOLD
+                        CPU threshold to compare with the cpu limits
+  --mem_threshold MEM_THRESHOLD
+                        Memory threshold to compare with the memory limits
 ```
 
 If you provide the input values through command-line arguments, the corresponding config file inputs would be ignored.
@@ -97,7 +107,7 @@ After obtaining telemetry data, sourced either locally or from Prometheus, the t
 
 ## Customizing Thresholds and Options
 
-You can customize the thresholds and options used for data analysis by modifying the `krkn/kraken/chaos_recommender/analysis.py` file. For example, you can adjust the threshold for identifying outliers by changing the value of the `threshold` variable in the `identify_outliers` function.
+You can customize the thresholds and options used for data analysis and identifying the outliers by setting the threshold, cpu_threshold and mem_threshold parameters in the config.
 
 ## Additional Files
 

utils/chaos_recommender/chaos_recommender.py

@@ -26,7 +26,6 @@ def parse_arguments(parser):
     parser.add_argument("-t", "--token", action="store", default="", help="Kubernetes authentication token")
     parser.add_argument("-s", "--scrape-duration", action="store", default="10m", help="Prometheus scrape duration")
     parser.add_argument("-L", "--log-level", action="store", default="INFO", help="log level (DEBUG, INFO, WARNING, ERROR, CRITICAL")
-
     parser.add_argument("-M", "--MEM", nargs='+', action="store", default=[],
                         help="Memory related chaos tests (space separated list)")
     parser.add_argument("-C", "--CPU", nargs='+', action="store", default=[],
@@ -35,7 +34,9 @@ def parse_arguments(parser):
                         help="Network related chaos tests (space separated list)")
     parser.add_argument("-G", "--GENERIC", nargs='+', action="store", default=[],
                         help="Memory related chaos tests (space separated list)")
-
+    parser.add_argument("--threshold", action="store", default="", help="Threshold")
+    parser.add_argument("--cpu_threshold", action="store", default="", help="CPU threshold")
+    parser.add_argument("--mem_threshold", action="store", default="", help="Memory threshold")
 
     return parser.parse_args()
 
@@ -55,8 +56,11 @@ def read_configuration(config_file_path):
     auth_token = config.get("auth_token", "")
     scrape_duration = config.get("scrape_duration", "10m")
     chaos_tests = config.get("chaos_tests" , {})
+    threshold = config.get("threshold", ".7")
+    heatmap_cpu_threshold = config.get("cpu_threshold", ".5")
+    heatmap_mem_threshold = config.get("mem_threshold", ".5")
     return (namespace, kubeconfig, prometheus_endpoint, auth_token, scrape_duration,
-            chaos_tests, log_level)
+            chaos_tests, log_level, threshold, heatmap_cpu_threshold, heatmap_mem_threshold)
 
 def prompt_input(prompt, default_value):
     user_input = input(f"{prompt} [{default_value}]: ")
@@ -81,7 +85,10 @@ def main():
          auth_token,
          scrape_duration,
          chaos_tests,
-         log_level
+         log_level,
+         threshold,
+         heatmap_cpu_threshold,
+         heatmap_mem_threshold
          ) = read_configuration(args.config_file)
 
     if args.options:
@@ -92,6 +99,9 @@ def main():
         log_level = args.log_level
         prometheus_endpoint = args.prometheus_endpoint
         chaos_tests = {"MEM": args.MEM, "GENERIC": args.GENERIC, "CPU": args.CPU, "NETWORK": args.NETWORK}
+        threshold = args.threshold
+        heatmap_mem_threshold = args.heatmap_mem_threshold
+        heatmap_cpu_threshold = args.heatmap_cpu_threshold
 
     if log_level not in ["DEBUG","INFO", "WARNING", "ERROR","CRITICAL"]:
         logging.error(f"{log_level} not a valid log level")
@@ -111,7 +121,7 @@ def main():
     logging.info("Fetching the Telemetry data")
 
     file_path = prometheus.fetch_utilization_from_prometheus(prometheus_endpoint, auth_token, namespace, scrape_duration)
-    analysis(file_path, chaos_tests)
+    analysis(file_path, chaos_tests, threshold, heatmap_cpu_threshold, heatmap_mem_threshold)
 
 if __name__ == "__main__":
     main()