From 10aa798e5029feb9c96d547db6a93317d1c2acc0 Mon Sep 17 00:00:00 2001 From: Paige Patton <64206430+paigerube14@users.noreply.github.com> Date: Thu, 7 May 2026 15:18:51 -0400 Subject: [PATCH] more error checking when telemetry enabled (#1274) Signed-off-by: Paige Patton --- CI/config/common_test_config.yaml | 6 +++--- CI/tests/test_telemetry.sh | 1 + config/config.yaml | 8 ++++---- krkn/scenario_plugins/abstract_scenario_plugin.py | 2 +- requirements.txt | 2 +- run_kraken.py | 9 +++++---- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/CI/config/common_test_config.yaml b/CI/config/common_test_config.yaml index c63d9b68..ed468074 100644 --- a/CI/config/common_test_config.yaml +++ b/CI/config/common_test_config.yaml @@ -32,8 +32,8 @@ tunings: iterations: 1 # Number of times to execute the scenarios. daemon_mode: False # Iterations are set to infinity which means that the kraken will cause chaos forever. telemetry: - enabled: False # enable/disables the telemetry collection feature - api_url: https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test #telemetry service endpoint + enabled: True # enable/disables the telemetry collection feature + api_url: #telemetry service endpoint username: $TELEMETRY_USERNAME # telemetry service username password: $TELEMETRY_PASSWORD # telemetry service password prometheus_namespace: 'monitoring' # prometheus namespace @@ -46,7 +46,7 @@ telemetry: max_retries: 0 # maximum number of upload retries (if 0 will retry forever) run_tag: '' # if set, this will be appended to the run folder in the bucket (useful to group the runs) archive_size: 10000 # the size of the prometheus data archive size in KB. The lower the size of archive is - logs_backup: True + logs_backup: False logs_filter_patterns: - "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+" # Sep 9 11:20:36.123425532 - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+" # kinit 2023/09/15 11:20:36 log diff --git a/CI/tests/test_telemetry.sh b/CI/tests/test_telemetry.sh index 0c252e4d..c31878b8 100644 --- a/CI/tests/test_telemetry.sh +++ b/CI/tests/test_telemetry.sh @@ -12,6 +12,7 @@ function functional_test_telemetry { [ -z "$AWS_BUCKET" ] && echo "AWS bucket not set in environment" && exit 1 export RUN_TAG="funtest-telemetry" + yq -i '.telemetry.api_url="https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test"' CI/config/common_test_config.yaml yq -i '.telemetry.enabled=True' CI/config/common_test_config.yaml yq -i '.telemetry.full_prometheus_backup=True' CI/config/common_test_config.yaml yq -i '.performance_monitoring.check_critical_alerts=True' CI/config/common_test_config.yaml diff --git a/config/config.yaml b/config/config.yaml index 8dbb6679..b5c340ef 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -93,11 +93,11 @@ tunings: iterations: 1 # Number of times to execute the scenarios daemon_mode: False # Iterations are set to infinity which means that the kraken will cause chaos forever telemetry: - enabled: False # enable/disables the telemetry collection feature - api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint + enabled: True # enable/disables the telemetry collection feature + api_url: #telemetry service endpoint username: username # telemetry service username password: password # telemetry service password - prometheus_backup: True # enables/disables prometheus data collection + prometheus_backup: False # enables/disables prometheus data collection prometheus_namespace: "" # namespace where prometheus is deployed (if distribution is kubernetes) prometheus_container_name: "" # name of the prometheus container name (if distribution is kubernetes) prometheus_pod_name: "" # name of the prometheus pod (if distribution is kubernetes) @@ -114,7 +114,7 @@ telemetry: # For unstable/slow connection is better to keep this value low # increasing the number of backup_threads, in this way, on upload failure, the retry will happen only on the # failed chunk without affecting the whole upload. - logs_backup: True + logs_backup: False logs_filter_patterns: - "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+" # Sep 9 11:20:36.123425532 - "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+" # kinit 2023/09/15 11:20:36 log diff --git a/krkn/scenario_plugins/abstract_scenario_plugin.py b/krkn/scenario_plugins/abstract_scenario_plugin.py index de99cab1..e259d581 100644 --- a/krkn/scenario_plugins/abstract_scenario_plugin.py +++ b/krkn/scenario_plugins/abstract_scenario_plugin.py @@ -158,7 +158,7 @@ class AbstractScenarioPlugin(ABC): end_time ) - if events_backup: + if events_backup: utils.populate_cluster_events( krkn_config, parsed_scenario_config, diff --git a/requirements.txt b/requirements.txt index e149769a..4611bda8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ ibm_vpc==0.26.3 # Requires ibm_cloud_sdk_core jinja2==3.1.6 lxml==6.1.0 kubernetes>=35.0.0 -krkn-lib==6.0.7 +krkn-lib==6.0.9 numpy==1.26.4 pandas==2.2.0 openshift-client==1.0.21 diff --git a/run_kraken.py b/run_kraken.py index f2d782c6..71d1c47b 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -173,7 +173,8 @@ def main(options, command: Optional[str]) -> int: check_critical_alerts = get_yaml_item_value( config["performance_monitoring"], "check_critical_alerts", False ) - telemetry_api_url = config["telemetry"].get("api_url") + telemetry_api_url = config["telemetry"].get("api_url", "") + telemetry_enabled = config["telemetry"].get("enabled", True) health_check_config = get_yaml_item_value(config, "health_checks",{}) kubevirt_check_config = get_yaml_item_value(config, "kubevirt_checks", {}) @@ -499,7 +500,7 @@ def main(options, command: Optional[str]) -> int: chaos_telemetry.post_virt_checks = post_kubevirt_check # Collect cluster metadata only when telemetry is enabled # (listing all k8s objects is very slow on large clusters) - if config["telemetry"].get("enabled", True): + if telemetry_enabled: if distribution == "openshift": logging.info( "collecting OCP cluster metadata, this may take few minutes...." @@ -562,7 +563,7 @@ def main(options, command: Optional[str]) -> int: f"failed to save telemetry on elastic search: {chaos_output.to_json()}" ) - if config["telemetry"]["enabled"]: + if telemetry_enabled and telemetry_api_url: logging.info( f"telemetry data will be stored on s3 bucket folder: {telemetry_api_url}/files/" f'{(config["telemetry"]["telemetry_group"] if config["telemetry"]["telemetry_group"] else "default")}/' @@ -625,7 +626,7 @@ def main(options, command: Optional[str]) -> int: except Exception as e: logging.error(f"failed to send telemetry data: {str(e)}") else: - logging.info("telemetry collection disabled, skipping.") + logging.info("api_url not set, skipping telemetry upload.") # Check for the alerts specified if enable_alerts: