github/krkn
1
0
Fork 0
mirror of https://github.com/krkn-chaos/krkn.git synced 2026-08-25 09:27:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

more error checking when telemetry enabled (#1274)

Browse Source 
Signed-off-by: Paige Patton <prubenda@redhat.com>
This commit is contained in:
Paige Patton
2026-05-07 15:18:51 -04:00
committed by GitHub
parent e857f33cdb
commit 10aa798e50
6 changed files with 15 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CI/config/common_test_config.yaml
+3 -3
View File
@@ -32,8 +32,8 @@ tunings:
iterations: 1 # Number of times to execute the scenarios.
daemon_mode: False # Iterations are set to infinity which means that the kraken will cause chaos forever.
telemetry:
enabled: False # enable/disables the telemetry collection feature
api_url: https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test #telemetry service endpoint
enabled: True # enable/disables the telemetry collection feature
api_url: #telemetry service endpoint
username: $TELEMETRY_USERNAME # telemetry service username
password: $TELEMETRY_PASSWORD # telemetry service password
prometheus_namespace: 'monitoring' # prometheus namespace
@@ -46,7 +46,7 @@ telemetry:
max_retries: 0 # maximum number of upload retries (if 0 will retry forever)
run_tag: '' # if set, this will be appended to the run folder in the bucket (useful to group the runs)
archive_size: 10000 # the size of the prometheus data archive size in KB. The lower the size of archive is
logs_backup: True
logs_backup: False
logs_filter_patterns:
- "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+" # Sep 9 11:20:36.123425532
- "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+" # kinit 2023/09/15 11:20:36 log
CI/tests/test_telemetry.sh
+1
View File
@@ -12,6 +12,7 @@ function functional_test_telemetry {
[ -z "$AWS_BUCKET" ] && echo "AWS bucket not set in environment" && exit 1
export RUN_TAG="funtest-telemetry"
yq -i '.telemetry.api_url="https://yvnn4rfoi7.execute-api.us-west-2.amazonaws.com/test"' CI/config/common_test_config.yaml
yq -i '.telemetry.enabled=True' CI/config/common_test_config.yaml
yq -i '.telemetry.full_prometheus_backup=True' CI/config/common_test_config.yaml
yq -i '.performance_monitoring.check_critical_alerts=True' CI/config/common_test_config.yaml
config/config.yaml
+4 -4
View File
@@ -93,11 +93,11 @@ tunings:
iterations: 1 # Number of times to execute the scenarios
daemon_mode: False # Iterations are set to infinity which means that the kraken will cause chaos forever
telemetry:
enabled: False # enable/disables the telemetry collection feature
api_url: https://ulnmf9xv7j.execute-api.us-west-2.amazonaws.com/production #telemetry service endpoint
enabled: True # enable/disables the telemetry collection feature
api_url: #telemetry service endpoint
username: username # telemetry service username
password: password # telemetry service password
prometheus_backup: True # enables/disables prometheus data collection
prometheus_backup: False # enables/disables prometheus data collection
prometheus_namespace: "" # namespace where prometheus is deployed (if distribution is kubernetes)
prometheus_container_name: "" # name of the prometheus container name (if distribution is kubernetes)
prometheus_pod_name: "" # name of the prometheus pod (if distribution is kubernetes)
@@ -114,7 +114,7 @@ telemetry:
# For unstable/slow connection is better to keep this value low
# increasing the number of backup_threads, in this way, on upload failure, the retry will happen only on the
# failed chunk without affecting the whole upload.
logs_backup: True
logs_backup: False
logs_filter_patterns:
- "(\\w{3}\\s\\d{1,2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+).+" # Sep 9 11:20:36.123425532
- "kinit (\\d+/\\d+/\\d+\\s\\d{2}:\\d{2}:\\d{2})\\s+" # kinit 2023/09/15 11:20:36 log
krkn/scenario_plugins/abstract_scenario_plugin.py
+1 -1
View File
@@ -158,7 +158,7 @@ class AbstractScenarioPlugin(ABC):
end_time
)
if events_backup:
if events_backup:
utils.populate_cluster_events(
krkn_config,
parsed_scenario_config,
requirements.txt
+1 -1
View File
@@ -17,7 +17,7 @@ ibm_vpc==0.26.3 # Requires ibm_cloud_sdk_core
jinja2==3.1.6
lxml==6.1.0
kubernetes>=35.0.0
krkn-lib==6.0.7
krkn-lib==6.0.9
numpy==1.26.4
pandas==2.2.0
openshift-client==1.0.21
run_kraken.py
+5 -4
View File
@@ -173,7 +173,8 @@ def main(options, command: Optional[str]) -> int:
check_critical_alerts = get_yaml_item_value(
config["performance_monitoring"], "check_critical_alerts", False
)
telemetry_api_url = config["telemetry"].get("api_url")
telemetry_api_url = config["telemetry"].get("api_url", "")
telemetry_enabled = config["telemetry"].get("enabled", True)
health_check_config = get_yaml_item_value(config, "health_checks",{})
kubevirt_check_config = get_yaml_item_value(config, "kubevirt_checks", {})
@@ -499,7 +500,7 @@ def main(options, command: Optional[str]) -> int:
chaos_telemetry.post_virt_checks = post_kubevirt_check
# Collect cluster metadata only when telemetry is enabled
# (listing all k8s objects is very slow on large clusters)
if config["telemetry"].get("enabled", True):
if telemetry_enabled:
if distribution == "openshift":
logging.info(
"collecting OCP cluster metadata, this may take few minutes...."
@@ -562,7 +563,7 @@ def main(options, command: Optional[str]) -> int:
f"failed to save telemetry on elastic search: {chaos_output.to_json()}"
)
if config["telemetry"]["enabled"]:
if telemetry_enabled and telemetry_api_url:
logging.info(
f"telemetry data will be stored on s3 bucket folder: {telemetry_api_url}/files/"
f'{(config["telemetry"]["telemetry_group"] if config["telemetry"]["telemetry_group"] else "default")}/'
@@ -625,7 +626,7 @@ def main(options, command: Optional[str]) -> int:
except Exception as e:
logging.error(f"failed to send telemetry data: {str(e)}")
else:
logging.info("telemetry collection disabled, skipping.")
logging.info("api_url not set, skipping telemetry upload.")
# Check for the alerts specified
if enable_alerts: