adding failure path catches (#1547)

krkn/scenario_plugins/abstract_scenario_plugin.py

@@ -150,22 +150,34 @@ class AbstractScenarioPlugin(ABC):
             scenario_telemetry.end_timestamp = time.time()
             start_time = int(scenario_telemetry.start_timestamp)
             end_time = int(scenario_telemetry.end_timestamp)
-            utils.collect_and_put_ocp_logs(
-                telemetry,
-                parsed_scenario_config,
-                telemetry.get_telemetry_request_id(),
-                start_time,
-                end_time
-            )
-
-            if events_backup:
-                utils.populate_cluster_events(
-                    krkn_config,
+            try:
+                utils.collect_and_put_ocp_logs(
+                    telemetry,
                     parsed_scenario_config,
-                    telemetry.get_lib_kubernetes(),
+                    telemetry.get_telemetry_request_id(),
                     start_time,
                     end_time
                 )
+            except Exception as e:
+                logging.error(
+                    f"failed to collect OCP logs for scenario "
+                    f"'{scenario_config}': {e}"
+                )
+
+            if events_backup:
+                try:
+                    utils.populate_cluster_events(
+                        krkn_config,
+                        parsed_scenario_config,
+                        telemetry.get_lib_kubernetes(),
+                        start_time,
+                        end_time
+                    )
+                except Exception as e:
+                    logging.error(
+                        f"failed to collect cluster events for scenario "
+                        f"'{scenario_config}': {e}"
+                    )
 
             if scenario_telemetry.exit_status != 0:
                 failed_scenarios.append(scenario_config)

krkn/utils/functions.py

@@ -15,6 +15,7 @@ import krkn_lib.utils
 from krkn_lib.k8s import KrknKubernetes
 from krkn_lib.models.telemetry import ScenarioTelemetry
 from krkn_lib.telemetry.ocp import KrknTelemetryOpenshift
+from kubernetes.client.exceptions import ApiException
 from tzlocal.unix import get_localzone
 import logging
 
@@ -59,11 +60,26 @@ def collect_and_put_ocp_logs(
     start_timestamp: int,
     end_timestamp: int,
 ):
+    try:
+        is_k8s = telemetry_ocp.get_lib_kubernetes().is_kubernetes()
+    except ApiException as e:
+        logging.error(
+            f"API error ({e.status}) checking cluster type for OCP log "
+            f"collection, skipping: {e.reason}"
+        )
+        return
+    except Exception as e:
+        logging.error(
+            f"unexpected error checking cluster type for OCP log "
+            f"collection, skipping: {e}"
+        )
+        return
+
     if (
         telemetry_ocp.get_telemetry_config()
         and telemetry_ocp.get_telemetry_config()["enabled"]
         and telemetry_ocp.get_telemetry_config()["logs_backup"]
-        and not telemetry_ocp.get_lib_kubernetes().is_kubernetes()
+        and not is_k8s
     ):
         namespaces = __retrieve_namespaces(
             scenario_config, telemetry_ocp.get_lib_kubernetes()