diff --git a/kraken/kubernetes/client.py b/kraken/kubernetes/client.py index 0986e90c..042df621 100644 --- a/kraken/kubernetes/client.py +++ b/kraken/kubernetes/client.py @@ -282,16 +282,16 @@ def get_job_status(name, namespace="default"): # Obtain node status -def get_node_status(node): +def get_node_status(node, timeout=60): try: - node_info = cli.read_node_status(node, pretty=True) + node_info = cli.read_node_status(node, pretty=True, _request_timeout=timeout) except ApiException as e: logging.error( "Exception when calling \ CoreV1Api->read_node_status: %s\n" % e ) - raise e + return None for condition in node_info.status.conditions: if condition.type == "Ready": return condition.status diff --git a/kraken/node_actions/common_node_functions.py b/kraken/node_actions/common_node_functions.py index a33eb57d..6eeba948 100644 --- a/kraken/node_actions/common_node_functions.py +++ b/kraken/node_actions/common_node_functions.py @@ -43,11 +43,17 @@ def wait_for_ready_status(node, timeout): # Wait till node status becomes NotReady def wait_for_unknown_status(node, timeout): for _ in range(timeout): - if kubecli.get_node_status(node) == "Unknown": - break + try: + node_status = kubecli.get_node_status(node, timeout) + if node_status is None or node_status == "Unknown": + break + except Exception: + logging.error("Encountered error while getting node status, waiting 3 seconds and retrying") time.sleep(3) - if kubecli.get_node_status(node) != "Unknown": - raise Exception("Node condition status isn't Unknown") + node_status = kubecli.get_node_status(node, timeout) + logging.info("node status " + str(node_status)) + if node_status is not None and node_status != "Unknown": + raise Exception("Node condition status isn't Unknown after %s seconds" % str(timeout)) # Get the ip of the cluster node