Updating unknown status for when cluster becomes disconnected

kraken/kubernetes/client.py

@@ -282,16 +282,16 @@ def get_job_status(name, namespace="default"):
 
 
 # Obtain node status
-def get_node_status(node):
+def get_node_status(node, timeout=60):
     try:
-        node_info = cli.read_node_status(node, pretty=True)
+        node_info = cli.read_node_status(node, pretty=True, _request_timeout=timeout)
     except ApiException as e:
         logging.error(
             "Exception when calling \
                        CoreV1Api->read_node_status: %s\n"
             % e
         )
-        raise e
+        return None
     for condition in node_info.status.conditions:
         if condition.type == "Ready":
             return condition.status

kraken/node_actions/common_node_functions.py

@@ -43,11 +43,17 @@ def wait_for_ready_status(node, timeout):
 # Wait till node status becomes NotReady
 def wait_for_unknown_status(node, timeout):
     for _ in range(timeout):
-        if kubecli.get_node_status(node) == "Unknown":
-            break
+        try:
+            node_status = kubecli.get_node_status(node, timeout)
+            if node_status is None or node_status == "Unknown":
+                break
+        except Exception:
+            logging.error("Encountered error while getting node status, waiting 3 seconds and retrying")
         time.sleep(3)
-    if kubecli.get_node_status(node) != "Unknown":
-        raise Exception("Node condition status isn't Unknown")
+    node_status = kubecli.get_node_status(node, timeout)
+    logging.info("node status " + str(node_status))
+    if node_status is not None and node_status != "Unknown":
+        raise Exception("Node condition status isn't Unknown after %s seconds" % str(timeout))
 
 
 # Get the ip of the cluster node