From 481fc439b61563ebc095cd77e2e326a46243e5cb Mon Sep 17 00:00:00 2001
From: Michael Goodness <mgoodness@gmail.com>
Date: Mon, 4 Sep 2017 09:24:45 -0500
Subject: [PATCH] [testing] Wait for ready containers (#1923)

* Test container readiness

* Test Pods running & containers Ready

* Include container name in output
---
 test/verify-release.sh | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/test/verify-release.sh b/test/verify-release.sh
index 942b9e5d16..d25cf581b5 100755
--- a/test/verify-release.sh
+++ b/test/verify-release.sh
@@ -14,27 +14,37 @@ fi
 # Ensure all pods in the namespace entered a Running state
 SUCCESS=0
 PODS_FOUND=0
-COUNT=0
+POD_RETRY_COUNT=0
 RETRY=18
 RETRY_DELAY=10
-while [ "$COUNT" -lt "$RETRY" ]; do
+while [ "$POD_RETRY_COUNT" -lt "$RETRY" ]; do
   POD_STATUS=`kubectl get pods --no-headers --namespace $NAMESPACE`
   if [ -z "$POD_STATUS" ];then
     echo "INFO: No pods found for this release, retrying after sleep"
-    COUNT=$((COUNT+1))
+    POD_RETRY_COUNT=$((POD_RETRY_COUNT+1))
     sleep $RETRY_DELAY
     continue
   else
     PODS_FOUND=1
   fi
+
   if ! echo "$POD_STATUS" | grep -v Running;then
     echo "INFO: All pods entered the Running state"
-    exit 0
-  else
-    echo "INFO: Sleeping waiting for containers to be ready"
-    COUNT=$((COUNT+1))
-    kubectl get pods  --no-headers --namespace $NAMESPACE
-    sleep $RETRY_DELAY
+
+    CONTAINER_RETRY_COUNT=0
+    while [ "$CONTAINER_RETRY_COUNT" -lt "$RETRY" ]; do
+      UNREADY_CONTAINERS=`kubectl get pods --namespace $NAMESPACE \
+        -o jsonpath="{.items[*].status.containerStatuses[?(@.ready!=true)].name}"`
+      if [ -n "$UNREADY_CONTAINERS" ];then
+        echo "INFO: Some containers are not yet ready; retrying after sleep"
+        CONTAINER_RETRY_COUNT=$((CONTAINER_RETRY_COUNT+1))
+        sleep $RETRY_DELAY
+        continue
+      else
+        echo "INFO: All containers are ready"
+        exit 0
+      fi
+    done
   fi
 done