[testing] Wait for ready containers (#1923)

* Test container readiness

* Test Pods running & containers Ready

* Include container name in output

test/verify-release.sh

@@ -14,27 +14,37 @@ fi
 # Ensure all pods in the namespace entered a Running state
 SUCCESS=0
 PODS_FOUND=0
-COUNT=0
+POD_RETRY_COUNT=0
 RETRY=18
 RETRY_DELAY=10
-while [ "$COUNT" -lt "$RETRY" ]; do
+while [ "$POD_RETRY_COUNT" -lt "$RETRY" ]; do
   POD_STATUS=`kubectl get pods --no-headers --namespace $NAMESPACE`
   if [ -z "$POD_STATUS" ];then
     echo "INFO: No pods found for this release, retrying after sleep"
-    COUNT=$((COUNT+1))
+    POD_RETRY_COUNT=$((POD_RETRY_COUNT+1))
     sleep $RETRY_DELAY
     continue
   else
     PODS_FOUND=1
   fi
+
   if ! echo "$POD_STATUS" | grep -v Running;then
     echo "INFO: All pods entered the Running state"
-    exit 0
-  else
-    echo "INFO: Sleeping waiting for containers to be ready"
-    COUNT=$((COUNT+1))
-    kubectl get pods  --no-headers --namespace $NAMESPACE
-    sleep $RETRY_DELAY
+
+    CONTAINER_RETRY_COUNT=0
+    while [ "$CONTAINER_RETRY_COUNT" -lt "$RETRY" ]; do
+      UNREADY_CONTAINERS=`kubectl get pods --namespace $NAMESPACE \
+        -o jsonpath="{.items[*].status.containerStatuses[?(@.ready!=true)].name}"`
+      if [ -n "$UNREADY_CONTAINERS" ];then
+        echo "INFO: Some containers are not yet ready; retrying after sleep"
+        CONTAINER_RETRY_COUNT=$((CONTAINER_RETRY_COUNT+1))
+        sleep $RETRY_DELAY
+        continue
+      else
+        echo "INFO: All containers are ready"
+        exit 0
+      fi
+    done
   fi
 done