From 8aa7b37e7eeaad4f0e2b79164e1e3f4fd5dc77d7 Mon Sep 17 00:00:00 2001
From: Tom Wilkie <tom@weave.works>
Date: Mon, 5 Oct 2015 11:48:37 +0000
Subject: [PATCH 1/3] Add tests for short-lived connection between hosts and
 containers.

---
 .../310_container_to_container_edge_test.sh   | 18 +++++++++++++
 .../320_container_edge_cross_host_2_test.sh   | 27 +++++++++++++++++++
 integration/setup.sh                          |  1 +
 3 files changed, 46 insertions(+)
 create mode 100755 integration/310_container_to_container_edge_test.sh
 create mode 100755 integration/320_container_edge_cross_host_2_test.sh

diff --git a/integration/310_container_to_container_edge_test.sh b/integration/310_container_to_container_edge_test.sh
new file mode 100755
index 000000000..b1585923e
--- /dev/null
+++ b/integration/310_container_to_container_edge_test.sh
@@ -0,0 +1,18 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Test short lived connections between containers"
+
+weave_on $HOST1 launch
+scope_on $HOST1 launch
+weave_on $HOST1 run -d --name nginx nginx
+weave_on $HOST1 run -d --name client alpine /bin/sh -c "while true; do wget http://nginx.weave.local:80/ >/dev/null; sleep 1; done"
+
+sleep 20 # give the probe a few seconds to build a report and send it to the app
+
+has_container $HOST1 nginx 1
+has_container $HOST1 client 1
+has_connection $HOST1 client nginx
+
+scope_end_suite
diff --git a/integration/320_container_edge_cross_host_2_test.sh b/integration/320_container_edge_cross_host_2_test.sh
new file mode 100755
index 000000000..ba98276e1
--- /dev/null
+++ b/integration/320_container_edge_cross_host_2_test.sh
@@ -0,0 +1,27 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Test short lived connections between containers on different hosts"
+
+weave_on $HOST1 launch $HOST1 $HOST2
+weave_on $HOST2 launch $HOST1 $HOST2
+
+scope_on $HOST1 launch
+scope_on $HOST2 launch
+
+weave_on $HOST1 run -d --name nginx nginx
+weave_on $HOST2 run -d --name client alpine /bin/sh -c "while true; do wget http://nginx.weave.local:80/ >/dev/null; sleep 1; done"
+
+sleep 30 # need to allow the scopes to poll dns, resolve the other app ids, and send them reports
+
+check() {
+	has_container $1 nginx 1
+	has_container $1 client 1
+	has_connection $1 client nginx
+}
+
+check $HOST1
+check $HOST2
+
+scope_end_suite
diff --git a/integration/setup.sh b/integration/setup.sh
index 53325d4c7..2ef69b345 100755
--- a/integration/setup.sh
+++ b/integration/setup.sh
@@ -21,6 +21,7 @@ echo Prefetching Images
 for HOST in $HOSTS; do
     weave_on $HOST setup
     docker_on $HOST pull peterbourgon/tns-db
+    docker_on $HOST pull alpine
 done
 
 curl -sL git.io/weave -o ./weave

From ffedae64c6cc720bb43f9f19cba4fe0ca3a0d991 Mon Sep 17 00:00:00 2001
From: Tom Wilkie <tom@weave.works>
Date: Fri, 9 Oct 2015 15:22:36 +0000
Subject: [PATCH 2/3] Make tests more reliable - Increase waits for system to
 settle - Add polling for containers and edges - Tabs formatting in setup.sh -
 300_internet_edge: Don't let curl fail in subshell (main source of flakyness)
 - Make last param (count) optional for has_container (from Paul)

---
 integration/100_launch_test.sh                |  8 ++--
 integration/105_launch_sans_weave_test.sh     |  4 +-
 integration/200_clustering_2_test.sh          |  4 +-
 .../205_clustering_sans_weave_2_test.sh       |  4 +-
 integration/300_internet_edge_test.sh         |  9 ++--
 .../310_container_to_container_edge_test.sh   |  6 +--
 .../320_container_edge_cross_host_2_test.sh   |  4 +-
 integration/config.sh                         | 44 ++++++++++++++++++-
 integration/setup.sh                          | 17 +++----
 9 files changed, 71 insertions(+), 29 deletions(-)

diff --git a/integration/100_launch_test.sh b/integration/100_launch_test.sh
index a51935a9f..3dd7ca853 100755
--- a/integration/100_launch_test.sh
+++ b/integration/100_launch_test.sh
@@ -7,10 +7,10 @@ start_suite "Launch scope and check it boots"
 weave_on $HOST1 launch
 scope_on $HOST1 launch
 
-sleep 5 # give the probe a few seconds to build a report and send it to the app
+wait_for_containers $HOST1 60 weave weaveproxy weavescope
 
-has_container $HOST1 weave 1
-has_container $HOST1 weaveproxy 1
-has_container $HOST1 weavescope 1
+has_container $HOST1 weave
+has_container $HOST1 weaveproxy
+has_container $HOST1 weavescope
 
 scope_end_suite
diff --git a/integration/105_launch_sans_weave_test.sh b/integration/105_launch_sans_weave_test.sh
index 8c0c702ea..d3a66a562 100755
--- a/integration/105_launch_sans_weave_test.sh
+++ b/integration/105_launch_sans_weave_test.sh
@@ -6,10 +6,10 @@ start_suite "Launch scope (without weave installed) and check it boots"
 
 scope_on $HOST1 launch
 
-sleep 5 # give the probe a few seconds to build a report and send it to the app
+wait_for_containers $HOST1 60 weavescope
 
 has_container $HOST1 weave 0
 has_container $HOST1 weaveproxy 0
-has_container $HOST1 weavescope 1
+has_container $HOST1 weavescope
 
 scope_end_suite
diff --git a/integration/200_clustering_2_test.sh b/integration/200_clustering_2_test.sh
index 64b3310fe..3982d11aa 100755
--- a/integration/200_clustering_2_test.sh
+++ b/integration/200_clustering_2_test.sh
@@ -19,8 +19,8 @@ check() {
 	has_container $1 weave 2
 	has_container $1 weaveproxy 2
 	has_container $1 weavescope 2
-	has_container $1 db1 1
-	has_container $1 db2 1
+	has_container $1 db1
+	has_container $1 db2
 }
 
 check $HOST1
diff --git a/integration/205_clustering_sans_weave_2_test.sh b/integration/205_clustering_sans_weave_2_test.sh
index 0a929fece..abd498682 100755
--- a/integration/205_clustering_sans_weave_2_test.sh
+++ b/integration/205_clustering_sans_weave_2_test.sh
@@ -16,8 +16,8 @@ check() {
 	has_container $1 weave 0
 	has_container $1 weaveproxy 0
 	has_container $1 weavescope 2
-	has_container $1 db1 1
-	has_container $1 db2 1
+	has_container $1 db1
+	has_container $1 db2
 }
 
 check $HOST1
diff --git a/integration/300_internet_edge_test.sh b/integration/300_internet_edge_test.sh
index 1b3a61a5d..cb56e038b 100755
--- a/integration/300_internet_edge_test.sh
+++ b/integration/300_internet_edge_test.sh
@@ -10,16 +10,17 @@ docker_on $HOST1 run -d -p 80:80 --name nginx nginx
 
 do_connections() {
 	while true; do
-		curl -s http://$HOST1:80/ >/dev/null
+		curl -s http://$HOST1:80/ >/dev/null || true
 		sleep 1
 	done
 }
 do_connections&
 
-sleep 5 # give the probe a few seconds to build a report and send it to the app
+wait_for_containers $HOST1 60 nginx "The Internet"
 
-has_container $HOST1 nginx 1
-has_connection $HOST1 "The Internet" nginx
+has_container $HOST1 nginx
+has_container $HOST1 "The Internet"
+has_connection $HOST1 "The Internet" nginx 60
 
 kill %do_connections
 
diff --git a/integration/310_container_to_container_edge_test.sh b/integration/310_container_to_container_edge_test.sh
index b1585923e..87327e34e 100755
--- a/integration/310_container_to_container_edge_test.sh
+++ b/integration/310_container_to_container_edge_test.sh
@@ -9,10 +9,10 @@ scope_on $HOST1 launch
 weave_on $HOST1 run -d --name nginx nginx
 weave_on $HOST1 run -d --name client alpine /bin/sh -c "while true; do wget http://nginx.weave.local:80/ >/dev/null; sleep 1; done"
 
-sleep 20 # give the probe a few seconds to build a report and send it to the app
+wait_for_containers $HOST1 60 nginx client
 
-has_container $HOST1 nginx 1
-has_container $HOST1 client 1
+has_container $HOST1 nginx
+has_container $HOST1 client
 has_connection $HOST1 client nginx
 
 scope_end_suite
diff --git a/integration/320_container_edge_cross_host_2_test.sh b/integration/320_container_edge_cross_host_2_test.sh
index ba98276e1..67d1b4886 100755
--- a/integration/320_container_edge_cross_host_2_test.sh
+++ b/integration/320_container_edge_cross_host_2_test.sh
@@ -16,8 +16,8 @@ weave_on $HOST2 run -d --name client alpine /bin/sh -c "while true; do wget http
 sleep 30 # need to allow the scopes to poll dns, resolve the other app ids, and send them reports
 
 check() {
-	has_container $1 nginx 1
-	has_container $1 client 1
+	has_container $1 nginx
+	has_container $1 client
 	has_connection $1 client nginx
 }
 
diff --git a/integration/config.sh b/integration/config.sh
index d41e06ef4..daf72dfbd 100644
--- a/integration/config.sh
+++ b/integration/config.sh
@@ -35,7 +35,7 @@ weave_on() {
 has_container() {
 	local host=$1
 	local name=$2
-	local count=$3
+	local count=${3:-1}
 	assert "curl -s http://$host:4040/api/topology/containers?system=show | jq -r '[.nodes[] | select(.label_major == \"$name\")] | length'" $count
 }
 
@@ -57,7 +57,47 @@ has_connection() {
 	local host="$1"
 	local from="$2"
 	local to="$3"
+	local timeout="${4:-10}"
 	local from_id=$(container_id "$host" "$from")
 	local to_id=$(container_id "$host" "$to")
-	assert "curl -s http://$host:4040/api/topology/containers?system=show | jq -r '.nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])'" true
+
+	for i in $(seq $timeout); do
+		local containers="$(curl -s http://$host:4040/api/topology/containers?system=show)"
+		local edge=$(echo "$containers" |  jq -r ".nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])")
+		if [ "$edge" = "true" ]; then
+			echo "Found edge $from -> $to after $i secs"
+			assert "curl -s http://$host:4040/api/topology/containers?system=show |  jq -r '.nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])'" true
+			return
+		fi
+		sleep 1
+	done
+
+	echo "Failed to fing edge $from -> $to after $timeout secs"
+	assert "curl -s http://$host:4040/api/topology/containers?system=show |  jq -r '.nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])'" true
+}
+
+wait_for_containers() {
+	local host="$1"
+	local timeout="$2"
+	shift 2
+
+	for i in $(seq $timeout); do
+		local containers="$(curl -s http://$host:4040/api/topology/containers?system=show)"
+		local found=0
+		for name in "$@"; do
+			local count=$(echo "$containers" | jq -r "[.nodes[] | select(.label_major == \"$name\")] | length")
+			if [ "$count" -ge 1 ]; then
+				found=$(( found + 1 ))
+			fi
+		done
+
+		if [ "$found" -eq $# ]; then
+			echo "Found $found containers after $i secs"
+			return
+		fi
+
+		sleep 1
+	done
+
+	echo "Failed to find containers $@ after $i secs"
 }
diff --git a/integration/setup.sh b/integration/setup.sh
index 2ef69b345..b16b475f6 100755
--- a/integration/setup.sh
+++ b/integration/setup.sh
@@ -6,22 +6,23 @@ set -e
 
 echo Copying scope images and scripts to hosts
 for HOST in $HOSTS; do
-    docker_on $HOST load -i ../scope.tar
-    upload_executable $HOST ../scope
-    upload_executable $HOST ../scope /usr/local/scope/bin/scope
+	docker_on $HOST load -i ../scope.tar
+	upload_executable $HOST ../scope
+	upload_executable $HOST ../scope /usr/local/scope/bin/scope
 done
 
 echo Installing weave
 for HOST in $HOSTS; do
-    run_on $HOST "sudo curl -sL git.io/weave -o /usr/local/bin/weave"
-    run_on $HOST "sudo chmod a+x /usr/local/bin/weave"
+	run_on $HOST "sudo curl -sL git.io/weave -o /usr/local/bin/weave"
+	run_on $HOST "sudo chmod a+x /usr/local/bin/weave"
 done
 
 echo Prefetching Images
 for HOST in $HOSTS; do
-    weave_on $HOST setup
-    docker_on $HOST pull peterbourgon/tns-db
-    docker_on $HOST pull alpine
+	weave_on $HOST setup
+	docker_on $HOST pull peterbourgon/tns-db
+	docker_on $HOST pull alpine
+	docker_on $HOST pull nginx
 done
 
 curl -sL git.io/weave -o ./weave

From 1577b90c61f8e678998b18d82e50308f35712a18 Mon Sep 17 00:00:00 2001
From: Tom Wilkie <tom@weave.works>
Date: Mon, 12 Oct 2015 11:18:09 +0000
Subject: [PATCH 3/3] Review feedback

---
 integration/300_internet_edge_test.sh               | 8 +++++++-
 integration/310_container_to_container_edge_test.sh | 5 ++++-
 integration/320_container_edge_cross_host_2_test.sh | 5 ++++-
 integration/config.sh                               | 8 ++++----
 4 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/integration/300_internet_edge_test.sh b/integration/300_internet_edge_test.sh
index cb56e038b..3e5b07a1b 100755
--- a/integration/300_internet_edge_test.sh
+++ b/integration/300_internet_edge_test.sh
@@ -4,6 +4,12 @@
 
 start_suite "Test short lived connections from the Internet"
 
+if ! echo "$HOST1" | grep "us-central1-a"; then
+	echo "Skipping; test needs to be run against VMs on GCE."
+	scope_end_suite
+	exit
+fi
+
 weave_on $HOST1 launch
 scope_on $HOST1 launch
 docker_on $HOST1 run -d -p 80:80 --name nginx nginx
@@ -20,7 +26,7 @@ wait_for_containers $HOST1 60 nginx "The Internet"
 
 has_container $HOST1 nginx
 has_container $HOST1 "The Internet"
-has_connection $HOST1 "The Internet" nginx 60
+has_connection $HOST1 "The Internet" nginx
 
 kill %do_connections
 
diff --git a/integration/310_container_to_container_edge_test.sh b/integration/310_container_to_container_edge_test.sh
index 87327e34e..ec19d8ca9 100755
--- a/integration/310_container_to_container_edge_test.sh
+++ b/integration/310_container_to_container_edge_test.sh
@@ -7,7 +7,10 @@ start_suite "Test short lived connections between containers"
 weave_on $HOST1 launch
 scope_on $HOST1 launch
 weave_on $HOST1 run -d --name nginx nginx
-weave_on $HOST1 run -d --name client alpine /bin/sh -c "while true; do wget http://nginx.weave.local:80/ >/dev/null; sleep 1; done"
+weave_on $HOST1 run -d --name client alpine /bin/sh -c "while true; do \
+	wget http://nginx.weave.local:80/ >/dev/null || true; \
+	sleep 1; \
+done"
 
 wait_for_containers $HOST1 60 nginx client
 
diff --git a/integration/320_container_edge_cross_host_2_test.sh b/integration/320_container_edge_cross_host_2_test.sh
index 67d1b4886..b13cd097c 100755
--- a/integration/320_container_edge_cross_host_2_test.sh
+++ b/integration/320_container_edge_cross_host_2_test.sh
@@ -11,7 +11,10 @@ scope_on $HOST1 launch
 scope_on $HOST2 launch
 
 weave_on $HOST1 run -d --name nginx nginx
-weave_on $HOST2 run -d --name client alpine /bin/sh -c "while true; do wget http://nginx.weave.local:80/ >/dev/null; sleep 1; done"
+weave_on $HOST2 run -d --name client alpine /bin/sh -c "while true; do \
+	wget http://nginx.weave.local:80/ >/dev/null || true; \
+	sleep 1; \
+done"
 
 sleep 30 # need to allow the scopes to poll dns, resolve the other app ids, and send them reports
 
diff --git a/integration/config.sh b/integration/config.sh
index daf72dfbd..274813ab7 100644
--- a/integration/config.sh
+++ b/integration/config.sh
@@ -57,13 +57,13 @@ has_connection() {
 	local host="$1"
 	local from="$2"
 	local to="$3"
-	local timeout="${4:-10}"
+	local timeout="${4:-60}"
 	local from_id=$(container_id "$host" "$from")
 	local to_id=$(container_id "$host" "$to")
 
 	for i in $(seq $timeout); do
 		local containers="$(curl -s http://$host:4040/api/topology/containers?system=show)"
-		local edge=$(echo "$containers" |  jq -r ".nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])")
+		local edge=$(echo "$containers" |  jq -r ".nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])" 2>/dev/null)
 		if [ "$edge" = "true" ]; then
 			echo "Found edge $from -> $to after $i secs"
 			assert "curl -s http://$host:4040/api/topology/containers?system=show |  jq -r '.nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])'" true
@@ -72,7 +72,7 @@ has_connection() {
 		sleep 1
 	done
 
-	echo "Failed to fing edge $from -> $to after $timeout secs"
+	echo "Failed to find edge $from -> $to after $timeout secs"
 	assert "curl -s http://$host:4040/api/topology/containers?system=show |  jq -r '.nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])'" true
 }
 
@@ -86,7 +86,7 @@ wait_for_containers() {
 		local found=0
 		for name in "$@"; do
 			local count=$(echo "$containers" | jq -r "[.nodes[] | select(.label_major == \"$name\")] | length")
-			if [ "$count" -ge 1 ]; then
+			if [ -n "$count" ] && [ "$count" -ge 1 ]; then
 				found=$(( found + 1 ))
 			fi
 		done