Merge pull request #1466 from abronan/fix_ci_discovery_race

Integration: Wait for nodes to be registered and discovered before testing
docker-archive · Dec 8, 2015 · a4a1d3b · a4a1d3b
2 parents 67a4d55 + f1b7b7b
commit a4a1d3b
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 14 deletions.
diff --git a/test/integration/discovery/consul.bats b/test/integration/discovery/consul.bats
@@ -99,7 +99,7 @@ function teardown() {
 	swarm_join "$DISCOVERY"
 
 	# Start a manager. It should keep retrying
-	swarm_manage "$DISCOVERY"
+	swarm_manage_no_wait "$DISCOVERY"
 
 	# Now start the store
 	start_store

diff --git a/test/integration/discovery/discovery_helpers.bash b/test/integration/discovery/discovery_helpers.bash
@@ -2,14 +2,6 @@
 
 load ../helpers
 
-# Returns true if all nodes have joined the swarm.
-function discovery_check_swarm_info() {
-	local total="$1"
-	[ -z "$total" ] && total="${#HOSTS[@]}"
-
-	docker_swarm info | grep -q "Nodes: $total"
-}
-
 # Returns true if swarm info outputs is empty (0 nodes).
 function discovery_check_swarm_info_empty() {
 	docker_swarm info | grep -q "Nodes: 0"

diff --git a/test/integration/discovery/etcd.bats b/test/integration/discovery/etcd.bats
@@ -106,7 +106,7 @@ function teardown() {
 	swarm_join "$DISCOVERY"
 
 	# Start a manager. It should keep retrying
-	swarm_manage "$DISCOVERY"
+	swarm_manage_no_wait "$DISCOVERY"
 
 	# Now start the store
 	start_store

diff --git a/test/integration/discovery/file.bats b/test/integration/discovery/file.bats
@@ -82,7 +82,7 @@ function setup_discovery_file() {
 	start_docker 2
 
 	# Start a manager. It should keep retrying
-	swarm_manage "$DISCOVERY"
+	swarm_manage_no_wait "$DISCOVERY"
 
 	# Now create the discovery file.
 	setup_discovery_file

diff --git a/test/integration/discovery/zk.bats b/test/integration/discovery/zk.bats
@@ -99,7 +99,7 @@ function teardown() {
 	swarm_join "$DISCOVERY"
 
 	# Start a manager. It should keep retrying
-	swarm_manage "$DISCOVERY"
+	swarm_manage_no_wait "$DISCOVERY"
 
 	# Now start the store
 	start_store

diff --git a/test/integration/helpers.bash b/test/integration/helpers.bash
@@ -85,8 +85,27 @@ function wait_until_reachable() {
 	retry 10 1 docker -H $1 info
 }
 
-# Start the swarm manager in background.
+# Returns true if all nodes have joined the swarm.
+function discovery_check_swarm_info() {
+	local total="$1"
+	[ -z "$total" ] && total="${#HOSTS[@]}"
+	local host="$2"
+	[ -z "$host" ] && host="${SWARM_HOSTS[0]}"
+
+	retry 10 1 eval "docker -H $host info | grep -q -e \"Nodes: $total\" -e \"Offers: $total\""
+}
+
 function swarm_manage() {
+	local i=${#SWARM_MANAGE_PID[@]}
+
+	swarm_manage_no_wait "$@"
+
+	# Wait for nodes to be discovered
+	discovery_check_swarm_info "${#HOSTS[@]}" "${SWARM_HOSTS[$i]}"
+}
+
+# Start the swarm manager in background.
+function swarm_manage_no_wait() {
 	local discovery
 	if [ $# -eq 0 ]; then
 		discovery=`join , ${HOSTS[@]}`
@@ -101,6 +120,8 @@ function swarm_manage() {
 	"$SWARM_BINARY" -l debug manage -H "$host" --heartbeat=1s $discovery &
 	SWARM_MANAGE_PID[$i]=$!
 	SWARM_HOSTS[$i]=$host
+
+	# Wait for the Manager to be reachable
 	wait_until_reachable "$host"
 }
 

diff --git a/test/integration/mesos/zk/zk.bats b/test/integration/mesos/zk/zk.bats
@@ -32,7 +32,6 @@ function teardown() {
 	start_mesos_zk $DISCOVERY
 
 	swarm_manage --cluster-driver mesos-experimental $DISCOVERY
-	sleep 5
 
 	run docker_swarm info
 	[ "$status" -eq 0 ]