ctdb-tests: Fix "setreclock" test

1. Waiting for generation to change isn't enough.  It now changes
   early in recovery and reclock might not have been set on all nodes.
   So wait until recovery is complete before checking reclock.

2. Use onnode -p when setting reclock on all nodes.  That will cause
   it to be set quickly on all nodes, to avoid the situation where the
   recovery master is the late in the list and resets the reclock on
   earlier nodes back to the old value.  This shouldn't matter if (1)
   is done but it can't hurt.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Michael Adam <obnox@samba.org>

1 files changed, 12 insertions, 5 deletions

diff --git a/ctdb/tests/simple/35_set_reclock.sh b/ctdb/tests/simple/35_set_reclock.sh
index 7628ffba584..07e31855e34 100755
--- a/ctdb/tests/simple/35_set_reclock.sh
+++ b/ctdb/tests/simple/35_set_reclock.sh
@@ -42,6 +42,12 @@ wait_until_generation_has_changed ()
     wait_until 60 generation_has_changed
 }
 
+wait_until_recovered ()
+{
+    wait_until_generation_has_changed
+    wait_until_node_has_status all recovered
+}
+
 echo "Check that recovery lock is set the same on all nodes..."
 try_command_on_node -v -q all $CTDB getreclock
 n=$(echo "$out" | sort -u | wc -l)
@@ -68,7 +74,8 @@ echo "Remember original recovery lock file: \"${orig_reclock}\""
 
 echo
 echo "Unset and test the recovery lock on all nodes..."
-try_command_on_node -q all $CTDB setreclock
+try_command_on_node -pq all $CTDB setreclock
+wait_until_recovered
 try_command_on_node -v -q all $CTDB getreclock
 t=$(sort -u <<<"$out")
 if [ "$t" = "No reclock file used." ] ; then
@@ -85,7 +92,8 @@ echo "Current generation is ${generation}"
 alt="${orig_reclock}.test"
 echo
 echo "Set alternative recovery lock (${alt}) and test on all nodes..."
-try_command_on_node -q all $CTDB setreclock "$alt"
+try_command_on_node -pq all $CTDB setreclock "$alt"
+wait_until_recovered
 try_command_on_node -v -q all $CTDB getreclock
 t=$(echo "$out" | sed -e 's@^Reclock file:@@' | sort -u)
 if [ "$t" = "$alt" ] ; then
@@ -97,12 +105,12 @@ else
 fi
 
 # Setting or updating the recovery lock file must cause a recovery
-wait_until_generation_has_changed
 echo "Current generation is ${generation}"
 
 echo
 echo "Restore and test the recovery lock on all nodes..."
-try_command_on_node -q all $CTDB setreclock "$orig_reclock"
+try_command_on_node -pq all $CTDB setreclock "$orig_reclock"
+wait_until_recovered
 try_command_on_node -v all rm -vf "$alt"
 try_command_on_node -v -q all $CTDB getreclock
 t=$(echo "$out" | sed -e 's@^Reclock file:@@' | sort -u)
@@ -113,5 +121,4 @@ else
     exit 1
 fi
 
-wait_until_generation_has_changed
 echo "Current generation is ${generation}"