ctdb-tests: Factor out getting leader and waiting for leader change

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

4 files changed, 63 insertions, 32 deletions

diff --git a/ctdb/tests/INTEGRATION/database/recovery.002.large.sh b/ctdb/tests/INTEGRATION/database/recovery.002.large.sh
index 3dd0d66e000..47360719d3b 100755
--- a/ctdb/tests/INTEGRATION/database/recovery.002.large.sh
+++ b/ctdb/tests/INTEGRATION/database/recovery.002.large.sh
@@ -75,12 +75,10 @@ else
 	exit 1
 fi
 
-echo "Find out which node is recmaster"
-try_command_on_node 0 $CTDB recmaster
-recmaster="$out"
-
+echo
+leader_get 0
 # Set RecBufferSizeLimit to 10000
-try_command_on_node $recmaster $CTDB setvar RecBufferSizeLimit 10000
+ctdb_onnode "$leader" setvar RecBufferSizeLimit 10000
 
 # Do a recovery
 echo "force recovery"
diff --git a/ctdb/tests/INTEGRATION/simple/cluster.002.recmaster_yield.sh b/ctdb/tests/INTEGRATION/simple/cluster.002.recmaster_yield.sh
index 67406dc29d2..4a2932d42d1 100755
--- a/ctdb/tests/INTEGRATION/simple/cluster.002.recmaster_yield.sh
+++ b/ctdb/tests/INTEGRATION/simple/cluster.002.recmaster_yield.sh
@@ -8,22 +8,13 @@ set -e
 
 ctdb_test_init
 
-echo "Finding out which node is the recovery master..."
-try_command_on_node -v 0 "$CTDB recmaster"
-test_node=$out
+leader_get 0
 
-echo "Stopping node ${test_node} - it is the current recmaster..."
-try_command_on_node 1 $CTDB stop -n $test_node
+# leader set by leader_get()
+# shellcheck disable=SC2154
+echo "Stopping leader ${leader}..."
+ctdb_onnode 1 stop -n "$leader"
 
-wait_until_node_has_status $test_node stopped
+wait_until_node_has_status "$leader" stopped
 
-echo "Checking which node is the recovery master now..."
-try_command_on_node -v 0 "$CTDB recmaster"
-recmaster=$out
-
-if [ "$recmaster" != "$test_node" ] ; then
-    echo "OK: recmaster moved to node $recmaster"
-else
-    echo "BAD: recmaster did not move"
-    exit 1
-fi
+wait_until_leader_has_changed 0
diff --git a/ctdb/tests/INTEGRATION/simple/cluster.015.reclock_remove_lock.sh b/ctdb/tests/INTEGRATION/simple/cluster.015.reclock_remove_lock.sh
index a9119df6e4f..36b72818f24 100755
--- a/ctdb/tests/INTEGRATION/simple/cluster.015.reclock_remove_lock.sh
+++ b/ctdb/tests/INTEGRATION/simple/cluster.015.reclock_remove_lock.sh
@@ -54,11 +54,7 @@ echo "Recovery lock setting is \"${reclock_setting}\""
 echo "Recovery lock file is \"${reclock}\""
 echo
 
-echo "Get current recovery master"
-ctdb_onnode "$test_node" recmaster
-recmaster="$out"
-echo "Recovery master is node ${recmaster}"
-echo
+leader_get "$test_node"
 
 echo "Get initial generation"
 ctdb_onnode "$test_node" status
@@ -79,15 +75,17 @@ echo
 echo "Generation changed to ${generation_new}"
 echo
 
-echo "Get current recovery master"
-ctdb_onnode "$test_node" recmaster
-recmaster_new="$out"
+# shellcheck disable=SC2154
+# $leader set by leader_get() above
+leader_old="$leader"
+
+leader_get "$test_node"
 
-if [ "$recmaster" != "$recmaster_new" ] ; then
+if [ "$leader" != "$leader_old" ] ; then
 	ctdb_test_fail \
-		"BAD: Recovery master has changed to node ${recmaster_new}"
+		"BAD: Leader has changed to node ${leader}"
 fi
-echo "GOOD: Recovery master is still node ${recmaster_new}"
+echo "GOOD: Leader is still node ${leader}"
 echo
 
 cluster_is_healthy
diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index cff2903b339..c9457aaf850 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -644,6 +644,50 @@ ctdb_base_show ()
 
 #######################################
 
+# sets: leader
+_leader_get ()
+{
+	local node="$1"
+
+	ctdb_onnode "$node" recmaster
+	# shellcheck disable=SC2154
+	# $out set by ctdb_onnode() above
+	leader="$out"
+}
+
+leader_get ()
+{
+	local node="$1"
+
+	echo "Get leader"
+	_leader_get "$node"
+	echo "Leader is ${leader}"
+	echo
+}
+
+_leader_has_changed ()
+{
+	local node="$1"
+	local leader_old="$2"
+
+	_leader_get "$node"
+
+	[ "$leader" != "$leader_old" ]
+}
+
+# uses: leader
+wait_until_leader_has_changed ()
+{
+	local node="$1"
+
+	echo
+	echo "Wait until leader changes..."
+	wait_until 30 _leader_has_changed "$node" "$leader"
+	echo "Leader changed to ${leader}"
+}
+
+#######################################
+
 wait_for_monitor_event ()
 {
     local pnn="$1"