diff options
author | Martin Schwenke <martin@meltin.net> | 2019-05-07 15:29:19 +1000 |
---|---|---|
committer | Amitay Isaacs <amitay@samba.org> | 2019-05-14 07:25:37 +0000 |
commit | 8be4ee1a28d5c037955832b6f827d40f28f02796 (patch) | |
tree | 193efeb96b2db258e1408b2ef4d10dc1cfe40d25 /ctdb/tests | |
parent | cf00db40355b49443263187f9d97934f91287e51 (diff) | |
download | samba-8be4ee1a28d5c037955832b6f827d40f28f02796.tar.gz |
ctdb-tests: Make ctdb reloadips tests more reliable
ctdb reloadips will fail if it can't disable takover runs. The most
likely reason for this is that there is already a takeover run in
progress. We can't predict when this will happen, so retry if this
occurs.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
Diffstat (limited to 'ctdb/tests')
-rwxr-xr-x | ctdb/tests/complex/18_ctdb_reloadips.sh | 35 | ||||
-rwxr-xr-x | ctdb/tests/simple/18_ctdb_reloadips.sh | 33 |
2 files changed, 61 insertions, 7 deletions
diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh index 4ba1b26a8e8..26782130430 100755 --- a/ctdb/tests/complex/18_ctdb_reloadips.sh +++ b/ctdb/tests/complex/18_ctdb_reloadips.sh @@ -184,6 +184,33 @@ check_ips () fi } +# ctdb reloadips will fail if it can't disable takover runs. The most +# likely reason for this is that there is already a takeover run in +# progress. We can't predict when this will happen, so retry if this +# occurs. +do_ctdb_reloadips () +{ + local retry_max=10 + local retry_count=0 + while : ; do + if try_command_on_node "$test_node" "$CTDB reloadips" ; then + return 0 + fi + + if [ "$out" != "Failed to disable takeover runs" ] ; then + return 1 + fi + + if [ $retry_count -ge $retry_max ] ; then + return 1 + fi + + retry_count=$((retry_count + 1)) + echo "Retrying..." + sleep_for 1 + done +} + #################### new_ip_max=100 @@ -193,7 +220,7 @@ new_ip_max=100 add_ips_to_original_config \ $test_node "$addresses" "$iface" "$prefix" 1 $new_ip_max -try_command_on_node $test_node "$CTDB reloadips" +do_ctdb_reloadips check_ips $test_node "$iface" "$prefix" 1 $new_ip_max @@ -207,7 +234,7 @@ echo "Using 'ctdb reloadips' to remove the 1st address just added..." add_ips_to_original_config \ $test_node "$addresses" "$iface" "$prefix" 2 $new_ip_max -try_command_on_node $test_node "$CTDB reloadips" +do_ctdb_reloadips check_ips $test_node "$iface" "$prefix" 2 $new_ip_max @@ -222,7 +249,7 @@ echo "Updating to include only about 1/2 of the new IPs..." add_ips_to_original_config \ $test_node "$addresses" "$iface" "$prefix" $start $new_ip_max -try_command_on_node $test_node "$CTDB reloadips" +do_ctdb_reloadips check_ips $test_node "$iface" "$prefix" $start $new_ip_max @@ -234,6 +261,6 @@ try_command_on_node any $CTDB sync echo "Restoring original IP configuration..." restore_public_addresses -try_command_on_node $test_node "$CTDB reloadips" +do_ctdb_reloadips check_ips $test_node "$iface" "$prefix" 0 diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh index 9c60a14a93e..61c3a561da9 100755 --- a/ctdb/tests/simple/18_ctdb_reloadips.sh +++ b/ctdb/tests/simple/18_ctdb_reloadips.sh @@ -42,9 +42,36 @@ restore_public_addresses () } ctdb_test_exit_hook_add restore_public_addresses +# ctdb reloadips will fail if it can't disable takover runs. The most +# likely reason for this is that there is already a takeover run in +# progress. We can't predict when this will happen, so retry if this +# occurs. +do_ctdb_reloadips () +{ + local retry_max=10 + local retry_count=0 + while : ; do + if try_command_on_node any "$CTDB reloadips all" ; then + return 0 + fi + + if [ "$out" != "Failed to disable takeover runs" ] ; then + return 1 + fi + + if [ $retry_count -ge $retry_max ] ; then + return 1 + fi + + retry_count=$((retry_count + 1)) + echo "Retrying..." + sleep_for 1 + done +} + try_command_on_node $test_node "mv $addresses $backup && touch $addresses" -try_command_on_node any $CTDB reloadips all +do_ctdb_reloadips echo "Getting list of public IPs on node $test_node" try_command_on_node $test_node "$CTDB ip | tail -n +2" @@ -64,7 +91,7 @@ try_command_on_node any $CTDB sync echo "Restoring addresses" restore_public_addresses -try_command_on_node any $CTDB reloadips all +do_ctdb_reloadips echo "Getting list of public IPs on node $test_node" try_command_on_node $test_node "$CTDB ip | tail -n +2" @@ -85,7 +112,7 @@ echo "Removing IP $test_ip from node $test_node" try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses" -try_command_on_node any $CTDB reloadips all +do_ctdb_reloadips try_command_on_node $test_node $CTDB ip |