ctdb-tests: Move simple tests to INTEGRATION/ subdirectory

Split some tests out into database/ and failover/ subdirectories.
Rename the remaining tests in simple/.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

11 files changed, 671 insertions, 0 deletions

diff --git a/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh b/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh
new file mode 100755
index 00000000000..d9fdd28f775
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb ip' shows the correct output.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb ip' on one of the nodes and verify the list of IP
+   addresses displayed (cross check the result with the output of
+   'ip addr show' on the node).
+3. Verify that pipe-separated output is generated with the -X option.
+
+Expected results:
+
+* 'ctdb ip' shows the list of public IPs being served by a node.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 1 "$CTDB ip all | tail -n +2"
+ips=$(sed \
+	-e 's@ node\[@ @' \
+	-e 's@\].*$@@' \
+	"$outfile")
+machineout=$(sed -r \
+	-e 's@^| |$@\|@g' \
+	-e 's@[[:alpha:]]+\[@@g' \
+	-e 's@\]@@g' \
+	"$outfile")
+
+if [ -z "$TEST_LOCAL_DAEMONS" ]; then
+	while read ip pnn ; do
+		try_command_on_node $pnn "ip addr show to ${ip}"
+		if [ -n "$out" ] ; then
+			echo "GOOD: node $pnn appears to have $ip assigned"
+		else
+			die "BAD: node $pnn does not appear to have $ip assigned"
+		fi
+	done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
+fi
+
+echo "Looks good!"
+
+cmd="$CTDB -X ip all | tail -n +2"
+echo "Checking that \"$cmd\" produces expected output..."
+
+try_command_on_node 1 "$cmd"
+if [ "$out" = "$machineout" ] ; then
+    echo "Yep, looks good!"
+else
+    echo "Nope, it looks like this:"
+    echo "$out"
+    echo "Should be like this:"
+    echo "$machineout"
+    exit 1
+fi
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh b/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh
new file mode 100755
index 00000000000..aefed54c45e
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that an IP address can be added to a node using 'ctdb addip'.
+
+This test does not do any network level checks to make sure IP
+addresses are actually on interfaces.  It just consults "ctdb ip".
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+get_test_ip_mask_and_iface
+
+echo "Deleting IP $test_ip from all nodes"
+delete_ip_from_all_nodes $test_ip
+try_command_on_node -v $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node '!' $test_node $test_ip
+
+# Debugging...
+try_command_on_node -v all $CTDB ip
+
+echo "Adding IP ${test_ip}/${mask} on ${iface}, node ${test_node}"
+try_command_on_node $test_node $CTDB addip ${test_ip}/${mask} $iface
+try_command_on_node $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node $test_node $test_ip
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh b/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh
new file mode 100755
index 00000000000..d3f0f3a012b
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that a node's public IP address can be deleted using 'ctdb deleteip'.
+
+This test does not do any network level checks to make sure IP
+addresses are actually on interfaces.  It just consults "ctdb ip".
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+
+echo "Deleting IP ${test_ip} from node ${test_node}"
+try_command_on_node $test_node $CTDB delip $test_ip
+try_command_on_node $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node '!' $test_node $test_ip
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh b/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh
new file mode 100755
index 00000000000..451fca3a866
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh
@@ -0,0 +1,133 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that IPs can be rearrranged using 'ctdb reloadips'.
+
+Various sub-tests that remove addresses from the public_addresses file
+on a node or delete the entire contents of the public_addresses file.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Expected results:
+
+* When addresses are deconfigured "ctdb ip" no longer reports them and
+  when added they are seen again.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+
+try_command_on_node $test_node $CTDB_TEST_WRAPPER ctdb_base_show
+addresses="${out}/public_addresses"
+echo "Public addresses file on node $test_node is \"$addresses\""
+backup="${addresses}.$$"
+
+restore_public_addresses ()
+{
+    try_command_on_node $test_node "mv $backup $addresses >/dev/null 2>&1 || true"
+}
+ctdb_test_exit_hook_add restore_public_addresses
+
+# ctdb reloadips will fail if it can't disable takover runs.  The most
+# likely reason for this is that there is already a takeover run in
+# progress.  We can't predict when this will happen, so retry if this
+# occurs.
+do_ctdb_reloadips ()
+{
+	local retry_max=10
+	local retry_count=0
+	while : ; do
+		if try_command_on_node any "$CTDB reloadips all" ; then
+			return 0
+		fi
+
+		if [ "$out" != "Failed to disable takeover runs" ] ; then
+			return 1
+		fi
+
+		if [ $retry_count -ge $retry_max ] ; then
+			return 1
+		fi
+
+		retry_count=$((retry_count + 1))
+		echo "Retrying..."
+		sleep_for 1
+	done
+}
+
+
+echo "Removing IP $test_ip from node $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
+
+do_ctdb_reloadips
+
+try_command_on_node $test_node $CTDB ip
+
+if grep "^${test_ip} " <<<"$out" ; then
+    cat <<EOF
+BAD: node $test_node can still host IP $test_ip:
+$out
+EOF
+    exit 1
+fi
+
+cat <<EOF
+GOOD: node $test_node is no longer hosting IP $test_ip:
+$out
+EOF
+
+try_command_on_node any $CTDB sync
+
+
+echo "Restoring addresses"
+restore_public_addresses
+
+do_ctdb_reloadips
+
+echo "Getting list of public IPs on node $test_node"
+try_command_on_node $test_node "$CTDB ip | tail -n +2"
+
+if [ -z "$out" ] ; then
+    echo "BAD: node $test_node has no ips"
+    exit 1
+fi
+
+cat <<EOF
+GOOD: node $test_node has these addresses:
+$out
+EOF
+
+try_command_on_node any $CTDB sync
+
+
+echo "Emptying public addresses file on $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
+
+do_ctdb_reloadips
+
+echo "Getting list of public IPs on node $test_node"
+try_command_on_node $test_node "$CTDB ip | tail -n +2"
+
+if [ -n "$out" ] ; then
+    cat <<EOF
+BAD: node $test_node still has ips:
+$out
+EOF
+    exit 1
+fi
+
+echo "GOOD: no IPs left on node $test_node"
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh b/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh
new file mode 100755
index 00000000000..d162f0e5fba
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Check that CTDB operates correctly if:
+
+* failover is disabled; or
+* there are 0 public IPs configured
+
+This test only does anything with local daemons.  On a real cluster it
+has no way of updating configuration.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
+	echo "SKIPPING this test - only runs against local daemons"
+	exit 0
+fi
+
+echo "Starting CTDB with failover disabled..."
+ctdb_test_init -F
+
+cluster_is_healthy
+
+echo "Getting IP allocation..."
+try_command_on_node -v any "$CTDB ip all | tail -n +2"
+
+while read ip pnn ; do
+	if [ "$pnn" != "-1" ] ; then
+		die "BAD: IP address ${ip} is assigned to node ${pnn}"
+	fi
+done <"$outfile"
+
+echo "GOOD: All IP addresses are unassigned"
+
+echo "----------------------------------------"
+
+echo "Starting CTDB with an empty public addresses configuration..."
+ctdb_test_init -P /dev/null
+
+cluster_is_healthy
+
+echo "Trying explicit ipreallocate..."
+try_command_on_node any $CTDB ipreallocate
+
+echo "Good, that seems to work!"
+echo
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh b/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh
new file mode 100755
index 00000000000..2756f64ceb1
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that an interface is deleted when all IPs on it are deleted.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+
+# Find interfaces on test node
+try_command_on_node $test_node "$CTDB ifaces -X"
+ifaces=$(awk -F'|' 'NR > 1 { print $2 }' "$outfile")
+echo "Node ${test_node} has interfaces: ${ifaces}"
+
+# Delete all IPs on each interface...  deleting IPs from one interface
+# can cause other interfaces to disappear, so we need to be careful...
+for i in $ifaces ; do
+    try_command_on_node $test_node "$CTDB ifaces -X"
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
+
+    if [ -z "$info" ] ; then
+	echo "Interface ${i} missing... assuming already deleted!"
+	continue
+    fi
+
+    echo "Deleting IPs on interface ${i}, with this information:"
+    echo " $info"
+
+    try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2"
+    awk -F'|' -v i="$i" \
+	'$6 == i { print $2 }' "$outfile" |
+    while read ip ; do
+	echo "  $ip"
+	try_command_on_node $test_node "$CTDB delip $ip"
+    done
+    try_command_on_node $test_node "$CTDB ipreallocate"
+
+    try_command_on_node $test_node "$CTDB ifaces -X"
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
+
+    if [ -z "$info" ] ; then
+	echo "GOOD: Interface ${i} has been garbage collected"
+    else
+	echo "BAD: Interface ${i} still exists"
+	echo "$out"
+	exit 1
+    fi
+done
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh b/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh
new file mode 100755
index 00000000000..699ccc3d2ee
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that  'ctdb moveip' allows movement of public IPs between cluster nodes.
+
+This test does not do any network level checks to make sure IP
+addresses are actually on interfaces.  It just consults "ctdb ip".
+
+To work, this test ensures that IPAllocAlgorithm is not set to 0
+(Deterministic IPs) and sets NoIPFailback.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+
+sanity_check_ips ()
+{
+    echo "Sanity checking IPs..."
+
+    local x ipp prev
+    prev=""
+    while read x ipp ; do
+	[ "$ipp" = "-1" ] && break
+	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+	    echo "OK"
+	    return 0
+	fi
+	prev="$ipp"
+    done <"$outfile"
+
+    echo "BAD: a node was -1 or IPs are only assigned to one node:"
+    cat "$outfile"
+    echo "Are you running an old version of CTDB?"
+    return 1
+}
+
+sanity_check_ips
+
+# Find a target node - it must be willing to host $test_ip
+try_command_on_node any "$CTDB listnodes | wc -l"
+num_nodes="$out"
+to_node=""
+for i in $(seq 0 $(($num_nodes - 1)) ) ; do
+    [ $i -ne $test_node ] || continue
+    all_ips_on_node $i
+    while read ip x ; do
+	if [ "$ip" = "$test_ip" ] ; then
+	    to_node="$i"
+	    break 2
+	fi
+    done <"$outfile"
+done
+
+if [ -z "$to_node" ] ; then
+    echo "Unable to find target node"
+    exit 1
+fi
+
+echo "Target node is ${to_node}"
+
+echo "Setting IPAllocAlgorithm=2 to avoid Deterministic IPs..."
+try_command_on_node -q all $CTDB setvar IPAllocAlgorithm 2
+
+echo "Turning on NoIPFailback..."
+try_command_on_node -q all $CTDB setvar NoIPFailback 1
+
+echo "Attempting to move ${test_ip} from node ${test_node} to node ${to_node}"
+try_command_on_node $test_node $CTDB moveip $test_ip $to_node
+wait_until_ips_are_on_node '!' $test_node $test_ip
+wait_until_ips_are_on_node $to_node $test_ip
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh b/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh
new file mode 100755
index 00000000000..c0bb62d1991
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of "ctdb disable" and "ctdb enable"
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+########################################
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+
+echo "Disabling node $test_node"
+try_command_on_node 1 $CTDB disable -n $test_node
+wait_until_node_has_status $test_node disabled
+wait_until_node_has_no_ips "$test_node"
+
+echo "Re-enabling node $test_node"
+try_command_on_node 1 $CTDB enable -n $test_node
+wait_until_node_has_status $test_node enabled
+wait_until_node_has_some_ips "$test_node"
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh b/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh
new file mode 100755
index 00000000000..d9a64508180
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of "ctdb stop" and "ctdb continue"
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+
+echo "Stopping node ${test_node}..."
+try_command_on_node 1 $CTDB stop -n $test_node
+wait_until_node_has_status $test_node stopped
+wait_until_node_has_no_ips "$test_node"
+
+echo "Continuing node $test_node"
+try_command_on_node 1 $CTDB continue -n $test_node
+wait_until_node_has_status $test_node notstopped
+wait_until_node_has_some_ips "$test_node"
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh b/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh
new file mode 100755
index 00000000000..9af55d8e7ef
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that  'ctdb setvar NoIPTakeover 1' stops ip addresses from being failed 
+over onto the node.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+   served.
+3. Use 'ctdb moveip' to move an address from one node to another.
+4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node.
+
+Expected results:
+
+* 'ctdb moveip' allows an IP address to be moved between cluster nodes.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+echo "There are $num_nodes nodes..."
+
+if [ $num_nodes -lt 2 ] ; then
+    echo "Less than 2 nodes!"
+    exit 1
+fi
+
+
+echo "Wait until the ips are reallocated"
+sleep_for 30
+try_command_on_node 0 "$CTDB ipreallocate"
+
+num=`try_command_on_node -v 1 "$CTDB ip" | grep -v Public | egrep " 1$" | wc -l`
+echo "Number of addresses on node 1 : $num"
+
+
+echo "Turning on NoIPTakeover on all nodes"
+try_command_on_node all "$CTDB setvar NoIPTakeover 1"
+try_command_on_node 1 "$CTDB ipreallocate"
+
+echo Disable node 1
+try_command_on_node 1 "$CTDB disable"
+try_command_on_node 1 "$CTDB ipreallocate"
+num=`try_command_on_node -v 1 "$CTDB ip" | grep -v Public | egrep " 1$" | wc -l`
+echo "Number of addresses on node 1 : $num"
+[ "$num" != "0" ] && {
+    echo "BAD: node 1 still hosts ip addresses"
+    exit 1
+}
+
+
+echo "Enable node 1 again"
+try_command_on_node 1 "$CTDB enable"
+sleep_for 30
+try_command_on_node 1 "$CTDB ipreallocate"
+try_command_on_node 1 "$CTDB ipreallocate"
+num=`try_command_on_node -v 1 "$CTDB ip" | grep -v Public | egrep " 1$" | wc -l`
+echo "Number of addresses on node 1 : $num"
+[ "$num" != "0" ] && {
+    echo "BAD: node took over ip addresses"
+    exit 1
+}
+
+
+echo "OK. ip addresses were not taken over"
+exit 0
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh b/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh
new file mode 100755
index 00000000000..5b11f9f3885
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the recovery daemon handles unhosted IPs properly.
+
+This test does not do any network level checks to make sure the IP
+address is actually on an interface.  It just consults "ctdb ip".
+
+This is a variation of the "addip" test.
+EOF
+}
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+ctdb_test_init
+
+set -e
+
+cluster_is_healthy
+
+select_test_node_and_ips
+
+echo "Running test against node $test_node and IP $test_ip"
+
+get_test_ip_mask_and_iface
+
+echo "Deleting IP $test_ip from all nodes"
+delete_ip_from_all_nodes $test_ip
+try_command_on_node -v $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node ! $test_node $test_ip
+
+try_command_on_node -v all $CTDB ip
+
+my_exit_hook ()
+{
+    if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
+	onnode -q all $CTDB event script enable legacy "10.interface"
+    fi
+}
+
+ctdb_test_exit_hook_add my_exit_hook
+
+# This forces us to wait until the ipreallocated associated with the
+# delips is complete.
+try_command_on_node $test_node $CTDB sync
+
+# Wait for a monitor event.  Then the next steps are unlikely to occur
+# in the middle of a monitor event and will have the expected effect.
+wait_for_monitor_event $test_node
+
+if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
+    # Stop monitor events from bringing up the link status of an interface
+    try_command_on_node $test_node $CTDB event script disable legacy 10.interface
+fi
+
+echo "Marking interface $iface down on node $test_node"
+try_command_on_node $test_node $CTDB setifacelink $iface down
+
+echo "Adding IP $test_ip to node $test_node"
+try_command_on_node $test_node $CTDB addip $test_ip/$mask $iface
+try_command_on_node $test_node $CTDB ipreallocate
+
+echo "Wait long enough for IP verification to have taken place"
+sleep_for 15
+
+echo "Ensuring that IP ${test_ip} is not hosted on node ${test_node} when interface is down"
+if ips_are_on_node '!' $test_node $test_ip; then
+    echo "GOOD: the IP has not been hosted while the interface is down"
+else
+    echo "BAD: the IP is hosted but the interface is down"
+    exit 1
+fi
+
+echo "Marking interface $iface up on node $test_node"
+try_command_on_node $test_node $CTDB setifacelink $iface up
+wait_until_ips_are_on_node $test_node $test_ip