ctdb-tests: Add vacuuming tests

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
Autobuild-Date(master): Thu Oct 24 05:28:21 UTC 2019 on sn-devel-184

9 files changed, 1011 insertions, 0 deletions

diff --git a/ctdb/tests/INTEGRATION/database/scripts/local.bash b/ctdb/tests/INTEGRATION/database/scripts/local.bash
index c36f155d6ce..5334b1cb7d8 100644
--- a/ctdb/tests/INTEGRATION/database/scripts/local.bash
+++ b/ctdb/tests/INTEGRATION/database/scripts/local.bash
@@ -29,3 +29,56 @@ check_cattdb_num_records ()
 
 	return $ret
 }
+
+vacuum_test_key_dmaster ()
+{
+	local node="$1"
+	local db="$2"
+	local key="$3"
+	local dmaster="${4:-${node}}"
+
+	testprog_onnode "$node" "ctdb-db-test local-read ${db} ${key}"
+
+	# shellcheck disable=SC2154
+	# $outfile is set above by try_command_on_node()
+	if ! grep -Fqx "dmaster: ${dmaster}" "$outfile" ; then
+		echo "BAD: node ${dmaster} is not dmaster"
+		cat "$outfile"
+		ctdb_test_fail
+	fi
+}
+
+vacuum_confirm_key_empty_dmaster ()
+{
+	local node="$1"
+	local db="$2"
+	local key="$3"
+	local dmaster="${4:-${node}}"
+
+	echo "Confirm record key=\"${key}\" is empty and dmaster=${dmaster}"
+
+	vacuum_test_key_dmaster "$node" "$db" "$key" "$dmaster"
+
+	if ! grep -Fqx 'data(0) = ""' "$outfile" ; then
+		echo "BAD: record not empty"
+		cat "$outfile"
+		ctdb_test_fail
+	fi
+}
+
+db_confirm_key_has_value ()
+{
+	local node="$1"
+	local db="$2"
+	local key="$3"
+	local val="$4"
+
+	local out
+
+	ctdb_onnode "$node" "readkey ${db} ${key}"
+	outv=$(echo "$out" | sed -n 's|^Data: size:.* ptr:\[\(.*\)\]$|\1|p')
+	if [ "$val" != "$outv" ] ; then
+		ctdb_test_fail \
+			"BAD: value for \"${key}\"=\"${outv}\" (not \"${val}\")"
+	fi
+}
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.001.fast.sh b/ctdb/tests/INTEGRATION/database/vacuum.001.fast.sh
new file mode 100755
index 00000000000..651de4df57c
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.001.fast.sh
@@ -0,0 +1,159 @@
+#!/bin/bash
+
+# Ensure that vacuuming deletes records on all nodes
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+vacuum_test ()
+{
+	local db="$1"
+	local num_records="$2"
+	local delete_from_lmaster="${3:-false}"
+
+	local t
+	if "$delete_from_lmaster" ; then
+		t="lmaster"
+	else
+		t="non-lmaster"
+	fi
+
+	echo
+	echo '............................................................'
+	printf 'Creating %d record(s)\n' "$num_records"
+	printf 'Testing vacuuming of 1 record deleted from %s\n' "$t"
+	echo '............................................................'
+
+	echo
+	echo "Stall vacuuming on all nodes"
+	ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+	echo
+	echo "Getting list of nodes..."
+	local all_pnns
+	ctdb_get_all_pnns
+
+	local first
+	first=$(echo "$all_pnns" | sed -n -e '1p')
+
+	echo
+	echo "Create/wipe test database ${db}"
+	ctdb_onnode "$first" "attach ${db}"
+	ctdb_onnode "$first" "wipedb ${db}"
+
+	echo
+	echo "Write ${num_records} records to ${db}"
+	local i
+	for i in $(seq 1 "$num_records") ; do
+		ctdb_onnode "$first" "writekey ${db} test${i} value${i}"
+	done
+
+	echo
+	echo "Migrate record(s) to all nodes"
+	for i in $(seq 1 "$num_records") ; do
+		ctdb_onnode all "readkey ${db} test${i}"
+	done
+
+	echo
+	echo "Confirm that all nodes have all the records"
+	check_cattdb_num_records "$db" "$num_records" "$all_pnns"
+
+	local key="test1"
+	echo
+	echo "Delete key ${key}"
+
+	echo "  Find lmaster for key \"${key}\""
+	testprog_onnode "$first" "ctdb-db-test get-lmaster ${key}"
+	# out  is set above
+	# shellcheck disable=SC2154
+	lmaster="$out"
+	echo "  lmaster=${lmaster}"
+
+	if "$delete_from_lmaster" ; then
+		echo "  Delete key ${key} on lmaster node ${lmaster}"
+		dnode="$lmaster"
+	else
+		for i in $all_pnns ; do
+			if [ "$i" != "$lmaster" ] ; then
+				dnode="$i"
+				break
+			fi
+		done
+		echo "  Delete key ${key} on non-lmaster node ${dnode}"
+	fi
+	ctdb_onnode "$dnode" "deletekey ${db} ${key}"
+
+	echo
+	vacuum_confirm_key_empty_dmaster "$dnode" "$db" "$key"
+
+	echo
+	echo "Confirm all records still exist on all nodes"
+	check_cattdb_num_records "$db" "$num_records" "$all_pnns"
+
+	if ! "$delete_from_lmaster" ; then
+		# Ask the lmaster to fetch the deleted record
+		echo
+		echo "Vacuum on non-lmaster node ${dnode}"
+		testprog_onnode "$dnode" "ctdb-db-test vacuum ${db}"
+
+		echo
+		vacuum_confirm_key_empty_dmaster "$dnode" "$db" "$key"
+
+		# Fetch the record and put it in the delete queue in
+		# the main daemon for processing in next vacuuming run
+		# on the lmaster
+		echo
+		echo "Vacuum on lmaster node ${lmaster}"
+		testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+		echo
+		echo "Confirm all records still exist on all node nodes"
+		check_cattdb_num_records "$db" "$num_records" "$all_pnns"
+
+		echo
+		vacuum_confirm_key_empty_dmaster "$lmaster" "$db" "$key"
+	fi
+
+	echo
+	# In the delete-from-lmaster case, the record is already in
+	# the lmaster's delete-queue so only a single run is needed
+	echo "Vacuum on lmaster node ${lmaster}"
+	testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+	echo
+	echo "Confirm a record has been deleted on all nodes"
+	local n=$((num_records - 1))
+	check_cattdb_num_records "$db" "$n" "$all_pnns"
+
+	echo
+	echo "Confirm all other records still exist with expected values"
+	local i
+	for i in $(seq 1 "$num_records") ; do
+		local k="test${i}"
+		local v="value${i}"
+
+		if [ "$k" = "$key" ] ; then
+			continue
+		fi
+
+		db_confirm_key_has_value "$first" "$db" "$k" "$v"
+	done
+	echo "GOOD"
+}
+
+testdb="vacuum_test.tdb"
+
+# 1 record, delete from non-lmaster
+vacuum_test "$testdb" 1 false
+
+# 10 records, delete from non-lmaster
+vacuum_test "$testdb" 10 false
+
+# 1 record, delete from lmaster
+vacuum_test "$testdb" 1 true
+
+# 10 records, delete from lmaster
+vacuum_test "$testdb" 10 true
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.002.full.sh b/ctdb/tests/INTEGRATION/database/vacuum.002.full.sh
new file mode 100755
index 00000000000..cc1113b02f5
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.002.full.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# Ensure a full vacuuming run deletes records
+
+# Create some records, delete some of them on their lmaster (with a
+# test tool that doesn't do SCHEDULE_FOR_DELETION), run some fast
+# vacuuming runs (to ensure they don't delete records that haven't
+# been added to the delete queue) and then try a full vacuuming run,
+# which will actually do a traverse of the database to find empty
+# records and delete them.  Confirm that records that haven't been
+# deleted are still there, with expected values.
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+db="vacuum_test.tdb"
+
+echo "Stall vacuuming on all nodes"
+ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+echo
+echo "Getting list of nodes..."
+ctdb_get_all_pnns
+
+# all_pnns is set above by ctdb_get_all_pnns()
+# shellcheck disable=SC2154
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+echo
+echo "Create/wipe test database ${db}"
+ctdb_onnode "$first" "attach ${db}"
+ctdb_onnode "$first" "wipedb ${db}"
+
+echo
+echo "Create records in ${db}"
+for i in $(seq 1 10) ; do
+	ctdb_onnode "$first" "writekey ${db} delete${i} value${i}"
+	ctdb_onnode "$first" "writekey ${db} keep${i} value${i}"
+done
+
+echo
+echo "Migrate record(s) to all nodes"
+for i in $(seq 1 10) ; do
+	ctdb_onnode all "readkey ${db} delete${i}"
+	ctdb_onnode all "readkey ${db} keep${i}"
+done
+
+echo
+echo "Confirm that all nodes have all the records"
+check_cattdb_num_records "$db" 20 "$all_pnns"
+
+echo
+echo "Delete all 10 records from their lmaster node"
+for i in $(seq 1 10) ; do
+	key="delete${i}"
+
+	testprog_onnode "$first" "ctdb-db-test get-lmaster ${key}"
+	# $out is set above by testprog_onnode()
+	# shellcheck disable=SC2154
+	lmaster="$out"
+
+	echo
+	echo "Delete ${key} from lmaster node ${lmaster}"
+	testprog_onnode "$lmaster" \
+			     "ctdb-db-test fetch-local-delete $db ${key}"
+
+	vacuum_confirm_key_empty_dmaster "$lmaster" "$db" "$key"
+done
+
+echo "Do fast vacuuming run on all nodes"
+testprog_onnode "all" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm all records still exist on all nodes"
+check_cattdb_num_records "$db" 20 "$all_pnns"
+
+echo
+echo "Do full vacuuming run on all nodes"
+testprog_onnode "all" "ctdb-db-test vacuum ${db} full"
+
+echo
+echo "Confirm 10 records exist on all nodes"
+check_cattdb_num_records "$db" 10 "$all_pnns"
+
+echo
+echo "Confirm  that remaining records still exist with expected values"
+for i in $(seq 1 10) ; do
+	k="keep${i}"
+	v="value${i}"
+
+	db_confirm_key_has_value "$first" "$db" "$k" "$v"
+done
+echo "GOOD"
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.003.recreate.sh b/ctdb/tests/INTEGRATION/database/vacuum.003.recreate.sh
new file mode 100755
index 00000000000..00cb2c3df29
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.003.recreate.sh
@@ -0,0 +1,139 @@
+#!/bin/bash
+
+# Ensure that vacuuming does not delete a record that is recreated
+# before vacuuming completes.  This needs at least 3 nodes.
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+db="vacuum_test.tdb"
+
+echo "Stall vacuuming on all nodes"
+ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+echo
+echo "Getting list of nodes..."
+ctdb_get_all_pnns
+
+# all_pnns is set above by ctdb_get_all_pnns()
+# shellcheck disable=SC2154
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+echo
+echo "Create/wipe test database ${db}"
+ctdb_onnode "$first" "attach ${db}"
+ctdb_onnode "$first" "wipedb ${db}"
+
+echo
+echo "Create a record in ${db}"
+ctdb_onnode "$first" "writekey ${db} key value1"
+
+echo
+echo "Migrate record to all nodes"
+ctdb_onnode all "readkey ${db} key"
+
+echo
+echo "Confirm that all nodes have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+echo "Determine lmaster node for key"
+testprog_onnode "$first" "ctdb-db-test get-lmaster key"
+# $out is set above by testprog_onnode()
+# shellcheck disable=SC2154
+lmaster="$out"
+echo "lmaster=${lmaster}"
+
+non_lmaster=""
+# Find a non-lmaster node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] ; then
+		non_lmaster="$i"
+		break
+	fi
+done
+if [ -z "$non_lmaster" ] ; then
+	ctdb_test_fail "Could not find non-lmaster node for key"
+fi
+
+another_non_lmaster=""
+# Find another non-lmaster node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] && [ "$i" != "$non_lmaster" ] ; then
+		another_non_lmaster="$i"
+		break
+	fi
+done
+if [ -z "$another_non_lmaster" ] ; then
+	ctdb_test_fail "Could not find another non-lmaster node for key"
+fi
+
+vacuum_test ()
+{
+	local db="$1"
+	local key="$2"
+	local val="$3"
+	local dnode="$4"
+	local rnode="$5"
+	local rrun="$6"
+
+	echo
+	echo '............................................................'
+	printf 'Delete key %s on node %d\n' "$key" "$dnode"
+	printf 'Recreate on node %d after %d vacuuming run(s)\n' \
+	       "$rnode" "$rrun"
+	echo '............................................................'
+
+	echo
+	echo "Delete key \"${key}\" from node ${dnode}"
+	ctdb_onnode "$dnode" "deletekey ${db} ${key}"
+
+	if [ "$rrun" -eq 0 ] ; then
+		echo "Recreate record on node ${rnode}"
+		ctdb_onnode "$rnode" "writekey ${db} ${key} ${val}"
+	fi
+
+	echo "Do a fast vacuuming run on node ${dnode}"
+	testprog_onnode "$dnode" "ctdb-db-test vacuum ${db}"
+
+	if [ "$rrun" -eq 1 ] ; then
+		echo "Recreate record on node ${rnode}"
+		ctdb_onnode "$rnode" "writekey ${db} ${key} ${val}"
+	fi
+
+	echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+	testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+	if [ "$rrun" -eq 2 ] ; then
+		echo "Recreate record on node ${rnode}"
+		ctdb_onnode "$rnode" "writekey ${db} ${key} ${val}"
+	fi
+
+	echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+	testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+	echo
+	echo "Confirm the record still exists on all nodes"
+	check_cattdb_num_records "$db" 1 "$all_pnns"
+
+	echo
+	echo "Confirm the record contains correct value"
+	db_confirm_key_has_value "$first" "$db" "$key" "$val"
+}
+
+vacuum_test "$db" "key" "value01" "$non_lmaster" "$non_lmaster" 0
+vacuum_test "$db" "key" "value02" "$non_lmaster" "$another_non_lmaster" 0
+vacuum_test "$db" "key" "value03" "$non_lmaster" "$lmaster" 0
+vacuum_test "$db" "key" "value04" "$lmaster" "$non_lmaster" 0
+vacuum_test "$db" "key" "value05" "$lmaster" "$lmaster" 0
+
+vacuum_test "$db" "key" "value06" "$non_lmaster" "$non_lmaster" 1
+vacuum_test "$db" "key" "value07" "$non_lmaster" "$lmaster" 1
+vacuum_test "$db" "key" "value08" "$non_lmaster" "$another_non_lmaster" 1
+
+vacuum_test "$db" "key" "value09" "$non_lmaster" "$non_lmaster" 2
+vacuum_test "$db" "key" "value10" "$non_lmaster" "$lmaster" 2
+vacuum_test "$db" "key" "value11" "$non_lmaster" "$another_non_lmaster" 2
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.030.locked.sh b/ctdb/tests/INTEGRATION/database/vacuum.030.locked.sh
new file mode 100755
index 00000000000..193a769096d
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.030.locked.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Confirm that a record is not vacuumed if it is locked when the 1st
+# fast vacuuming run occurs on the node on which it was deleted, but
+# is dropped from the delete queue
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+db="vacuum_test.tdb"
+key="key"
+
+echo "Stall vacuuming on all nodes"
+ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+echo
+echo "Getting list of nodes..."
+ctdb_get_all_pnns
+
+# all_pnns is set above by ctdb_get_all_pnns()
+# shellcheck disable=SC2154
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+echo
+echo "Determine lmaster node for key"
+testprog_onnode "$first" "ctdb-db-test get-lmaster key"
+# $out is set above by testprog_onnode()
+# shellcheck disable=SC2154
+lmaster="$out"
+echo "lmaster=${lmaster}"
+
+non_lmaster=""
+# Find a non-lmaster node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] ; then
+		non_lmaster="$i"
+		break
+	fi
+done
+if [ -z "$non_lmaster" ] ; then
+	ctdb_test_fail "Could not find non-lmaster node for key"
+fi
+
+echo "............................................................"
+echo "Delete key ${key} on non-lmaster node ${non_lmaster}"
+echo "Lock on node ${non_lmaster} during 1st vacuuming run"
+echo "............................................................"
+
+echo
+
+echo "Create/wipe test database ${db}"
+ctdb_onnode "$first" "attach ${db}"
+ctdb_onnode "$first" "wipedb ${db}"
+
+echo "Create a record in ${db}"
+ctdb_onnode "$first" "writekey ${db} ${key} value1"
+
+echo "Migrate record to all nodes"
+ctdb_onnode all "readkey ${db} ${key}"
+
+echo "Confirm that all nodes have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+
+echo "Delete key \"${key}\" from node ${non_lmaster}"
+ctdb_onnode "$non_lmaster" "deletekey $db ${key}"
+
+echo "Lock record on node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test local-lock ${db} ${key}"
+pid="${out#OK }"
+ctdb_test_cleanup_pid_set "$non_lmaster" "$pid"
+
+echo "Do a fast vacuuming run on node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Kill lock process ${pid} on node ${non_lmaster}"
+try_command_on_node "$non_lmaster" "kill ${pid}"
+ctdb_test_cleanup_pid_clear
+
+echo
+
+# If the record is still in the delete queue then this will process it
+echo "Do a fast vacuuming run on node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+
+echo "Confirm the record still exists on all nodes"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+vacuum_confirm_key_empty_dmaster "$non_lmaster" "$db" "$key"
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh b/ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh
new file mode 100755
index 00000000000..284808c8775
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# Confirm that a record is vacuumed if it is locked on the deleting
+# node when the 2nd fast vacuuming run occurs, but vacuuming is
+# delayed until the lock is released
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+db="vacuum_test.tdb"
+key="key"
+
+echo "Stall vacuuming on all nodes"
+ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+echo
+echo "Getting list of nodes..."
+ctdb_get_all_pnns
+
+# all_pnns is set above by ctdb_get_all_pnns()
+# shellcheck disable=SC2154
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+echo
+echo "Determine lmaster node for key"
+testprog_onnode "$first" "ctdb-db-test get-lmaster key"
+# $out is set above by testprog_onnode()
+# shellcheck disable=SC2154
+lmaster="$out"
+echo "lmaster=${lmaster}"
+
+non_lmaster=""
+# Find a non-lmaster node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] ; then
+		non_lmaster="$i"
+		break
+	fi
+done
+if [ -z "$non_lmaster" ] ; then
+	ctdb_test_fail "Could not find non-lmaster node for key"
+fi
+
+echo "............................................................"
+echo "Delete key ${key} on node ${non_lmaster}"
+echo "Lock on non-lmaster node ${non_lmaster} during 2nd vacuuming run"
+echo "............................................................"
+
+echo
+
+echo "Create/wipe test database ${db}"
+ctdb_onnode "$first" "attach ${db}"
+ctdb_onnode "$first" "wipedb ${db}"
+
+echo "Create a record in ${db}"
+ctdb_onnode "$first" "writekey ${db} ${key} value1"
+
+echo "Migrate record to all nodes"
+ctdb_onnode all "readkey ${db} ${key}"
+
+echo "Confirm that all nodes have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+
+echo "Delete key \"${key}\" from node ${non_lmaster}"
+ctdb_onnode "$non_lmaster" "deletekey $db ${key}"
+
+echo
+echo "Do a fast vacuuming run on node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that all nodes still have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+echo "Lock record on non-lmaster node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test local-lock ${db} ${key}"
+pid="${out#OK }"
+ctdb_test_cleanup_pid_set "$non_lmaster" "$pid"
+
+echo
+echo "Do a fast vacuuming run on lmaster node ${lmaster} - THIS WILL FAIL"
+status=0
+testprog_onnode "$lmaster" "ctdb-db-test -t 10 vacuum ${db}" || status=$?
+
+if [ $status -ne 110 ] ; then
+	ctdb_test_fail "$out"
+fi
+
+echo "Confirm record key=\"${key}\" has dmaster=${non_lmaster}"
+vacuum_test_key_dmaster "$lmaster" "$db" "$key" "$non_lmaster"
+
+echo "Kill lock process ${pid} on node ${non_lmaster}"
+try_command_on_node "$non_lmaster" "kill ${pid}"
+ctdb_test_cleanup_pid_clear
+
+echo "Wait until record is migrated to lmaster node ${lmaster}"
+wait_until 30 vacuum_test_key_dmaster "$lmaster" "$db" "$key"
+
+echo
+echo "Confirm that all nodes still have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo "Do a fast vacuuming run on node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that the record is gone from all nodes"
+check_cattdb_num_records "$db" 0 "$all_pnns"
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.032.locked.sh b/ctdb/tests/INTEGRATION/database/vacuum.032.locked.sh
new file mode 100755
index 00000000000..2f983aa79d6
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.032.locked.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Confirm that a record is not vacuumed if it is locked on the lmaster
+# when the 3rd fast vacuuming run occurs, but is dropped from the
+# lmaster delete queue
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+db="vacuum_test.tdb"
+key="key"
+
+echo "Stall vacuuming on all nodes"
+ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+echo
+echo "Getting list of nodes..."
+ctdb_get_all_pnns
+
+# all_pnns is set above by ctdb_get_all_pnns()
+# shellcheck disable=SC2154
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+echo
+echo "Determine lmaster node for key"
+testprog_onnode "$first" "ctdb-db-test get-lmaster key"
+# $out is set above by testprog_onnode()
+# shellcheck disable=SC2154
+lmaster="$out"
+echo "lmaster=${lmaster}"
+
+non_lmaster=""
+# Find a non-lmaster node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] ; then
+		non_lmaster="$i"
+		break
+	fi
+done
+if [ -z "$non_lmaster" ] ; then
+	ctdb_test_fail "Could not find non-lmaster node for key"
+fi
+
+echo "............................................................"
+echo "Delete key ${key} on node ${non_lmaster}"
+echo "Lock on lmaster node ${lmaster} during 3rd vacuuming run"
+echo "............................................................"
+
+echo
+
+echo "Create/wipe test database ${db}"
+ctdb_onnode "$first" "attach ${db}"
+ctdb_onnode "$first" "wipedb ${db}"
+
+echo "Create a record in ${db}"
+ctdb_onnode "$first" "writekey ${db} ${key} value1"
+
+echo "Migrate record to all nodes"
+ctdb_onnode all "readkey ${db} ${key}"
+
+echo "Confirm that all nodes have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+
+echo "Delete key \"${key}\" from node ${non_lmaster}"
+ctdb_onnode "$non_lmaster" "deletekey $db ${key}"
+
+echo "Do a fast vacuuming run on node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Lock record on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test local-lock ${db} ${key}"
+pid="${out#OK }"
+ctdb_test_cleanup_pid_set "$lmaster" "$pid"
+
+echo "Do a fast vacuuming run on node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Kill lock process ${pid} on node ${lmaster}"
+try_command_on_node "$lmaster" "kill ${pid}"
+ctdb_test_cleanup_pid_clear
+
+echo
+
+# If the record is still in the delete queue then this will process it
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+
+echo "Confirm the record still exists on all nodes"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+vacuum_confirm_key_empty_dmaster "$lmaster" "$db" "$key"
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.033.locked.sh b/ctdb/tests/INTEGRATION/database/vacuum.033.locked.sh
new file mode 100755
index 00000000000..47b6be2c7f2
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.033.locked.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Confirm that a record is not vacuumed if it is locked on the
+# deleting node when the 3rd fast vacuuming run occurs, but is dropped
+# from the lmaster delete list
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+db="vacuum_test.tdb"
+key="key"
+
+echo "Stall vacuuming on all nodes"
+ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+echo
+echo "Getting list of nodes..."
+ctdb_get_all_pnns
+
+# all_pnns is set above by ctdb_get_all_pnns()
+# shellcheck disable=SC2154
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+echo
+echo "Determine lmaster node for key"
+testprog_onnode "$first" "ctdb-db-test get-lmaster key"
+# $out is set above by testprog_onnode()
+# shellcheck disable=SC2154
+lmaster="$out"
+echo "lmaster=${lmaster}"
+
+non_lmaster=""
+# Find a non-lmaster node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] ; then
+		non_lmaster="$i"
+		break
+	fi
+done
+if [ -z "$non_lmaster" ] ; then
+	ctdb_test_fail "Could not find non-lmaster node for key"
+fi
+
+echo "............................................................"
+echo "Delete key ${key} on node ${non_lmaster}"
+echo "Lock on non-lmaster node ${non_lmaster} during 3rd vacuuming run"
+echo "............................................................"
+
+echo
+
+echo "Create/wipe test database ${db}"
+ctdb_onnode "$first" "attach ${db}"
+ctdb_onnode "$first" "wipedb ${db}"
+
+echo "Create a record in ${db}"
+ctdb_onnode "$first" "writekey ${db} ${key} value1"
+
+echo "Migrate record to all nodes"
+ctdb_onnode all "readkey ${db} ${key}"
+
+echo "Confirm that all nodes have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+
+echo "Delete key \"${key}\" from node ${non_lmaster}"
+ctdb_onnode "$non_lmaster" "deletekey $db ${key}"
+
+echo
+echo "Do a fast vacuuming run on node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that all nodes still have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that all nodes still have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+echo "Lock record on non-lmaster node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test local-lock ${db} ${key}"
+pid="${out#OK }"
+ctdb_test_cleanup_pid_set "$non_lmaster" "$pid"
+
+echo "Do a fast vacuuming run on node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Kill lock process ${pid} on node ${non_lmaster}"
+try_command_on_node "$non_lmaster" "kill ${pid}"
+ctdb_test_cleanup_pid_clear
+
+echo
+echo "Confirm that nodes ${lmaster} and ${non_lmaster} still have the record"
+check_cattdb_num_records "$db" 1 "${lmaster} ${non_lmaster}"
+
+vacuum_confirm_key_empty_dmaster "$lmaster" "$db" "$key"
+
+echo
+
+# Record has been dropped from the delete list so this will not pick it up
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that nodes ${lmaster} and ${non_lmaster} still have the record"
+check_cattdb_num_records "$db" 1 "${lmaster} ${non_lmaster}"
+
+vacuum_confirm_key_empty_dmaster "$lmaster" "$db" "$key"
diff --git a/ctdb/tests/INTEGRATION/database/vacuum.034.locked.sh b/ctdb/tests/INTEGRATION/database/vacuum.034.locked.sh
new file mode 100755
index 00000000000..405ddbb8034
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/database/vacuum.034.locked.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+# Confirm that a record is not vacuumed if it is locked on another
+# (non-lmaster, non-deleting) node when the 3rd fast vacuuming run
+# occurs, but is dropped from the lmaster delete tree
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+db="vacuum_test.tdb"
+key="key"
+
+echo "Stall vacuuming on all nodes"
+ctdb_onnode -p all "setvar VacuumInterval 99999"
+
+echo
+echo "Getting list of nodes..."
+ctdb_get_all_pnns
+
+# all_pnns is set above by ctdb_get_all_pnns()
+# shellcheck disable=SC2154
+first=$(echo "$all_pnns" | sed -n -e '1p')
+
+echo
+echo "Determine lmaster node for key"
+testprog_onnode "$first" "ctdb-db-test get-lmaster key"
+# $out is set above by testprog_onnode()
+# shellcheck disable=SC2154
+lmaster="$out"
+echo "lmaster=${lmaster}"
+
+non_lmaster=""
+# Find a non-lmaster node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] ; then
+		non_lmaster="$i"
+		break
+	fi
+done
+if [ -z "$non_lmaster" ] ; then
+	ctdb_test_fail "Could not find non-lmaster node for key"
+fi
+
+another_node=""
+# Find another node
+for i in $all_pnns ; do
+	if [ "$i" != "$lmaster" ] && [ "$i" != "$non_lmaster" ] ; then
+		another_node="$i"
+		break
+	fi
+done
+if [ -z "$another_node" ] ; then
+	ctdb_test_fail "Could not find another non-lmaster node for key"
+fi
+
+echo "............................................................"
+echo "Delete key ${key} on node ${non_lmaster}"
+echo "Lock on non-lmaster node ${non_lmaster} during 3rd vacuuming run"
+echo "............................................................"
+
+echo
+
+echo "Create/wipe test database ${db}"
+ctdb_onnode "$first" "attach ${db}"
+ctdb_onnode "$first" "wipedb ${db}"
+
+echo "Create a record in ${db}"
+ctdb_onnode "$first" "writekey ${db} ${key} value1"
+
+echo "Migrate record to all nodes"
+ctdb_onnode all "readkey ${db} ${key}"
+
+echo "Confirm that all nodes have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+
+echo "Delete key \"${key}\" from node ${non_lmaster}"
+ctdb_onnode "$non_lmaster" "deletekey $db ${key}"
+
+echo
+echo "Do a fast vacuuming run on node ${non_lmaster}"
+testprog_onnode "$non_lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that all nodes still have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that all nodes still have the record"
+check_cattdb_num_records "$db" 1 "$all_pnns"
+
+echo
+echo "Lock record on non-lmaster node ${another_node}"
+testprog_onnode "$another_node" "ctdb-db-test local-lock ${db} ${key}"
+pid="${out#OK }"
+ctdb_test_cleanup_pid_set "$another_node" "$pid"
+
+echo "Do a fast vacuuming run on node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo "Kill lock process ${pid} on node ${another_node}"
+try_command_on_node "$another_node" "kill ${pid}"
+ctdb_test_cleanup_pid_clear
+
+echo
+echo "Confirm that nodes ${lmaster} and ${another_node} still have the record"
+check_cattdb_num_records "$db" 1 "${lmaster} ${another_node}"
+
+vacuum_confirm_key_empty_dmaster "$lmaster" "$db" "$key"
+
+echo
+
+# Record has been dropped from the delete list so this will not pick it up
+echo "Do a fast vacuuming run on lmaster node ${lmaster}"
+testprog_onnode "$lmaster" "ctdb-db-test vacuum ${db}"
+
+echo
+echo "Confirm that nodes ${lmaster} and ${another_node} still have the record"
+check_cattdb_num_records "$db" 1 "${lmaster} ${another_node}"
+
+vacuum_confirm_key_empty_dmaster "$lmaster" "$db" "$key"