ctdb-tests: Add debug_locks.sh testing

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

14 files changed, 405 insertions, 1 deletions

diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.001.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.001.sh
new file mode 100755
index 00000000000..8f102001a97
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.001.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "DB S+ DB"
+
+setup
+
+do_test "DB" "S+" "DB"
diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.002.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.002.sh
new file mode 100755
index 00000000000..31ae3dff1f6
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.002.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "DB D. DB"
+
+setup
+
+do_test "DB" "D." "DB"
diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.003.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.003.sh
new file mode 100755
index 00000000000..89ab2f1bb58
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.003.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "RECORD S+ DB"
+
+setup
+
+do_test "RECORD" "S+" "DB"
diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.004.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.004.sh
new file mode 100755
index 00000000000..35500cbd04c
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.004.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "RECORD D. DB"
+
+setup
+
+do_test "RECORD" "D." "DB"
diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.005.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.005.sh
new file mode 100755
index 00000000000..10cbade46a5
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.005.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "DB S+ RECORD"
+
+setup
+
+do_test "DB" "S+" "RECORD"
diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.006.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.006.sh
new file mode 100755
index 00000000000..c4988b7e26d
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.006.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "DB D. RECORD"
+
+setup
+
+do_test "DB" "D." "RECORD"
diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.007.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.007.sh
new file mode 100755
index 00000000000..b186d2037a5
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.007.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "RECORD S+ RECORD"
+
+setup
+
+do_test "RECORD" "S+" "RECORD"
diff --git a/ctdb/tests/UNIT/eventscripts/debug_locks.sh.008.sh b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.008.sh
new file mode 100755
index 00000000000..7b7ac9be8d4
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/debug_locks.sh.008.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "RECORD D. RECORD"
+
+setup
+
+do_test "RECORD" "D." "RECORD"
diff --git a/ctdb/tests/UNIT/eventscripts/etc-ctdb/rc.local b/ctdb/tests/UNIT/eventscripts/etc-ctdb/rc.local
index 72a79bc66ac..e9a7f99829a 100755
--- a/ctdb/tests/UNIT/eventscripts/etc-ctdb/rc.local
+++ b/ctdb/tests/UNIT/eventscripts/etc-ctdb/rc.local
@@ -46,6 +46,9 @@ get_proc ()
 	meminfo)
 	    echo "$FAKE_PROC_MEMINFO"
 	    ;;
+	locks)
+	    echo "$FAKE_PROC_LOCKS"
+	    ;;
 	*)
 	    echo "get_proc: \"$1\" not implemented"
 	    exit 1
@@ -58,4 +61,9 @@ background_with_logging ()
     "$@" 2>&1 </dev/null | sed -e 's@^@\&@'
 }
 
+script_log ()
+{
+	cat
+}
+
 CTDB_INIT_STYLE="${EVENTSCRIPT_TESTS_INIT_STYLE:-redhat}"
diff --git a/ctdb/tests/UNIT/eventscripts/scripts/debug_locks.sh b/ctdb/tests/UNIT/eventscripts/scripts/debug_locks.sh
new file mode 100644
index 00000000000..1a3b83c1757
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/scripts/debug_locks.sh
@@ -0,0 +1,210 @@
+setup ()
+{
+	setup_dbdir
+}
+
+result_filter()
+{
+	sed -e 's|\( of debug locks PID=\)[0-9]*|\1PID|'
+}
+
+tdb_path ()
+{
+	echo "${CTDB_DBDIR}/${1}.${FAKE_CTDB_PNN}"
+}
+
+fake_file_id ()
+{
+	_path="$1"
+
+	echo "$FAKE_FILE_ID_MAP" |
+	awk -v path="$_path" '$1 == path { print $2 }'
+}
+
+fake_stack_trace ()
+{
+	_pid="$1"
+	_command="${2:-smbd}"
+	_state="$3"
+
+	echo "----- Stack trace for PID=${_pid} -----"
+
+	case "$_state" in
+	D*)
+		cat <<EOF
+----- Process in D state, printing kernel stack only
+[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff
+EOF
+		;;
+	*)
+		cat <<EOF
+Thread 1 (Thread 0x7f688fbfb180 (LWP ${_pid}) "${_command}"):
+#0  0x00007f688ff7a076 in open (FAKE ARGS...) at FAKE PLACE
+....
+#3  0x000055cd368ead72 in main (argc=<optimized out>, argv=<optimized out>) at ${_command}.c
+EOF
+		;;
+	esac
+}
+
+do_test ()
+{
+	_holder_scope="$1"
+	_holder_state="$2"
+	_helper_scope="$3"
+
+	FAKE_PS_MAP=$(cat <<EOF
+1234567 ctdbd S
+2345678 smbd S
+4131931 smbd ${_holder_state}
+4132032 ctdb_lock_helpe S+
+EOF
+		   )
+	export FAKE_PS_MAP
+
+	FAKE_FILE_ID_MAP=""
+	_tdbs="locking.tdb brlock.tdb test.tdb foo.tdb"
+	_n=1
+	for _t in $_tdbs ; do
+		_path=$(tdb_path "$_t")
+		_inode=$((19690818 + _n))
+		FAKE_FILE_ID_MAP=$(cat <<EOF
+${FAKE_FILE_ID_MAP}
+${_path} 103:04:${_inode}
+EOF
+				)
+		rm -f "$_path"
+		touch "$_path"
+		_n=$((_n + 1))
+	done
+	export FAKE_FILE_ID_MAP
+
+	_path=$(tdb_path "locking.tdb")
+	_locking_tdb_id=$(fake_file_id "$_path")
+
+	_t=$(cat <<EOF
+POSIX  ADVISORY  WRITE 3769740 103:04:24380821 1073741826 1073742335
+FLOCK  ADVISORY  WRITE 3632524 103:02:1059266 0 EOF
+FLOCK  ADVISORY  WRITE 4060231 00:17:17184 0 EOF
+POSIX  ADVISORY  READ 1234567 ${_locking_tdb_id} 4 4
+POSIX  ADVISORY  WRITE 59178 103:04:24380821 1073741826 1073742335
+POSIX  ADVISORY  READ 4427 103:04:22152234 1073741826 1073742335
+POSIX  ADVISORY  WRITE 4427 103:04:22152494 0 EOF
+POSIX  ADVISORY  READ 4427 103:04:22152702 1073741826 1073742335
+EOF
+	     )
+
+	_holder_lock=""
+	if [ "$_holder_scope" = "DB" ] ; then
+		_holder_lock=$(cat <<EOF
+POSIX  ADVISORY  WRITE 4131931 ${_locking_tdb_id} 168 EOF
+EOF
+		  )
+	elif [ "$_holder_scope" = "RECORD" ] ; then
+		_holder_lock=$(cat <<EOF
+POSIX  ADVISORY  WRITE 2345678 ${_locking_tdb_id} 112736 112736
+POSIX  ADVISORY  WRITE 4131931 ${_locking_tdb_id} 225472 225472
+EOF
+		  )
+	fi
+
+	_t=$(cat <<EOF
+$_t
+$_holder_lock
+EOF
+	  )
+
+	_helper_lock=""
+	if [ "$_helper_scope" = "DB" ] ; then
+		_helper_lock=$(cat <<EOF
+-> POSIX  ADVISORY  WRITE 4132032 ${_locking_tdb_id} 168 170
+EOF
+			    )
+	elif [ "$_helper_scope" = "RECORD" ] ; then
+		_helper_lock=$(cat <<EOF
+-> POSIX  ADVISORY  WRITE 4132032 ${_locking_tdb_id} 112736 112736
+EOF
+			    )
+	fi
+		_t=$(cat <<EOF
+$_t
+$_helper_lock
+EOF
+		  )
+
+	if [ "$_holder_scope" = "DB" ] ; then
+		_t=$(cat <<EOF
+$_t
+POSIX  ADVISORY  READ 4131931 ${_locking_tdb_id} 4 4
+EOF
+		  )
+	elif [ "$_holder_scope" = "RECORD" ] ; then
+		_t=$(cat <<EOF
+$_t
+POSIX  ADVISORY  READ 2345678 ${_locking_tdb_id} 4 4
+POSIX  ADVISORY  READ 4131931 ${_locking_tdb_id} 4 4
+EOF
+		  )
+	fi
+
+		_t=$(cat <<EOF
+$_t
+POSIX  ADVISORY  READ 3769740 103:04:24390149 1073741826 1073742335
+POSIX  ADVISORY  WRITE 3769740 103:04:24380839 1073741826 1073742335
+FLOCK  ADVISORY  WRITE 3769302 103:02:1180313 0 EOF
+FLOCK  ADVISORY  WRITE 3769302 103:02:1177487 0 EOF
+FLOCK  ADVISORY  WRITE 3769302 103:02:1180308 0 EOF
+OFDLCK ADVISORY  READ -1 00:05:6 0 EOF
+EOF
+		  )
+
+	FAKE_PROC_LOCKS=$(echo "$_t" | awk '{ printf "%d: %s\n", NR, $0 }')
+	export FAKE_PROC_LOCKS
+
+	_out=''
+	_nl='
+'
+	_db="locking.tdb.${FAKE_CTDB_PNN}"
+
+	_pids=''
+	# fake lock info
+	while read -r _ _ _  _pid _ _start _end ; do
+		_comm="smbd"
+		_out="${_out:+${_out}${_nl}}"
+		_out="${_out}${_pid} smbd ${_db} ${_start} ${_end}"
+		_pids="${_pids:+${_pids} }${_pid}"
+	done <<EOF
+$_holder_lock
+EOF
+
+	if [ -n "$_helper_lock" ] ; then
+		read -r _ _ _ _ _pid _ _start _end <<EOF
+$_helper_lock
+EOF
+		_out="${_out}${_nl}"
+		_out="${_out}${_pid} ctdb_lock_helpe ${_db} ${_start} ${_end} W"
+	fi
+
+	# fake stack traces
+	for _pid in $_pids ; do
+		_comm="smbd"
+		if [ "$_pid" = "4131931" ] ; then
+			_state="$_holder_state"
+		else
+			_state="S"
+		fi
+		_out=$(cat <<EOF
+$_out
+$(fake_stack_trace "$_pid" "$_comm" "$_state")
+EOF
+		    )
+	done
+
+	ok <<EOF
+===== Start of debug locks PID=PID =====
+$_out
+===== End of debug locks PID=PID =====
+EOF
+
+	script_test "${script_dir}/${script}"
+}
diff --git a/ctdb/tests/UNIT/eventscripts/scripts/local.sh b/ctdb/tests/UNIT/eventscripts/scripts/local.sh
index 14017e95365..0f78fcb3845 100644
--- a/ctdb/tests/UNIT/eventscripts/scripts/local.sh
+++ b/ctdb/tests/UNIT/eventscripts/scripts/local.sh
@@ -36,6 +36,7 @@ else
 fi
 
 setup_ctdb_base "$CTDB_TEST_TMP_DIR" "etc-ctdb" \
+		debug_locks.sh \
 		functions \
 		nfs-checks.d \
 		nfs-linux-kernel-callout \
@@ -475,6 +476,7 @@ define_test ()
 		;;
 	*)
 		script="${_f%.*}"
+		script="$_f"
 		unset event
 		script_dir="${CTDB_BASE}"
 	esac
@@ -483,7 +485,11 @@ define_test ()
 	[ -r "$_s" ] || \
 		die "Internal error - unable to find script \"${_s}\""
 
-	script_short="${script%.script}"
+	case "$script" in
+	*.script) script_short="${script%.script}" ;;
+	*.sh) script_short="${script%.sh}" ;;
+	*) script_short="$script" ;;
+	esac
 
 	printf "%-17s %-10s %-4s - %s\n\n" \
 	       "$script_short" "$event" "$_num" "$desc"
diff --git a/ctdb/tests/UNIT/eventscripts/stubs/gstack b/ctdb/tests/UNIT/eventscripts/stubs/gstack
new file mode 100755
index 00000000000..9ee41cca65a
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/stubs/gstack
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+pid="$1"
+
+if [ -n "$FAKE_PS_MAP" ] ; then
+	command=$(echo "$FAKE_PS_MAP" |
+		  awk -v pid="$pid" '$1 == pid { print $2 }')
+fi
+
+if [ -z "$command" ] ; then
+	command="smbd"
+fi
+
+cat <<EOF
+Thread 1 (Thread 0x7f688fbfb180 (LWP ${pid}) "${command}"):
+#0  0x00007f688ff7a076 in open (FAKE ARGS...) at FAKE PLACE
+....
+#3  0x000055cd368ead72 in main (argc=<optimized out>, argv=<optimized out>) at ${command}.c
+EOF
diff --git a/ctdb/tests/UNIT/eventscripts/stubs/ps b/ctdb/tests/UNIT/eventscripts/stubs/ps
index f8e0ae0ba8a..744ba3becd8 100755
--- a/ctdb/tests/UNIT/eventscripts/stubs/ps
+++ b/ctdb/tests/UNIT/eventscripts/stubs/ps
@@ -1,5 +1,41 @@
 #!/bin/sh
 
+usage ()
+{
+	echo "ps [ -p PID | -o FORMAT | aufxww ]"
+	exit 1
+}
+
+while getopts "o:p:h:?" opt ; do
+	case "$opt" in
+	o) format="$OPTARG" ;;
+	p) pid="$OPTARG" ;;
+	\?|h) usage ;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if [ -n "$pid" ] && [ -n "$FAKE_PS_MAP" ] ; then
+	# shellcheck disable=SC1001
+	case "$format" in
+	comm\=)
+		echo "$FAKE_PS_MAP" |
+		awk -v pid="$pid" '$1 == pid { print $2 }'
+		;;
+	state\=)
+		echo "$FAKE_PS_MAP" |
+		awk -v pid="$pid" '$1 == pid { print $3 }'
+		;;
+	esac
+
+	exit
+fi
+
+if [ "$1" != "auxfww" ] ; then
+	echo "option $1 not supported"
+	usage
+fi
+
 cat <<EOF
 USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
 root         2  0.0  0.0      0     0 ?        S    Aug28   0:00 [kthreadd]
diff --git a/ctdb/tests/UNIT/eventscripts/stubs/stat b/ctdb/tests/UNIT/eventscripts/stubs/stat
new file mode 100755
index 00000000000..04c654883d2
--- /dev/null
+++ b/ctdb/tests/UNIT/eventscripts/stubs/stat
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+usage ()
+{
+	echo "stat -c FMT FILE ..."
+	exit 1
+}
+
+format=""
+
+while getopts "c:h:?" opt ; do
+	case "$opt" in
+	c) format="$OPTARG" ;;
+	\?|h) usage ;;
+	esac
+done
+shift $((OPTIND - 1))
+
+fake_inode ()
+{
+	_path="$1"
+
+	_t=$(echo "$FAKE_FILE_ID_MAP" |
+	     awk -v path="${_path}" '$1 == path { print $2 }')
+	echo "${_t##*:}"
+}
+
+if [ -n "$format" ] ; then
+	for f ; do
+		if [ ! -e "$f" ] ; then
+			continue
+		fi
+		case "$f" in
+		/*) path="$f" ;;
+		*) path="${PWD}/${f}" ;;
+		esac
+
+		case "$format" in
+		"s#[0-9a-f]*:[0-9a-f]*:%i #%n #")
+			inode=$(fake_inode "$path")
+			echo "s#[0-9a-f]*:[0-9a-f]*:${inode} #${f} #"
+			;;
+		*)
+			echo "Unsupported format \"${format}\""
+			usage
+			;;
+		esac
+	done
+
+	exit
+fi
+
+usage