From e113a61bc0537e485fbe7769f1ef0ebf02ac5e00 Mon Sep 17 00:00:00 2001 From: "Frank Ch. Eigler" Date: Wed, 16 Jun 2021 18:49:10 -0400 Subject: debuginfod test: fix groom/stale race condition Additional tracing, and use of "% make check VERBOSE=1" in a .spec file allowed tracking down of this intermittent problem. The race was between a SIGUSR1 or two to a debuginfod server (triggering two traverse/scan phases), followed shortly by a SIGUSR2 (triggering a groom). If those signals were received too close together, the groom phase could be stopped early, and the rm'd files not noticed. New testsuite code adds metric polls after SIGUSR1 & SIGUSR2 to ensure the respective processing phases are complete. It also turns on "set -x" tracing, so as to avoid pulling out quite as much hair next time. "make check VERBOSE=1" is also important for spec files. Signed-off-by: Frank Ch. Eigler --- tests/ChangeLog | 5 +++++ tests/run-debuginfod-find.sh | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/ChangeLog b/tests/ChangeLog index c02fb050..d8fa97fa 100644 --- a/tests/ChangeLog +++ b/tests/ChangeLog @@ -1,3 +1,8 @@ +2021-06-16 Frank Ch. Eigler + + * run-debuginfod-find.sh: Fix intermittent groom/stale failure, + due to SIGUSR1/SIGUSR2 races. Trace more. + 2021-06-15 Frank Ch. Eigler * run-debuginfod-find.sh (err): Elaborate. Use as the reliable diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh index c9ee11b1..456dc2f8 100755 --- a/tests/run-debuginfod-find.sh +++ b/tests/run-debuginfod-find.sh @@ -26,7 +26,7 @@ bsdtar --version | grep -q zstd && zstd=true || zstd=false echo "zstd=$zstd bsdtar=`bsdtar --version`" # for test case debugging, uncomment: -#set -x +set -x VERBOSE=-vvv DB=${PWD}/.debuginfod_tmp.sqlite @@ -74,7 +74,6 @@ errfiles() { } - # find an unused port number while true; do PORT1=`expr '(' $RANDOM % 1000 ')' + 9000` @@ -317,6 +316,11 @@ fi cp -rvp ${abs_srcdir}/debuginfod-tars Z kill -USR1 $PID1 +# Wait till both files are in the index and scan/index fully finished +wait_ready $PORT1 'thread_work_total{role="traverse"}' 4 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 + # All rpms need to be in the index, except the dummy permission-000 one rpms=$(find R -name \*rpm | grep -v nothing | wc -l) wait_ready $PORT1 'scanned_files_total{source=".rpm archive"}' $rpms @@ -326,6 +330,11 @@ tb2=$(find Z -name \*tar.bz2 | wc -l) wait_ready $PORT1 'scanned_files_total{source=".tar.bz2 archive"}' $tb2 kill -USR1 $PID1 # two hits of SIGUSR1 may be needed to resolve .debug->dwz->srefs +# Wait till both files are in the index and scan/index fully finished +wait_ready $PORT1 'thread_work_total{role="traverse"}' 5 +wait_ready $PORT1 'thread_work_pending{role="scan"}' 0 +wait_ready $PORT1 'thread_busy{role="scan"}' 0 + # Expect all source files found in the rpms (they are all called hello.c :) # We will need to extract all rpms (in their own directory) and could all # sources referenced in the .debug files. @@ -495,6 +504,10 @@ if type bsdtar 2>/dev/null; then # copy in the deb files cp -rvp ${abs_srcdir}/debuginfod-debs/*deb D kill -USR1 $PID2 + wait_ready $PORT2 'thread_work_total{role="traverse"}' 2 + wait_ready $PORT2 'thread_work_pending{role="scan"}' 0 + wait_ready $PORT2 'thread_busy{role="scan"}' 0 + # All debs need to be in the index debs=$(find D -name \*.deb | wc -l) wait_ready $PORT2 'scanned_files_total{source=".deb archive"}' `expr $debs` -- cgit v1.2.1