From a57397b0d6709d75ab23145fd4fea5ce65dc1054 Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Mon, 17 Jun 2013 11:18:46 +0200 Subject: test-lib: enable MALLOC_* for the actual tests 1b3185f (MALLOC_CHECK: various clean-ups, 2012-09-14) moved around the MALLOC_CHECK_ and MALLOC_PERTURB_ assignments, intending to limit their effect to only the test runs. However, they were actually enabled only during test cleanup. Call setup/teardown_malloc_check also around the evaluation of the actual test snippet. Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index ca6bdef63d..229f5f7ff1 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -337,8 +337,10 @@ test_eval_ () { test_run_ () { test_cleanup=: expecting_failure=$2 + setup_malloc_check test_eval_ "$1" eval_ret=$? + teardown_malloc_check if test -z "$immediate" || test $eval_ret = 0 || test -n "$expecting_failure" then -- cgit v1.2.1 From e6a6ddc93a89862319c7876536d79d6d3538de5a Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Tue, 18 Jun 2013 14:25:58 +0200 Subject: test-lib: refactor $GIT_SKIP_TESTS matching It's already used twice, and we will have more of the same kind of matching in a minute. Helped-by: Johannes Sixt Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index 229f5f7ff1..4c583b033d 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -328,6 +328,20 @@ test_debug () { test "$debug" = "" || eval "$1" } +match_pattern_list () { + arg="$1" + shift + test -z "$*" && return 1 + for pattern_ + do + case "$arg" in + $pattern_) + return 0 + esac + done + return 1 +} + test_eval_ () { # This is a separate function because some tests use # "return" to end a test_expect_success block early. @@ -358,14 +372,10 @@ test_run_ () { test_skip () { test_count=$(($test_count+1)) to_skip= - for skp in $GIT_SKIP_TESTS - do - case $this_test.$test_count in - $skp) - to_skip=t - break - esac - done + if match_pattern_list $this_test.$test_count $GIT_SKIP_TESTS + then + to_skip=t + fi if test -z "$to_skip" && test -n "$test_prereq" && ! test_have_prereq "$test_prereq" then @@ -630,15 +640,12 @@ cd -P "$TRASH_DIRECTORY" || exit 1 this_test=${0##*/} this_test=${this_test%%-*} -for skp in $GIT_SKIP_TESTS -do - case "$this_test" in - $skp) - say_color info >&3 "skipping test $this_test altogether" - skip_all="skip all tests in $this_test" - test_done - esac -done +if match_pattern_list "$this_test" $GIT_SKIP_TESTS +then + say_color info >&3 "skipping test $this_test altogether" + skip_all="skip all tests in $this_test" + test_done +fi # Provide an implementation of the 'yes' utility yes () { -- cgit v1.2.1 From ae75342cff145a326b2213128cca453023521400 Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Tue, 18 Jun 2013 14:25:59 +0200 Subject: test-lib: rearrange start/end of test_expect_* and test_skip This moves * the early setup part from test_skip to a new function test_start_ * the final common parts of test_expect_* to a new function test_finish_ to make the next commit more obvious. Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index 4c583b033d..0985ca5eb7 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -369,8 +369,15 @@ test_run_ () { return "$eval_ret" } -test_skip () { +test_start_ () { test_count=$(($test_count+1)) +} + +test_finish_ () { + echo >&3 "" +} + +test_skip () { to_skip= if match_pattern_list $this_test.$test_count $GIT_SKIP_TESTS then -- cgit v1.2.1 From 517cd55fd51ebf4c6c20597055ff191fefe5dda0 Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Sun, 23 Jun 2013 20:12:55 +0200 Subject: test-lib: self-test that --verbose works t0000 contains some light self-tests of test-lib.sh, but --verbose was not covered. Add a test. The only catch is that the presence of a test harness influences the output (specifically, the presence of some empty lines). So we need to unset TEST_HARNESS or set it to a known value. Leaving it unset leads to spurious test failures in the final summary, which come from the subtest. So we always set it. Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index 0985ca5eb7..432248dfb8 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -414,6 +414,8 @@ test_at_end_hook_ () { test_done () { GIT_EXIT_OK=t + # Note: t0000 relies on $HARNESS_ACTIVE disabling the .counts + # output file if test -z "$HARNESS_ACTIVE" then test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results" -- cgit v1.2.1 From ff09af3fb8f5bede0de523723bbd00d6ef2ab71e Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Sun, 23 Jun 2013 20:12:56 +0200 Subject: test-lib: verbose mode for only tests matching a pattern With the new --verbose-only= option, one can enable --verbose at a per-test granularity. The pattern is matched against the test number, e.g. ./t0000-basic.sh --verbose-only='2[0-2]' to see only the full output of test 20-22, while showing the rest in the one-liner format. As suggested by Jeff King, this takes care to wrap the entire test_expect_* block, but nothing else, in the verbose toggling. We can use the test_start/end functions from the previous commit for the purpose. This is arguably not *too* useful on its own, but makes the next patch easier to follow. Helped-by: Jeff King Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index 432248dfb8..95df832ed9 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -184,6 +184,9 @@ do help=t; shift ;; -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose) verbose=t; shift ;; + --verbose-only=*) + verbose_only=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; -q|--q|--qu|--qui|--quie|--quiet) # Ignore --quiet under a TAP::Harness. Saying how many tests # passed without the ok/not ok details is always an error. @@ -342,6 +345,32 @@ match_pattern_list () { return 1 } +maybe_teardown_verbose () { + test -z "$verbose_only" && return + exec 4>/dev/null 3>/dev/null + verbose= +} + +last_verbose=t +maybe_setup_verbose () { + test -z "$verbose_only" && return + if match_pattern_list $test_count $verbose_only + then + exec 4>&2 3>&1 + # Emit a delimiting blank line when going from + # non-verbose to verbose. Within verbose mode the + # delimiter is printed by test_expect_*. The choice + # of the initial $last_verbose is such that before + # test 1, we do not print it. + test -z "$last_verbose" && echo >&3 "" + verbose=t + else + exec 4>/dev/null 3>/dev/null + verbose= + fi + last_verbose=$verbose +} + test_eval_ () { # This is a separate function because some tests use # "return" to end a test_expect_success block early. @@ -371,10 +400,12 @@ test_run_ () { test_start_ () { test_count=$(($test_count+1)) + maybe_setup_verbose } test_finish_ () { echo >&3 "" + maybe_teardown_verbose } test_skip () { -- cgit v1.2.1 From 5dfc368f5ec4c448cf3c76908b4c0e77cb4b902a Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Sun, 23 Jun 2013 20:12:57 +0200 Subject: test-lib: valgrind for only tests matching a pattern With the new --valgrind-only= option, one can enable --valgrind at a per-test granularity, exactly analogous to --verbose-only from the previous commit. The options are wired such that --valgrind implies --verbose (as before), but --valgrind-only= implies --verbose-only= unless --verbose is also in effect. Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index 95df832ed9..e9f3294bc8 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -201,6 +201,9 @@ do --valgrind=*) valgrind=$(expr "z$1" : 'z[^=]*=\(.*\)') shift ;; + --valgrind-only=*) + valgrind_only=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; --tee) shift ;; # was handled already --root=*) @@ -211,7 +214,14 @@ do esac done -test -n "$valgrind" && verbose=t +if test -n "$valgrind_only" +then + test -z "$valgrind" && valgrind=memcheck + test -z "$verbose" && verbose_only="$valgrind_only" +elif test -n "$valgrind" +then + verbose=t +fi if test -n "$color" then @@ -371,6 +381,25 @@ maybe_setup_verbose () { last_verbose=$verbose } +maybe_teardown_valgrind () { + test -z "$GIT_VALGRIND" && return + GIT_VALGRIND_ENABLED= +} + +maybe_setup_valgrind () { + test -z "$GIT_VALGRIND" && return + if test -z "$valgrind_only" + then + GIT_VALGRIND_ENABLED=t + return + fi + GIT_VALGRIND_ENABLED= + if match_pattern_list $test_count $valgrind_only + then + GIT_VALGRIND_ENABLED=t + fi +} + test_eval_ () { # This is a separate function because some tests use # "return" to end a test_expect_success block early. @@ -401,10 +430,12 @@ test_run_ () { test_start_ () { test_count=$(($test_count+1)) maybe_setup_verbose + maybe_setup_valgrind } test_finish_ () { echo >&3 "" + maybe_teardown_valgrind maybe_teardown_verbose } @@ -590,6 +621,9 @@ then export GIT_VALGRIND GIT_VALGRIND_MODE="$valgrind" export GIT_VALGRIND_MODE + GIT_VALGRIND_ENABLED=t + test -n "$valgrind_only" && GIT_VALGRIND_ENABLED= + export GIT_VALGRIND_ENABLED elif test -n "$GIT_TEST_INSTALLED" then GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path) || -- cgit v1.2.1 From e939e15d241e942662b9f88f6127ab470ab0a0b9 Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Sun, 23 Jun 2013 20:12:58 +0200 Subject: test-lib: allow prefixing a custom string before "ok N" etc. This is not really meant for external use, and thus not documented. It allows the next commit to neatly distinguish between sub-tests and the main run. The format is intentionally not valid TAP. The use in the next commit would not result in anything valid either way, and it seems better to make it obvious. Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index e9f3294bc8..8944e70e5c 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -209,6 +209,9 @@ do --root=*) root=$(expr "z$1" : 'z[^=]*=\(.*\)') shift ;; + --statusprefix=*) + statusprefix=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; *) echo "error: unknown test option '$1'" >&2; exit 1 ;; esac @@ -316,12 +319,12 @@ trap 'die' EXIT test_ok_ () { test_success=$(($test_success + 1)) - say_color "" "ok $test_count - $@" + say_color "" "${statusprefix}ok $test_count - $@" } test_failure_ () { test_failure=$(($test_failure + 1)) - say_color error "not ok $test_count - $1" + say_color error "${statusprefix}not ok $test_count - $1" shift echo "$@" | sed -e 's/^/# /' test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; } @@ -329,12 +332,12 @@ test_failure_ () { test_known_broken_ok_ () { test_fixed=$(($test_fixed+1)) - say_color error "ok $test_count - $@ # TODO known breakage vanished" + say_color error "${statusprefix}ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { test_broken=$(($test_broken+1)) - say_color warn "not ok $test_count - $@ # TODO known breakage" + say_color warn "${statusprefix}not ok $test_count - $@ # TODO known breakage" } test_debug () { @@ -458,8 +461,8 @@ test_skip () { of_prereq=" of $test_prereq" fi - say_color skip >&3 "skipping test: $@" - say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" + say_color skip >&3 "${statusprefix}skipping test: $@" + say_color skip "${statusprefix}ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" : true ;; *) @@ -497,11 +500,11 @@ test_done () { if test "$test_fixed" != 0 then - say_color error "# $test_fixed known breakage(s) vanished; please update test(s)" + say_color error "${statusprefix}# $test_fixed known breakage(s) vanished; please update test(s)" fi if test "$test_broken" != 0 then - say_color warn "# still have $test_broken known breakage(s)" + say_color warn "${statusprefix}# still have $test_broken known breakage(s)" fi if test "$test_broken" != 0 || test "$test_fixed" != 0 then @@ -524,9 +527,9 @@ test_done () { then if test $test_remaining -gt 0 then - say_color pass "# passed all $msg" + say_color pass "${statusprefix}# passed all $msg" fi - say "1..$test_count$skip_all" + say "${statusprefix}1..$test_count$skip_all" fi test -d "$remove_trash" && @@ -540,8 +543,8 @@ test_done () { *) if test $test_external_has_tap -eq 0 then - say_color error "# failed $test_failure among $msg" - say "1..$test_count" + say_color error "${statusprefix}# failed $test_failure among $msg" + say "${statusprefix}1..$test_count" fi exit 1 ;; -- cgit v1.2.1 From ad0e6233320b004f0d686f6887c803e508607bd2 Mon Sep 17 00:00:00 2001 From: Thomas Rast Date: Sun, 23 Jun 2013 20:12:59 +0200 Subject: test-lib: support running tests under valgrind in parallel With the new --valgrind-parallel= option, we support running the tests in a single test script under valgrind in parallel using 'n' processes. This really follows the dumbest approach possible, as follows: * We spawn the test script 'n' times, using a throw-away TEST_OUTPUT_DIRECTORY. Each of the instances is given options that ensures that it only runs every n-th test under valgrind, but together they cover the entire range. * We add up the numbers from the individual tests, and provide the usual output. This is really a gross hack at this point, and should be improved. In particular we should keep the actual outputs somewhere more easily discoverable, and summarize them to the user. Nevertheless, this is already workable and gives a speedup of more than 2 on a dual-core (hyperthreaded) machine, using n=4. This is expected since the overhead of valgrind is so big (on the order of 20x under good conditions, and a large startup overhead at every git invocation) that redundantly running the non-valgrind tests in between is not that expensive. Helped-by: Jeff King Signed-off-by: Thomas Rast Signed-off-by: Junio C Hamano --- t/test-lib.sh | 106 ++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 22 deletions(-) (limited to 't/test-lib.sh') diff --git a/t/test-lib.sh b/t/test-lib.sh index 8944e70e5c..5b5ea6d386 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -204,6 +204,15 @@ do --valgrind-only=*) valgrind_only=$(expr "z$1" : 'z[^=]*=\(.*\)') shift ;; + --valgrind-parallel=*) + valgrind_parallel=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; + --valgrind-only-stride=*) + valgrind_only_stride=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; + --valgrind-only-offset=*) + valgrind_only_offset=$(expr "z$1" : 'z[^=]*=\(.*\)') + shift ;; --tee) shift ;; # was handled already --root=*) @@ -217,7 +226,7 @@ do esac done -if test -n "$valgrind_only" +if test -n "$valgrind_only" || test -n "$valgrind_only_stride" then test -z "$valgrind" && valgrind=memcheck test -z "$verbose" && verbose_only="$valgrind_only" @@ -367,7 +376,9 @@ maybe_teardown_verbose () { last_verbose=t maybe_setup_verbose () { test -z "$verbose_only" && return - if match_pattern_list $test_count $verbose_only + if match_pattern_list $test_count $verbose_only || + { test -n "$valgrind_only_stride" && + expr $test_count "%" $valgrind_only_stride - $valgrind_only_offset = 0 >/dev/null; } then exec 4>&2 3>&1 # Emit a delimiting blank line when going from @@ -391,13 +402,17 @@ maybe_teardown_valgrind () { maybe_setup_valgrind () { test -z "$GIT_VALGRIND" && return - if test -z "$valgrind_only" + if test -z "$valgrind_only" && test -z "$valgrind_only_stride" then GIT_VALGRIND_ENABLED=t return fi GIT_VALGRIND_ENABLED= if match_pattern_list $test_count $valgrind_only + then + GIT_VALGRIND_ENABLED=t + elif test -n "$valgrind_only_stride" && + expr $test_count "%" $valgrind_only_stride - $valgrind_only_offset = 0 >/dev/null then GIT_VALGRIND_ENABLED=t fi @@ -552,6 +567,9 @@ test_done () { esac } + +# Set up a directory that we can put in PATH which redirects all git +# calls to 'valgrind git ...'. if test -n "$valgrind" then make_symlink () { @@ -599,33 +617,42 @@ then make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit } - # override all git executables in TEST_DIRECTORY/.. - GIT_VALGRIND=$TEST_DIRECTORY/valgrind - mkdir -p "$GIT_VALGRIND"/bin - for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/test-* - do - make_valgrind_symlink $file - done - # special-case the mergetools loadables - make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools" - OLDIFS=$IFS - IFS=: - for path in $PATH - do - ls "$path"/git-* 2> /dev/null | - while read file + # In the case of --valgrind-parallel, we only need to do the + # wrapping once, in the main script. The worker children all + # have $valgrind_only_stride set, so we can skip based on that. + if test -z "$valgrind_only_stride" + then + # override all git executables in TEST_DIRECTORY/.. + GIT_VALGRIND=$TEST_DIRECTORY/valgrind + mkdir -p "$GIT_VALGRIND"/bin + for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/test-* do - make_valgrind_symlink "$file" + make_valgrind_symlink $file done - done - IFS=$OLDIFS + # special-case the mergetools loadables + make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools" + OLDIFS=$IFS + IFS=: + for path in $PATH + do + ls "$path"/git-* 2> /dev/null | + while read file + do + make_valgrind_symlink "$file" + done + done + IFS=$OLDIFS + fi PATH=$GIT_VALGRIND/bin:$PATH GIT_EXEC_PATH=$GIT_VALGRIND/bin export GIT_VALGRIND GIT_VALGRIND_MODE="$valgrind" export GIT_VALGRIND_MODE GIT_VALGRIND_ENABLED=t - test -n "$valgrind_only" && GIT_VALGRIND_ENABLED= + if test -n "$valgrind_only" || test -n "$valgrind_only_stride" + then + GIT_VALGRIND_ENABLED= + fi export GIT_VALGRIND_ENABLED elif test -n "$GIT_TEST_INSTALLED" then @@ -711,6 +738,41 @@ then else mkdir -p "$TRASH_DIRECTORY" fi + +# Gross hack to spawn N sub-instances of the tests in parallel, and +# summarize the results. Note that if this is enabled, the script +# terminates at the end of this 'if' block. +if test -n "$valgrind_parallel" +then + for i in $(test_seq 1 $valgrind_parallel) + do + root="$TRASH_DIRECTORY/vgparallel-$i" + mkdir "$root" + TEST_OUTPUT_DIRECTORY="$root" \ + ${SHELL_PATH} "$0" \ + --root="$root" --statusprefix="[$i] " \ + --valgrind="$valgrind" \ + --valgrind-only-stride="$valgrind_parallel" \ + --valgrind-only-offset="$i" & + pids="$pids $!" + done + trap "kill $pids" INT TERM HUP + wait $pids + trap - INT TERM HUP + for i in $(test_seq 1 $valgrind_parallel) + do + root="$TRASH_DIRECTORY/vgparallel-$i" + eval "$(cat "$root/test-results/$(basename "$0" .sh)"-*.counts | + sed 's/^\([a-z][a-z]*\) \([0-9][0-9]*\)/inner_\1=\2/')" + test_count=$(expr $test_count + $inner_total) + test_success=$(expr $test_success + $inner_success) + test_fixed=$(expr $test_fixed + $inner_fixed) + test_broken=$(expr $test_broken + $inner_broken) + test_failure=$(expr $test_failure + $inner_failed) + done + test_done +fi + # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 -- cgit v1.2.1