diff options
-rw-r--r-- | NEWS | 6 | ||||
-rwxr-xr-x | lib/test-driver | 13 | ||||
-rw-r--r-- | t/ax/test-lib.sh | 3 | ||||
-rw-r--r-- | t/list-of-tests.mk | 1 | ||||
-rw-r--r-- | t/parallel-tests-exit-status-reported.sh | 68 |
5 files changed, 87 insertions, 4 deletions
@@ -73,6 +73,12 @@ New in 1.15: (3) the "set -f" and "set +f" shell commands work, and, respectively, disable and enable shell globbing. +* Automake-generated testsuites: + + - The default test-driver used by the Automake-generates testsuites now + append the result and exit status of each "plain" test to the associated + log file (automake bug#118149). + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ New in 1.14.1: diff --git a/lib/test-driver b/lib/test-driver index d30605660..110eec409 100755 --- a/lib/test-driver +++ b/lib/test-driver @@ -106,11 +106,14 @@ trap "st=143; $do_exit" 15 # Test script is run here. "$@" >$log_file 2>&1 estatus=$? + if test $enable_hard_errors = no && test $estatus -eq 99; then - estatus=1 + tweaked_estatus=1 +else + tweaked_estatus=$estatus fi -case $estatus:$expect_failure in +case $tweaked_estatus:$expect_failure in 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; 0:*) col=$grn res=PASS recheck=no gcopy=no;; 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; @@ -119,6 +122,12 @@ case $estatus:$expect_failure in *:*) col=$red res=FAIL recheck=yes gcopy=yes;; esac +# Report the test outcome and exit status in the logs, so that one can +# know whether the test passed or failed simply by looking at the '.log' +# file, without the need of also peaking into the corresponding '.trs' +# file (automake bug#11814). +echo "$res $test_name (exit status: $estatus)" >>$log_file + # Report outcome to console. echo "${col}${res}${std}: $test_name" diff --git a/t/ax/test-lib.sh b/t/ax/test-lib.sh index aa015d68d..b8c72533c 100644 --- a/t/ax/test-lib.sh +++ b/t/ax/test-lib.sh @@ -254,7 +254,7 @@ am_exit_trap () # behaviour, while from time to time useful to developers, is not # meant to be enabled by default, as it could cause spurious failures # in the wild. Thus it will be enabled only when the variable - # "am_explicit_skips" is set to a "true" value. + # 'am_explicit_skips' is set to a "true" value. case $am_explicit_skips in [yY]|[yY]es|1) if test $exit_status -eq 77 && test $am__test_skipped != yes; then @@ -266,7 +266,6 @@ am_exit_trap () fi am_keeping_testdirs || rm_rf_ $am_test_subdir set +x - echo "$me: exit $exit_status" # Spurious escaping to ensure we do not call our "exit" alias. \exit $exit_status } diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk index 462497ee3..ba6578903 100644 --- a/t/list-of-tests.mk +++ b/t/list-of-tests.mk @@ -765,6 +765,7 @@ t/parallel-tests-basics.sh \ t/parallel-tests-concurrency.sh \ t/parallel-tests-concurrency-2.sh \ t/parallel-tests-empty.sh \ +t/parallel-tests-exit-status-reported.sh \ t/parallel-tests-generated-and-distributed.sh \ t/parallel-tests-recheck.sh \ t/parallel-tests-trailing-whitespace.sh \ diff --git a/t/parallel-tests-exit-status-reported.sh b/t/parallel-tests-exit-status-reported.sh new file mode 100644 index 000000000..d40f1f58a --- /dev/null +++ b/t/parallel-tests-exit-status-reported.sh @@ -0,0 +1,68 @@ +#! /bin/sh +# Copyright (C) 2013 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# The exit status of a test should be reported in the test logs, so +# that one can see at a glance whether the test has succeeded or failed, +# without having to look also into the corresponding .trs file. +# See automake bug#11814. + +. test-init.sh + +echo AC_OUTPUT >> configure.ac + +echo XFAIL_TESTS = t3.test t00.test > Makefile.am +echo TESTS = t00.test >> Makefile.am +for s in 0 1 2 3 5 77 78 99 100 126 127; do + echo "TESTS += t${s}.test" >> Makefile.am + cat > t${s}.test <<END +#!/bin/sh +printf "%s\\n%s\\n" 'random' 'will exit with status $s' +exit $s +END +done +cp t0.test t00.test + +chmod a+x *.test + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure +run_make -e FAIL check +ls -l # For debugging. + +match_result () +{ + cat "$1.log" # For debugging. + test $(wc -l <"$1.log") -eq 3 + sed -n '$p' "$1.log" | grep "^$2 $1\\.test (exit status: $3)$" +} + +match_result t0 PASS 0 +match_result t00 XPASS 0 +match_result t1 FAIL 1 +match_result t2 FAIL 2 +match_result t3 XFAIL 3 +match_result t5 FAIL 5 +match_result t77 SKIP 77 +match_result t78 FAIL 78 +match_result t99 ERROR 99 +match_result t100 FAIL 100 +match_result t126 FAIL 126 +match_result t127 FAIL 127 + +: |