testsuite harness: report test exit status in log file

The exit status of a test should be reported in the test logs, so
that one can see at a glance whether the test has succeeded or failed,
without having to look also into the corresponding .trs file.

This fixes automake bug#11814.

* lib/test-driver: Also report the test script exit status in the
test log (as the last line).
* t/check-exit-status-reported.sh: Test this new behaviour.
* t/list-of-tests.mk: Add the new test.
* t/ax/test-lib.sh( am_exit_trap): No longer log the test exit status;
this has been made redundant by the change to 'test-driver'.  While at
it, fix an imperfect quoting.

Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>

5 files changed, 87 insertions, 4 deletions

diff --git a/NEWS b/NEWS
index 5cc001977..61ed194cb 100644
--- a/NEWS
+++ b/NEWS
@@ -73,6 +73,12 @@ New in 1.15:
     (3) the "set -f" and "set +f" shell commands work, and, respectively,
         disable and enable shell globbing.
 
+* Automake-generated testsuites:
+
+  - The default test-driver used by the Automake-generates testsuites now
+    append the result and exit status of each "plain" test to the associated
+    log file (automake bug#118149).
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 New in 1.14.1:
diff --git a/lib/test-driver b/lib/test-driver
index d30605660..110eec409 100755
--- a/lib/test-driver
+++ b/lib/test-driver
@@ -106,11 +106,14 @@ trap "st=143; $do_exit" 15
 # Test script is run here.
 "$@" >$log_file 2>&1
 estatus=$?
+
 if test $enable_hard_errors = no && test $estatus -eq 99; then
-  estatus=1
+  tweaked_estatus=1
+else
+  tweaked_estatus=$estatus
 fi
 
-case $estatus:$expect_failure in
+case $tweaked_estatus:$expect_failure in
   0:yes) col=$red res=XPASS recheck=yes gcopy=yes;;
   0:*)   col=$grn res=PASS  recheck=no  gcopy=no;;
   77:*)  col=$blu res=SKIP  recheck=no  gcopy=yes;;
@@ -119,6 +122,12 @@ case $estatus:$expect_failure in
   *:*)   col=$red res=FAIL  recheck=yes gcopy=yes;;
 esac
 
+# Report the test outcome and exit status in the logs, so that one can
+# know whether the test passed or failed simply by looking at the '.log'
+# file, without the need of also peaking into the corresponding '.trs'
+# file (automake bug#11814).
+echo "$res $test_name (exit status: $estatus)" >>$log_file
+
 # Report outcome to console.
 echo "${col}${res}${std}: $test_name"
 
diff --git a/t/ax/test-lib.sh b/t/ax/test-lib.sh
index aa015d68d..b8c72533c 100644
--- a/t/ax/test-lib.sh
+++ b/t/ax/test-lib.sh
@@ -254,7 +254,7 @@ am_exit_trap ()
     # behaviour, while from time to time useful to developers, is not
     # meant to be enabled by default, as it could cause spurious failures
     # in the wild.  Thus it will be enabled only when the variable
-    # "am_explicit_skips" is set to a "true" value.
+    # 'am_explicit_skips' is set to a "true" value.
     case $am_explicit_skips in
       [yY]|[yY]es|1)
         if test $exit_status -eq 77 && test $am__test_skipped != yes; then
@@ -266,7 +266,6 @@ am_exit_trap ()
   fi
   am_keeping_testdirs || rm_rf_ $am_test_subdir
   set +x
-  echo "$me: exit $exit_status"
   # Spurious escaping to ensure we do not call our "exit" alias.
   \exit $exit_status
 }
diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk
index 462497ee3..ba6578903 100644
--- a/t/list-of-tests.mk
+++ b/t/list-of-tests.mk
@@ -765,6 +765,7 @@ t/parallel-tests-basics.sh \
 t/parallel-tests-concurrency.sh \
 t/parallel-tests-concurrency-2.sh \
 t/parallel-tests-empty.sh \
+t/parallel-tests-exit-status-reported.sh \
 t/parallel-tests-generated-and-distributed.sh \
 t/parallel-tests-recheck.sh \
 t/parallel-tests-trailing-whitespace.sh \
diff --git a/t/parallel-tests-exit-status-reported.sh b/t/parallel-tests-exit-status-reported.sh
new file mode 100644
index 000000000..d40f1f58a
--- /dev/null
+++ b/t/parallel-tests-exit-status-reported.sh
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2013 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# The exit status of a test should be reported in the test logs, so
+# that one can see at a glance whether the test has succeeded or failed,
+# without having to look also into the corresponding .trs file.
+# See automake bug#11814.
+
+. test-init.sh
+
+echo AC_OUTPUT >> configure.ac
+
+echo XFAIL_TESTS = t3.test t00.test > Makefile.am
+echo TESTS = t00.test >> Makefile.am
+for s in 0 1 2 3 5 77 78 99 100 126 127; do
+  echo "TESTS += t${s}.test" >> Makefile.am
+  cat > t${s}.test <<END
+#!/bin/sh
+printf "%s\\n%s\\n" 'random' 'will exit with status $s'
+exit $s
+END
+done
+cp t0.test t00.test
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+run_make -e FAIL check
+ls -l  # For debugging.
+
+match_result ()
+{
+  cat "$1.log"  # For debugging.
+  test $(wc -l <"$1.log") -eq 3
+  sed -n '$p' "$1.log" | grep "^$2 $1\\.test (exit status: $3)$"
+}
+
+match_result  t0   PASS  0
+match_result  t00  XPASS 0
+match_result  t1   FAIL  1
+match_result  t2   FAIL  2
+match_result  t3   XFAIL 3
+match_result  t5   FAIL  5
+match_result  t77  SKIP  77
+match_result  t78  FAIL  78
+match_result  t99  ERROR 99
+match_result  t100 FAIL  100
+match_result  t126 FAIL  126
+match_result  t127 FAIL  127
+
+: