Autotest: add official way to execute code before all/each test.zack/autotest-new-hooks

Currently, there isn’t any documented way for an Autotest testsuite to
add custom code to be run either right before the main driver loop, or
at the point of each AT_SETUP.  For instance, there’s no good place to
put environment variable sanitization that should apply to the entire
testsuite (but isn’t universally relevant), or shell function
definitions to be used by custom test macros.

Autoconf’s test suite is poking shell functions directly into the
PREPARE_TESTS diversion, and doing environment variable sanitization
in each individual test.  Both of these are obviously undesirable.

This patch adds three new AT_* macros that can be used to do these
things in an officially-supported way: AT_PREPARE_TESTS adds code to
be run right before the main driver loop, AT_PREPARE_EACH_TEST adds
code to be run at the beginning of each test, and AT_TEST_HELPER_FN
defines a shell function that will be available to each test.  In
Autoconf’s test suite, I use AT_PREPARE_TESTS to factor out
environment variable sanitization that *ought* to apply across the
board, and AT_TEST_HELPER_FN for the helper function used by
AT_CHECK_ENV.

(This fixes the testsuite bug reported by Jannick at
https://lists.gnu.org/archive/html/autoconf/2020-10/msg00052.html .)

Comments requested particularly on the names and implementations of
the new macros.  Also, it would be nice to have an example of when
AT_PREPARE_EACH_TEST is useful, but I didn’t find one in the autoconf
test suite.

* lib/autotest/general.m4 (AT_PREPARE_TESTS, AT_PREPARE_EACH_TEST)
  (AT_TEST_HELPER_FN): New macros.
  (AT_INIT, AT_TESTED): Emit the code to report tested programs only
  if it’s needed, and make sure it’s after any code added by
  AT_PREPARE_TESTS.

* tests/local.at: Add AT_PREPARE_TESTS block that ensures
  $MAKE is set sensibly and $MAKEFLAGS and $CONFIG_SITE are unset.
  Use AT_TEST_HELPER_FN for the helper function needed by AT_CHECK_ENV.
  (AT_CHECK_MAKE): No need to sanitize $MAKE or $MAKEFLAGS here.
* tests/base.at, tests/compile.at, tests/m4sh.at, tests/torture.at:
  No need to unset or neutralize $CONFIG_SITE in individual tests.
* tests/autotest.at: Add tests for new macros.

* doc/autoconf.texi, NEWS: Document new macros.

1 files changed, 82 insertions, 33 deletions

diff --git a/lib/autotest/general.m4 b/lib/autotest/general.m4
index f29ce490..955ed0d9 100644
--- a/lib/autotest/general.m4
+++ b/lib/autotest/general.m4
@@ -429,9 +429,6 @@ m4_divert_text([DEFAULTS],
 [
 # Whether to enable colored test results.
 at_color=m4_ifdef([AT_color], [AT_color], [no])
-# List of the tested programs.
-at_tested='m4_ifdef([AT_tested],
-  [m4_translit(m4_dquote(m4_defn([AT_tested])), [ ], m4_newline)])'
 # As many question marks as there are digits in the last test group number.
 # Used to normalize the test group numbers so that `ls' lists them in
 # numerical order.
@@ -992,33 +989,7 @@ AS_BOX(m4_defn([AT_TESTSUITE_NAME])[.])
 } >&AS_MESSAGE_LOG_FD
 
 m4_divert_pop([TESTS_BEGIN])dnl
-m4_divert_push([PREPARE_TESTS])dnl
-{
-  AS_BOX([Tested programs.])
-  echo
-} >&AS_MESSAGE_LOG_FD
-
-# Report what programs are being tested.
-for at_program in : `eval echo $at_tested`
-do
-  test "$at_program" = : && continue
-  case $at_program in
-    [[\\/]* | ?:[\\/]* ) $at_program_=$at_program ;;]
-    * )
-    _AS_PATH_WALK([$PATH], [test -f "$as_dir$at_program" && break])
-    at_program_=$as_dir$at_program ;;
-  esac
-  if test -f "$at_program_"; then
-    {
-      AS_ECHO(["$at_srcdir/AT_LINE: $at_program_ --version"])
-      "$at_program_" --version </dev/null
-      echo
-    } >&AS_MESSAGE_LOG_FD 2>&1
-  else
-    AS_ERROR([cannot find $at_program])
-  fi
-done
-
+m4_divert_push([TESTS])dnl
 {
   AS_BOX([Running the tests.])
 } >&AS_MESSAGE_LOG_FD
@@ -1026,8 +997,6 @@ done
 at_start_date=`date`
 at_start_time=`date +%s 2>/dev/null`
 AS_ECHO(["$as_me: starting at: $at_start_date"]) >&AS_MESSAGE_LOG_FD
-m4_divert_pop([PREPARE_TESTS])dnl
-m4_divert_push([TESTS])dnl
 
 # Create the master directory if it doesn't already exist.
 AS_MKDIR_P(["$at_suite_dir"]) ||
@@ -1794,10 +1763,88 @@ m4_defun([AT_ARG_OPTION_ARG],[_AT_ARG_OPTION([$1],[$2],1,[$3],[$4])])
 # versions are logged, and in the case of embedded test suite, they
 # must correspond to the version of the package.  PATH should be
 # already preset so the proper executable will be selected.
-m4_define([AT_TESTED],
+m4_defun([AT_TESTED],
+[m4_require([_AT_TESTED])]dnl
 [m4_foreach_w([AT_test], [$1],
   [m4_append_uniq([AT_tested], "m4_defn([AT_test])", [ ])])])
 
+m4_defun([_AT_TESTED],
+[m4_wrap([m4_divert_text([DEFAULTS],
+[# List of the tested programs.
+at_tested='m4_translit(m4_dquote(m4_defn([AT_tested])), [ ], m4_newline)'
+])]dnl
+[m4_divert_text([PREPARE_TESTS],
+[{
+  AS_BOX([Tested programs.])
+  echo
+} >&AS_MESSAGE_LOG_FD
+
+# Report what programs are being tested.
+for at_program in : `eval echo $at_tested`
+do
+  AS_CASE([$at_program],
+    [:], [continue],
+    [[[\\/]* | ?:[\\/]*]], [$at_program_=$at_program],
+    [_AS_PATH_WALK([$PATH], [test -f "$as_dir$at_program" && break])
+    at_program_=$as_dir$at_program])
+
+  if test -f "$at_program_"; then
+    {
+      AS_ECHO(["$at_srcdir/AT_LINE: $at_program_ --version"])
+      "$at_program_" --version </dev/null
+      echo
+    } >&AS_MESSAGE_LOG_FD 2>&1
+  else
+    AS_ERROR([cannot find $at_program])
+  fi
+done
+])])])
+
+
+# AT_PREPARE_TESTS(SHELL-CODE)
+# ----------------------------
+# Execute @var{shell-code} in the main testsuite process,
+# after initializing the test suite and processing command-line options,
+# but before running any tests.
+m4_define([AT_PREPARE_TESTS],
+[m4_divert_once([PREPARE_TESTS],
+[m4_text_box([Prepare for this testsuite.])
+])]dnl
+[m4_divert_text([PREPARE_TESTS], [$1])])
+
+
+# AT_PREPARE_EACH_TEST([SHELL-CODE])
+# ----------------------------------
+# Execute @var{shell-code} in each test group's subshell,
+# at the point of the AT_SETUP that starts each test group.
+m4_define([AT_PREPARE_EACH_TEST],
+[m4_append([AT_prepare_each_test], [$1], [
+])])
+
+
+# AT_TEST_HELPER_FN(NAME, ARGS, DESCRIPTION, CODE)
+# ------------------------------------------------
+# Define a shell function that will be available to the code for each test
+# group.  Its name will be ath_fn_NAME, and its body will be CODE.
+#
+# Implementation note: you might think this would use AT_PREPARE_EACH_TEST,
+# but shell functions defined in AT_PREPARE_TESTS *are* (currently) available
+# to test group subshells, and this way the code is only emitted once, not
+# once for each test group.
+m4_define([AT_TEST_HELPER_FN],
+[AS_LITERAL_WORD_IF([$1], [],
+  [m4_fatal([invalid shell function name "$1"])])]dnl
+[m4_ifdef([ATH_fn_$1_defined],
+  [m4_fatal([helper function "$1" defined twice])])]dnl
+[m4_define([ATH_fn_$1_defined])]dnl
+[AT_PREPARE_TESTS([
+AS_FUNCTION_DESCRIBE([ath_fn_$1], [$2], [$3])
+ath_fn_$1 ()
+{
+  $4
+}
+])])
+
 
 # AT_COPYRIGHT(TEXT, [FILTER = m4_newline])
 # -----------------------------------------
@@ -1837,6 +1884,8 @@ at_fn_group_banner AT_ordinal 'm4_defn([AT_line])' \
   "AS_ESCAPE(m4_dquote(m4_defn([AT_description])))" m4_format(["%*s"],
   m4_max(0, m4_eval(47 - m4_qlen(m4_defn([AT_description])))), [])m4_if(
   AT_banner_ordinal, [0], [], [ AT_banner_ordinal])
+m4_ifset([AT_prepare_each_test], [AT_prepare_each_test
+])dnl
 m4_divert_push([TEST_SCRIPT])dnl
 ])