summaryrefslogtreecommitdiff
path: root/t/ax/tap-summary-aux.sh
diff options
context:
space:
mode:
authorStefano Lattarini <stefano.lattarini@gmail.com>2012-04-06 13:12:25 +0200
committerStefano Lattarini <stefano.lattarini@gmail.com>2012-04-06 21:29:30 +0200
commit8a5096d150cf9803b8963768b7366cd68edcce03 (patch)
treea49c0e54876b8b9877d7ca2df386377e7b6037d1 /t/ax/tap-summary-aux.sh
parentf8e822bbc197f01fc722aa6def7cddb4182e3c66 (diff)
downloadautomake-8a5096d150cf9803b8963768b7366cd68edcce03.tar.gz
tests: rename 'tests/' => 't/', '*.test' => '*.sh'
When we (soon) convert the Automake testsuite to a non-recursive make setup, we'll have to fix the entries of $(TESTS) to be prepended with the subdirectory they are in; this will increase the length of $(TESTS), and thus increase the possibility of exceeding the command-line length limits on some systems (most notably, MinGW/MSYS). See automake bug#7868 for more information. Thus we rename the 'tests/' subdirectory to 't/', and each 'x.test' script in there to 'x.sh'; this way, the $(TESTS) entry 'foo.test' will become 't/foo.sh', which have the same number of characters. * tests/: Rename ... * t/: ... to this. * t/*.test: Rename ... * t/*.sh: ... to this. * t/.gitignore: Removed as obsolete. * t/defs: Adjust. * t/gen-testsuite-part: Likewise. * t/list-of-tests.mk: Likewise. * t/ccnoco.sh: Likewise. * t/ccnoco3.sh: Likewise. * t/self-check-cleanup.tap: Likewise. * t/self-check-dir.tap: Likewise. * t/self-check-me.tap: Likewise. * t/self-check-reexec.tap: Likewise. * README: Likewise. * bootstrap: Likewise * configure.ac: Likewise. * Makefile.am: Likewise. * .gitignore: Likewise. * syntax-check.mk: Likewise. Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>
Diffstat (limited to 't/ax/tap-summary-aux.sh')
-rw-r--r--t/ax/tap-summary-aux.sh359
1 files changed, 359 insertions, 0 deletions
diff --git a/t/ax/tap-summary-aux.sh b/t/ax/tap-summary-aux.sh
new file mode 100644
index 000000000..6965281bc
--- /dev/null
+++ b/t/ax/tap-summary-aux.sh
@@ -0,0 +1,359 @@
+#! /bin/sh
+# Copyright (C) 2011-2012 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary script for tests on TAP support: checking testsuite summary.
+
+am_parallel_tests=yes
+. ./defs || Exit 1
+
+br='============================================================================'
+
+case $use_colors in
+ yes|no) ;;
+ *) fatal_ "invalid \$use_colors value '$use_colors'"
+esac
+
+fetch_tap_driver
+
+cat > configure.ac <<END
+AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
+AM_INIT_AUTOMAKE([parallel-tests])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(srcdir)/tap-driver
+TEST_LOG_COMPILER = cat
+TESTS = all.test
+END
+
+# The following shell variables are influential for this function:
+# - expect_failure
+# - use_colors
+do_check ()
+{
+ case $#,$1 in
+ 1,--pass) expect_failure=no;;
+ 1,--fail) expect_failure=yes;;
+ *) fatal_ "invalid usage of 'do_check'";;
+ esac
+ shift
+ cat > summary.exp
+ cat all.test
+ st=0
+ if test $use_colors = yes; then
+ make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
+ else
+ make_cmd=$MAKE
+ fi
+ $make_cmd check > stdout || st=$?
+ cat stdout
+ if test $expect_failure = yes; then
+ test $st -gt 0 || Exit 1
+ else
+ test $st -eq 0 || Exit 1
+ fi
+ $PERL "$am_testauxdir"/extract-testsuite-summary.pl stdout >summary.got \
+ || fatal_ "cannot extract testsuite summary"
+ cat summary.exp
+ cat summary.got
+ if test $use_colors = yes; then
+ # Use cmp, not diff, because the files might contain binary data.
+ compare=cmp
+ else
+ compare=diff
+ fi
+ $compare summary.exp summary.got || Exit 1
+}
+
+if test $use_colors = yes; then
+ red=''
+ grn=''
+ lgn=''
+ blu=''
+ mgn=''
+ brg=''
+ std=''
+ echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
+else
+ red= grn= lgn= blu= mgn= brg= std=
+fi
+
+ success_header="\
+${grn}${br}${std}
+${grn}Testsuite summary for GNU AutoTAP 5.12${std}
+${grn}${br}${std}"
+
+ success_footer=${grn}${br}${std}
+
+ failure_header="\
+${red}${br}${std}
+${red}Testsuite summary for GNU AutoTAP 5.12${std}
+${red}${br}${std}"
+
+ failure_footer="\
+${red}${br}${std}
+${red}See ./test-suite.log${std}
+${red}Please report to bug-automake@gnu.org${std}
+${red}${br}${std}"
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+./configure
+
+# 1 pass.
+{ echo 1..1 && echo ok; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 skip.
+{ echo 1..1 && echo 'ok # SKIP'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 xfail.
+{ echo 1..1 && echo 'not ok # TODO'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 fail.
+{ echo 1..1 && echo not ok; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 xpass.
+{ echo 1..1 && echo 'ok # TODO'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 hard error.
+{ echo 1..1 && echo 'Bail out!'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 3 non-failing results.
+cat > all.test <<END
+1..3
+ok
+not ok # TODO
+ok # SKIP
+END
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 pass, 1 skip, 1 fail.
+cat > all.test <<END
+1..3
+ok
+ok # SKIP
+not ok
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 pass, 1 xfail, 1 xpass.
+cat > all.test <<END
+1..3
+ok
+ok # TODO
+not ok # TODO
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 skip, 1 xfail, 1 error.
+cat > all.test <<END
+1..3
+ok # SKIP
+not ok # TODO
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 1 of each kind
+cat > all.test <<END
+1..6
+ok
+not ok
+ok # TODO
+not ok # TODO
+ok # SKIP
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 6${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+${red}# FAIL: 1${std}
+${red}# XPASS: 1${std}
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# Prepare some common data for later.
+for i in 0 1 2 3 4 5 6 7 8 9; do
+ for j in 0 1 2 3 4 5 6 7 8 9; do
+ echo "ok"
+ echo "not ok # TODO"
+ echo "ok # SKIP"
+ done
+done > tap
+
+# Lots of non-failures (300 per kind).
+(cat tap && cat tap && cat tap) > all.test
+test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
+echo 1..900 >> all.test # Test plan.
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 900${std}
+${grn}# PASS: 300${std}
+${blu}# SKIP: 300${std}
+${lgn}# XFAIL: 300${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 failure and lots of non-failures means failure.
+(cat tap && echo "not ok" && cat tap) > all.test
+test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
+echo 1..601 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 601${std}
+${grn}# PASS: 200${std}
+${blu}# SKIP: 200${std}
+${lgn}# XFAIL: 200${std}
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 error and lots of non-failures means failure.
+(cat tap && sed 30q tap && echo 'Bail out!') > all.test
+test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
+echo 1..331 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 331${std}
+${grn}# PASS: 110${std}
+${blu}# SKIP: 110${std}
+${lgn}# XFAIL: 110${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+: