1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
#! /bin/sh
# Copyright (C) 2009-2012 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Basic checks on parallel-tests support:
# - console output
# - log files, and what goes in 'test-suite.log'
# - make clean
# - dependencies between tests
# - TESTS redefinition at runtime (with and without test suffixes)
# - AM_LAZY_CHECK
. test-init.sh
cat >> configure.ac << 'END'
AC_OUTPUT
END
cat > Makefile.am << 'END'
TESTS = foo.test bar.test baz.test
XFAIL_TESTS = bar.test
foo.log: bar.log
bar.log: baz.log
END
cat > foo.test <<'END'
#! /bin/sh
echo "this is $0"
exit 0
END
cat > bar.test <<'END'
#! /bin/sh
echo "this is $0"
exit 99
END
cat > baz.test <<'END'
#! /bin/sh
echo "this is $0"
exit 1
END
chmod a+x foo.test bar.test baz.test
$ACLOCAL
$AUTOCONF
$AUTOMAKE -a
./configure
$MAKE check >stdout && { cat stdout; exit 1; }
cat stdout
count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
test -f test-suite.log
cat test-suite.log
test $(grep -c '^FAIL:' test-suite.log) -eq 1
test $(grep -c '^ERROR:' test-suite.log) -eq 1
$EGREP '^(X?PASS|XFAIL|SKIP)' test-suite.log && exit 1
test -f baz.log
test -f bar.log
test -f foo.log
$MAKE clean
test ! -e baz.log
test ! -e bar.log
test ! -e foo.log
test ! -e test-suite.log
# Check dependencies: baz.test needs to run before bar.test,
# but foo.test is not needed.
# FIXME: Note that this usage has a problem: the summary will only
# FIXME: take bar.log into account, because the $(TEST_SUITE_LOG)
# FIXME: rule does not "see" baz.log. Hmm.
$MAKE check TESTS=bar.test >stdout && { cat stdout; exit 1; }
cat stdout
grep '^FAIL: baz\.test$' stdout
grep '^ERROR: bar\.test$' stdout
test -f baz.log
test -f bar.log
test ! -e foo.log
test -f test-suite.log
# Upon a lazy rerun, foo.test should be run, but the others shouldn't.
# Note that the lazy rerun still exits with a failure, due to the previous
# test failures.
# Note that the previous test and this one taken together expose the timing
# issue that requires the check-TESTS rule to always remove TEST_SUITE_LOG
# before running the tests lazily.
$MAKE check AM_LAZY_CHECK=yes > stdout && { cat stdout; exit 1; }
cat stdout
test -f foo.log
grep '^PASS: foo\.test$' stdout
grep bar.test stdout && exit 1
grep baz.test stdout && exit 1
grep '^# PASS: *1$' stdout
grep '^# FAIL: *1$' stdout
grep '^# ERROR: *1$' stdout
# Now, explicitly retry with all test logs already updated, and ensure
# that the summary is still displayed.
$MAKE check AM_LAZY_CHECK=yes > stdout && { cat stdout; exit 1; }
cat stdout
grep foo.test stdout && exit 1
grep bar.test stdout && exit 1
grep baz.test stdout && exit 1
grep '^# PASS: *1$' stdout
grep '^# FAIL: *1$' stdout
grep '^# ERROR: *1$' stdout
$MAKE clean
$MAKE check TESTS=baz > stdout && { cat stdout; exit 1; }
cat stdout
grep foo.test stdout && exit 1
grep bar.test stdout && exit 1
grep baz.test stdout
$MAKE clean
$MAKE check TESTS=baz.test > stdout && { cat stdout; exit 1; }
cat stdout
grep foo.test stdout && exit 1
grep bar.test stdout && exit 1
grep baz.test stdout
:
|
