1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
#! /bin/sh
# Copyright (C) 2011-2017 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Check test counts in the testsuite summary, with test drivers allowing
# multiple test results per test script, and for a huge number of tests.
# Incidentally, this test also checks that the testsuite summary doesn't
# give any bug-report address if it's not defined.
. test-init.sh
for s in trivial-test-driver extract-testsuite-summary.pl; do
cp "$am_testaux_srcdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
done
br='============================================================================'
header="\
${br}
Testsuite summary for $me 1.0
${br}"
footer="\
${br}
See ./test-suite.log
${br}"
echo AC_OUTPUT >> configure.ac
cat > Makefile.am << 'END'
TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
TESTS = all.test
# Without this, the test driver will be horrendously slow.
END
cat > all.test <<'END'
#!/bin/sh
cat results.txt || { echo ERROR: weird; exit 99; }
END
chmod a+x all.test
$PERL -w -e '
use warnings FATAL => "all";
use strict;
my $base = 1000;
my %count = (
TOTAL => $base * 1000,
PASS => $base * 700,
SKIP => $base * 200,
XFAIL => $base * 80,
FAIL => $base * 10,
XPASS => $base * 7,
ERROR => $base * 3,
);
my @results = qw/PASS SKIP XFAIL FAIL XPASS ERROR/;
open (RES, ">results.txt") or die "opening results.txt: $!\n";
open (CNT, ">count.txt") or die "opening count.txt: $!\n";
printf CNT "# %-6s %d\n", "TOTAL:", $count{TOTAL};
for my $res (@results)
{
my $uc_res = uc $res;
print STDERR "Generating list of $res ...\n";
for (1..$count{$res})
{
print RES "$uc_res: $_\n";
}
printf CNT "# %-6s %d\n", $res . ":", $count{$res};
}
'
(echo "$header" && cat count.txt && echo "$footer") > summary.exp
$ACLOCAL
$AUTOMAKE -a
$AUTOCONF
./configure
($MAKE check || touch make.fail) | tee stdout
test -f make.fail
$PERL extract-testsuite-summary.pl stdout > summary.got
cat summary.exp
cat summary.got
diff summary.exp summary.got || exit 1
:
|
