From 16a69f358a38c60577e25bc0811a220f8e71d2cd Mon Sep 17 00:00:00 2001 From: Jelmer Vernooij Date: Thu, 4 Jun 2009 13:49:11 +0200 Subject: selftest: Use external processes for filtering known failures and pretty formatting. --- selftest/Subunit.pm | 42 ++++++++------- selftest/filter-xfail.pl | 7 +++ selftest/format-subunit.pl | 35 ++++++++++++- selftest/output/buildfarm.pm | 5 +- selftest/output/plain.pm | 12 +++-- selftest/output/subunit.pm | 2 +- selftest/selftest.pl | 118 ++++++++++++------------------------------- 7 files changed, 110 insertions(+), 111 deletions(-) (limited to 'selftest') diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm index 0eafc44146f..7feb5de9a43 100644 --- a/selftest/Subunit.pm +++ b/selftest/Subunit.pm @@ -41,6 +41,8 @@ sub parse_results($$$$) $msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1)); } elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) { $msg_ops->control_msg($_); + my $result = $1; + my $testname = $2; my $reason = undef; if ($3) { $reason = ""; @@ -53,33 +55,35 @@ sub parse_results($$$$) unless ($terminated) { $statistics->{TESTS_ERROR}++; - $msg_ops->end_test($open_tests, $2, $1, 1, "reason interrupted"); + $msg_ops->end_test($open_tests, $testname, $result, 1, "reason interrupted"); return 1; } } - my $result = $1; - if ($1 eq "success" or $1 eq "successful") { - pop(@$open_tests); #FIXME: Check that popped value == $2 + if ($result eq "success" or $result eq "successful") { + pop(@$open_tests); #FIXME: Check that popped value == $testname $statistics->{TESTS_EXPECTED_OK}++; - $msg_ops->end_test($open_tests, $2, $1, 0, $reason); - } elsif ($1 eq "xfail" or $1 eq "knownfail") { - pop(@$open_tests); #FIXME: Check that popped value == $2 + $msg_ops->end_test($open_tests, $testname, $result, 0, $reason); + } elsif ($result eq "xfail" or $result eq "knownfail") { + pop(@$open_tests); #FIXME: Check that popped value == $testname $statistics->{TESTS_EXPECTED_FAIL}++; - $msg_ops->end_test($open_tests, $2, $1, 0, $reason); + $msg_ops->end_test($open_tests, $testname, $result, 0, $reason); $expected_fail++; - } elsif ($1 eq "failure" or $1 eq "fail") { - pop(@$open_tests); #FIXME: Check that popped value == $2 + } elsif ($result eq "failure" or $result eq "fail") { + pop(@$open_tests); #FIXME: Check that popped value == $testname $statistics->{TESTS_UNEXPECTED_FAIL}++; - $msg_ops->end_test($open_tests, $2, $1, 1, $reason); + $msg_ops->end_test($open_tests, $testname, $result, 1, $reason); $unexpected_fail++; - } elsif ($1 eq "skip") { + } elsif ($result eq "skip") { $statistics->{TESTS_SKIP}++; - pop(@$open_tests); #FIXME: Check that popped value == $2 - $msg_ops->end_test($open_tests, $2, $1, 0, $reason); - } elsif ($1 eq "error") { + my $last = pop(@$open_tests); + if (defined($last) and $last ne $testname) { + push (@$open_tests, $testname); + } + $msg_ops->end_test($open_tests, $testname, $result, 0, $reason); + } elsif ($result eq "error") { $statistics->{TESTS_ERROR}++; - pop(@$open_tests); #FIXME: Check that popped value == $2 - $msg_ops->end_test($open_tests, $2, $1, 1, $reason); + pop(@$open_tests); #FIXME: Check that popped value == $testname + $msg_ops->end_test($open_tests, $testname, $result, 1, $reason); $unexpected_err++; } } else { @@ -114,7 +118,9 @@ sub end_test($$;$) my $result = shift; my $reason = shift; if ($reason) { - print "$result: $name [ $reason ]\n"; + print "$result: $name ["; + print "$reason"; + print "]\n"; } else { print "$result: $name\n"; } diff --git a/selftest/filter-xfail.pl b/selftest/filter-xfail.pl index edf0e050a71..f41bb77f44f 100755 --- a/selftest/filter-xfail.pl +++ b/selftest/filter-xfail.pl @@ -56,13 +56,20 @@ use lib "$RealBin"; use Subunit qw(parse_results); my $opt_expected_failures = undef; +my $opt_help = 0; my @expected_failures = (); my $result = GetOptions( 'expected-failures=s' => \$opt_expected_failures, + 'help' => \$opt_help, ); exit(1) if (not $result); +if ($opt_help) { + print "Usage: filter-xfail [--expected-failures=FILE]... < instream > outstream\n"; + exit(0); +} + sub read_test_regexes($) { my ($name) = @_; diff --git a/selftest/format-subunit.pl b/selftest/format-subunit.pl index c60902f8d75..acaac697aa0 100755 --- a/selftest/format-subunit.pl +++ b/selftest/format-subunit.pl @@ -3,6 +3,38 @@ # Copyright (C) Jelmer Vernooij # Published under the GNU GPL, v3 or later +=pod + +=head1 NAME + +format-subunit [--format=] [--immediate] < instream > outstream + +=head1 SYNOPSIS + +Format the output of a subunit stream. + +=head1 OPTIONS + +=over 4 + +=item I<--immediate> + +Show errors as soon as they happen rather than at the end of the test run. + +=item I<--format>=FORMAT + +Choose the format to print. Currently supported are plain, html or buildfarm. + +=head1 LICENSE + +GNU General Public License, version 3 or later. + +=head1 AUTHOR + +Jelmer Vernooij + +=cut + use Getopt::Long; use strict; use FindBin qw($RealBin $Script); @@ -60,8 +92,7 @@ if ($opt_format eq "buildfarm") { die("Invalid output format '$opt_format'"); } -my $expected_ret = parse_results( - $msg_ops, $statistics, *STDIN, []); +my $expected_ret = parse_results($msg_ops, $statistics, *STDIN, []); $msg_ops->summary(); diff --git a/selftest/output/buildfarm.pm b/selftest/output/buildfarm.pm index f4daf690108..701444359b5 100644 --- a/selftest/output/buildfarm.pm +++ b/selftest/output/buildfarm.pm @@ -29,9 +29,10 @@ use BuildFarm; use strict; sub new($$$) { - my ($class) = @_; + my ($class, $statistics) = @_; my $self = { test_output => {}, + statistics => $statistics, last_time => 0, start_time => undef, }; @@ -111,6 +112,8 @@ sub summary($) my ($self) = @_; BuildFarm::summary($self->{last_time} - $self->{start_time}); + + print "TEST STATUS: $self->{statistics}->{SUITES_FAIL}\n"; } sub skip_testsuite($$$) diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm index 20e5abc253a..9d92a3e0715 100644 --- a/selftest/output/plain.pm +++ b/selftest/output/plain.pm @@ -87,8 +87,10 @@ sub output_msg($$) require FileHandle; print $output; STDOUT->flush(); - } else { + } elsif (defined($self->{NAME})) { $self->{test_output}->{$self->{NAME}} .= $output; + } else { + print $output; } } @@ -96,7 +98,7 @@ sub control_msg($$) { my ($self, $output) = @_; - $self->output_msg($output); + #$self->output_msg($output); } sub end_testsuite($$$$$) @@ -157,6 +159,7 @@ sub end_test($$$$$) $self->{test_output}->{$self->{NAME}} = ""; if (not $self->{immediate}) { if ($result eq "failure") { print "f"; } + elsif ($result eq "xfail") { print "X"; } elsif ($result eq "skip") { print "s"; } elsif ($result eq "success") { print "."; } else { print "?($result)"; } @@ -231,10 +234,13 @@ sub summary($) } -sub skip_testsuite($$) +sub skip_testsuite($$$) { my ($self, $name, $reason) = @_; + unless (defined($reason)) { + $reason = "UNKNOWN"; + } push (@{$self->{skips}->{$reason}}, $name); if ($self->{totalsuites}) { diff --git a/selftest/output/subunit.pm b/selftest/output/subunit.pm index a7ca5d943d1..5bce99f68f8 100644 --- a/selftest/output/subunit.pm +++ b/selftest/output/subunit.pm @@ -95,7 +95,7 @@ sub skip_testsuite($$$$) my ($self, $name, $reason) = @_; Subunit::start_test($name); - Subunit::end_test($name, "skip"); + Subunit::end_test($name, "skip", $reason); } 1; diff --git a/selftest/selftest.pl b/selftest/selftest.pl index 8a2c8804bc6..43bdc7a8048 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -26,7 +26,7 @@ selftest - Samba test runner selftest --help -selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS] +selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS] =head1 DESCRIPTION @@ -56,10 +56,6 @@ Executable extention Change directory to run tests in. Default is 'st'. -=item I<--immediate> - -Show errors as soon as they happen rather than at the end of the test run. - =item I<--target samba4|samba3|win|kvm> Specify test target against which to run. Default is 'samba4'. @@ -143,7 +139,6 @@ my $opt_socket_wrapper = 0; my $opt_socket_wrapper_pcap = undef; my $opt_socket_wrapper_keep_pcap = undef; my $opt_one = 0; -my $opt_immediate = 0; my @opt_exclude = (); my @opt_include = (); my $opt_verbose = 0; @@ -154,7 +149,6 @@ my $opt_analyse_cmd = undef; my $opt_resetup_env = undef; my $opt_bindir = undef; my $opt_no_lazy_setup = undef; -my $opt_format = "plain"; my @testlists = (); my $srcdir = "."; @@ -165,17 +159,6 @@ my $prefix = "./st"; my @includes = (); my @excludes = (); -my $statistics = { - SUITES_FAIL => 0, - - TESTS_UNEXPECTED_OK => 0, - TESTS_EXPECTED_OK => 0, - TESTS_UNEXPECTED_FAIL => 0, - TESTS_EXPECTED_FAIL => 0, - TESTS_ERROR => 0, - TESTS_SKIP => 0, -}; - sub find_in_list($$) { my ($list, $fullname) = @_; @@ -216,65 +199,61 @@ sub setup_pcap($) return $pcap_file; } -sub cleanup_pcap($$$) +sub cleanup_pcap($$) { - my ($pcap_file, $expected_ret, $ret) = @_; + my ($pcap_file, $exitcode) = @_; return unless ($opt_socket_wrapper_pcap); return if ($opt_socket_wrapper_keep_pcap); - return unless ($expected_ret == $ret); + return unless ($exitcode == 0); return unless defined($pcap_file); unlink($pcap_file); } -sub run_testsuite($$$$$$) +sub run_testsuite($$$$$) { - my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_; + my ($envname, $name, $cmd, $i, $totalsuites) = @_; my $pcap_file = setup_pcap($name); - $msg_ops->report_time(time()); - $msg_ops->start_test([], $name); + Subunit::report_time(time()); + Subunit::start_test($name); - unless (open(RESULT, "$cmd 2>&1|")) { - $statistics->{TESTS_ERROR}++; - $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!"); - $statistics->{SUITES_FAIL}++; + my $ret = system("$cmd 2>&1"); + if ($ret == -1) { + Subunit::end_test($name, "error", "Unable to run $cmd: $!"); return 0; } - - my $expected_ret = parse_results( - $msg_ops, $statistics, *RESULT, [$name]); - my $envlog = getlog_env($envname); - $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne ""); - - $msg_ops->output_msg("CMD: $cmd\n"); + if ($envlog ne "") { + print "ENVLOG: $envlog\n"; + } - my $ret = close(RESULT); - $ret = 0 unless $ret == 1; + print "CMD: $cmd\n"; - my $exitcode = $? >> 8; + my $exitcode = $ret >> 8; - $msg_ops->report_time(time()); - if ($ret == 1) { - $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef); + Subunit::report_time(time()); + my $reason = "Exit code was $exitcode"; + my $result; + if ($exitcode == 0) { + $result = "success"; } else { - $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode"); + $result = "failure"; } + Subunit::end_test($name, $result, $reason); - cleanup_pcap($pcap_file, $expected_ret, $ret); + cleanup_pcap($pcap_file, $exitcode); if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) { - $msg_ops->output_msg("PCAP FILE: $pcap_file\n"); + print "PCAP FILE: $pcap_file\n"; } - if ($ret != $expected_ret) { - $statistics->{SUITES_FAIL}++; + if ($exitcode != 0) { exit(1) if ($opt_one); } - return ($ret == $expected_ret); + return $exitcode; } sub ShowHelp() @@ -312,7 +291,6 @@ Kvm Specific: Behaviour: --quick run quick overall test --one abort when the first test fails - --immediate print test output for failed tests during run --verbose be verbose --analyse-cmd CMD command to run after each test "; @@ -328,7 +306,6 @@ my $result = GetOptions ( 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap, 'quick' => \$opt_quick, 'one' => \$opt_one, - 'immediate' => \$opt_immediate, 'exclude=s' => \@opt_exclude, 'include=s' => \@opt_include, 'srcdir=s' => \$srcdir, @@ -341,7 +318,6 @@ my $result = GetOptions ( 'no-lazy-setup' => \$opt_no_lazy_setup, 'resetup-environment' => \$opt_resetup_env, 'bindir:s' => \$opt_bindir, - 'format=s' => \$opt_format, 'image=s' => \$opt_image, 'testlist=s' => \@testlists ); @@ -401,11 +377,6 @@ $ENV{BUILDDIR} = $builddir; $ENV{BUILDDIR_ABS} = $builddir_abs; $ENV{EXEEXT} = $exeext; -if (defined($ENV{RUN_FROM_BUILD_FARM}) and - ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) { - $opt_format = "buildfarm"; -} - my $tls_enabled = not $opt_quick; $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no"); $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb"; @@ -650,30 +621,13 @@ foreach my $fn (@testlists) { } } -my $msg_ops; -if ($opt_format eq "buildfarm") { - require output::buildfarm; - $msg_ops = new output::buildfarm($statistics); -} elsif ($opt_format eq "plain") { - require output::plain; - $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1); -} elsif ($opt_format eq "html") { - require output::html; - mkdir("test-results", 0777); - $msg_ops = new output::html("test-results", $statistics); -} elsif ($opt_format eq "subunit") { - require output::subunit; - $msg_ops = new output::subunit(); -} else { - die("Invalid output format '$opt_format'"); -} -$msg_ops->report_time(time()); +Subunit::report_time(time()); foreach (@available) { my $name = $$_[0]; my $skipreason = skip($name); if ($skipreason) { - $msg_ops->skip_testsuite($name, $skipreason); + Subunit::end_test($name, "skip", $skipreason); } else { push(@todo, $_); } @@ -861,13 +815,12 @@ $envvarstr my $envvars = setup_env($envname); if (not defined($envvars)) { - $msg_ops->skip_testsuite($name, + Subunit::end_test($name, "skip", "unable to set up environment $envname"); next; } - run_testsuite($envname, $name, $cmd, $i, $suitestotal, - $msg_ops); + run_testsuite($envname, $name, $cmd, $i, $suitestotal); if (defined($opt_analyse_cmd)) { system("$opt_analyse_cmd \"$name\""); @@ -883,8 +836,6 @@ teardown_env($_) foreach (keys %running_envs); $target->stop(); -$msg_ops->summary(); - my $failed = 0; # if there were any valgrind failures, show them @@ -897,9 +848,4 @@ foreach (<$prefix/valgrind.log*>) { system("cat $_"); } } - -if ($opt_format eq "buildfarm") { - print "TEST STATUS: $statistics->{SUITES_FAIL}\n"; -} - -exit $statistics->{SUITES_FAIL}; +exit 0; -- cgit v1.2.1