diff options
author | Daniel Stenberg <daniel@haxx.se> | 2019-12-10 14:29:48 +0100 |
---|---|---|
committer | Daniel Stenberg <daniel@haxx.se> | 2019-12-11 09:55:15 +0100 |
commit | 7c1bd035760faa5ad6d81b6280dffbdd7feb6302 (patch) | |
tree | 27f2ac04d28f4133bb033afb3f206f5ea67573fb /tests/runtests.pl | |
parent | 1d5c427d7fd6c206de4b010f618659a7a9b38cff (diff) | |
download | curl-7c1bd035760faa5ad6d81b6280dffbdd7feb6302.tar.gz |
runtests: introduce --shallow to reduce huge torture tests
When set, shallow mode limits runtests -t to make no more than NUM fails
per test case. If more are found, it will randomly discard entries until
the number is right. The random seed can also be set.
This is particularly useful when running MANY tests as then most torture
failures will already fail the same functions over and over and make the
total operation painfully tedious.
Closes #4699
Diffstat (limited to 'tests/runtests.pl')
-rwxr-xr-x | tests/runtests.pl | 38 |
1 files changed, 35 insertions, 3 deletions
diff --git a/tests/runtests.pl b/tests/runtests.pl index 961cdb753..6ee2cca20 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -321,6 +321,8 @@ my %runcert; # cert file currently in use by an ssl running server my $torture; my $tortnum; my $tortalloc; +my $shallow; +my $shallowseed; ####################################################################### # logmsg is our general message logging subroutine. @@ -598,13 +600,34 @@ sub torture { return 0; } - logmsg " $count functions to make fail\n"; + my @ttests = (1 .. $count); + if($shallow && ($shallow < $count)) { + my $discard = scalar(@ttests) - $shallow; + my $percent = sprintf("%.2f%%", $shallow * 100 / scalar(@ttests));; + logmsg " $count functions found, but only fail $shallow ($percent)\n"; + while($discard) { + my $rm; + do { + # find a test to discard + $rm = rand(scalar(@ttests)); + } while(!$ttests[$rm]); + $ttests[$rm] = undef; + $discard--; + } + } + else { + logmsg " $count functions to make fail\n"; + } - for ( 1 .. $count ) { + for (@ttests) { my $limit = $_; my $fail; my $dumped_core; + if(!defined($limit)) { + # --shallow can undefine them + next; + } if($tortalloc && ($tortalloc != $limit)) { next; } @@ -5022,6 +5045,14 @@ while(@ARGV) { $tortalloc = $1; } } + elsif($ARGV[0] =~ /--shallow=(\d+)(,|)(\d*)/) { + # Fail no more than this amount per tests when running + # torture. + my ($num, $seed)=($1,$3); + $shallow=$num; + $shallowseed=$seed?$seed:1234; # get a real seed later + srand($shallowseed); # make it predictable + } elsif($ARGV[0] eq "-a") { # continue anyway, even if a test fail $anyway=1; @@ -5070,6 +5101,7 @@ while(@ARGV) { print <<EOHELP Usage: runtests.pl [options] [test selection(s)] -a continue even if a test fails + -am automake style output PASS/FAIL: [number] [name] -bN use base port number N for test servers (default $base) -c path use this curl executable -d display server debug info @@ -5085,7 +5117,7 @@ Usage: runtests.pl [options] [test selection(s)] -r run time statistics -rf full run time statistics -s short output - -am automake style output PASS/FAIL: [number] [name] + --shallow=[num](,seed) make the torture tests thinner -t[N] torture (simulate function failures); N means fail Nth function -v verbose output -vc path use this curl only to verify the existing servers |