runtests.pl: add an -E option to specify an exclude file

It can contain additional restraints for test numbers,
keywords and tools.

The idea is to let third parties like the Privoxy project
distribute an exclude file with their tarballs that specifies
which curl tests are not expected to work when using Privoxy
as a proxy, without having to fork the whole curl test suite.

The syntax could be changed to be extendable and maybe
more closely reflect the "curl test" syntax. Currently
it's a bunch of lines like these:

test:$TESTNUMBER:Reason why this test with number $TESTNUMBER should be skipped
keyword:$KEYWORD:Reason why tests whose keywords contain the $KEYWORD should be skipped
tool:$TOOL:Reason why tests with tools that contain $TOOL should be skipped

To specify multiple $TESTNUMBERs, $KEYWORDs and $TOOLs
on a single line, split them with commas.

4 files changed, 90 insertions, 1 deletions

diff --git a/tests/data/DISABLED b/tests/data/DISABLED
index 1806a5dbe..2ff0fc49a 100644
--- a/tests/data/DISABLED
+++ b/tests/data/DISABLED
@@ -11,6 +11,10 @@
 836
 882
 938
+# test 1182 kills the test servers as a side effect
+# of running runtests.pl as a child of itself sharing
+# some of the directories.
+1182
 1209
 1211
 # fnmatch differences are just too common to make testing them sensible
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index 485fd72b6..5b60d1606 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -137,7 +137,7 @@ test1144 test1145 test1146 test1147 test1148 test1149 test1150 test1151 \
 test1152 test1153 test1154 test1155 test1156 test1157 test1158 test1159 \
 test1160 test1161 test1162 test1163 test1164 test1165 test1166 test1167 \
 test1168 test1169 test1170 test1171 test1172 test1173 test1174 test1175 \
-test1176 test1177 test1178 test1179 test1180 test1181 \
+test1176 test1177 test1178 test1179 test1180 test1181 test1182 \
 \
 test1188 \
 \
diff --git a/tests/data/test1182 b/tests/data/test1182
new file mode 100644
index 000000000..0e5a672d2
--- /dev/null
+++ b/tests/data/test1182
@@ -0,0 +1,36 @@
+<testcase>
+<info>
+<keywords>
+runtests.pl
+</keywords>
+</info>
+
+#
+# Client-side
+<client>
+<server>
+none
+</server>
+<name>
+Verify that runtests.pl accepts an exclude file with the -E option
+</name>
+
+<command type="perl">
+%SRCDIR/runtests.pl -o TESTDIR=%SRCDIR/log/data -o LOGDIR=%SRCDIR/log/log -E %SRCDIR/log/test1182-exclude-file 1
+</command>
+<precheck>
+mkdir %SRCDIR/log/data ; mkdir %SRCDIR/log/log; cp %SRCDIR/data/test1 %SRCDIR/log/data; echo 'test:1:Test should not run for unit test 1182' > %SRCDIR/log/test1182-exclude-file
+</precheck>
+<postcheck>
+grep -q "Test should not run for unit test 1182" %SRCDIR/log/stdout1182
+</postcheck>
+
+</client>
+
+<verify>
+<errorcode>
+1
+</errorcode>
+</verify>
+
+</testcase>
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 27800d817..8af6bfa97 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -165,6 +165,7 @@ my $HTTPUNIXPATH;        # HTTP server Unix domain socket path
 
 my $use_external_proxy = 0;
 my $proxy_address;
+my %custom_skip_reasons;
 
 my $SSHSRVMD5 = "[uninitialized]"; # MD5 of ssh server public key
 my $VERSION;             # curl's reported version number
@@ -3597,6 +3598,31 @@ sub singletest {
         }
     }
 
+    if (!$why && defined $custom_skip_reasons{test}{$testnum}) {
+        $why = $custom_skip_reasons{test}{$testnum};
+    }
+
+    if (!$why && defined $custom_skip_reasons{tool}) {
+        foreach my $tool (getpart("client", "tool")) {
+            foreach my $tool_skip_pattern (keys %{$custom_skip_reasons{tool}}) {
+                if ($tool =~ /$tool_skip_pattern/i) {
+                    $why = $custom_skip_reasons{tool}{$tool_skip_pattern};
+                }
+            }
+        }
+    }
+
+    if (!$why && defined $custom_skip_reasons{keyword}) {
+        foreach my $keyword (getpart("info", "keywords")) {
+            foreach my $keyword_skip_pattern (keys %{$custom_skip_reasons{keyword}}) {
+                if ($keyword =~ /$keyword_skip_pattern/i) {
+                    $why = $custom_skip_reasons{keyword}{$keyword_skip_pattern};
+                }
+            }
+        }
+    }
+
+
     # test definition may instruct to (un)set environment vars
     # this is done this early, so that the precheck can use environment
     # variables and still bail out fine on errors
@@ -5328,6 +5354,28 @@ while(@ARGV) {
         # run the tests cases event based if possible
         $run_event_based=1;
     }
+    elsif($ARGV[0] eq "-E") {
+        # load additional reasons to skip tests
+        shift @ARGV;
+        my $exclude_file = $ARGV[0];
+        open(my $fd, "<", $exclude_file) or die "Couldn't open '$exclude_file': $!";
+        while(my $line = <$fd>) {
+            next if ($line =~ /^#/);
+            chomp $line;
+            my ($type, $patterns, $skip_reason) = split(/\s*:\s*/, $line, 3);
+
+            die "Unsupported type: $type\n" if($type !~ /^keyword|test|tool$/);
+
+            foreach my $pattern (split(/,/, $patterns)) {
+                if($type =~ /^test$/) {
+                    # Strip leading zeros in the test number
+                    $pattern = int($pattern);
+                }
+                $custom_skip_reasons{$type}{$pattern} = $skip_reason;
+            }
+        }
+        close($fd);
+    }
     elsif ($ARGV[0] eq "-g") {
         # run this test with gdb
         $gdbthis=1;
@@ -5439,6 +5487,7 @@ Usage: runtests.pl [options] [test selection(s)]
   -c path  use this curl executable
   -d       display server debug info
   -e       event-based execution
+  -E file  load the specified file to exclude certain tests
   -g       run the test case with gdb
   -gw      run the test case with gdb as a windowed application
   -h       this help text