selftest: use an additional directory of knownfail/flapping files

This makes it easier to add a temporary knownfail to cover a patch
series.

Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Sat Jun  3 13:55:41 CEST 2017 on sn-devel-144

5 files changed, 62 insertions, 22 deletions

diff --git a/selftest/filter-subunit b/selftest/filter-subunit
index c3aba734ade..d71112296f2 100755
--- a/selftest/filter-subunit
+++ b/selftest/filter-subunit
@@ -27,11 +27,12 @@ sys.path.insert(0, "bin/python")
 import subunithelper
 
 parser = optparse.OptionParser("filter-subunit [options] < instream > outstream")
-parser.add_option("--expected-failures", type="string",
-    help="File containing list of regexes matching tests to consider known "
-         "failures")
-parser.add_option("--flapping", type="string",
-    help="File containing list of flapping tests, of which to ignore results.")
+parser.add_option("--expected-failures", type="string", action="append",
+    help=("File or directory containing lists of regexes matching tests "
+          "to consider known failures"))
+parser.add_option("--flapping", type="string", action="append",
+    help=("File or directory containing lists of flapping tests, "
+          "of which to ignore results."))
 parser.add_option("--strip-passed-output", action="store_true",
     help="Whether to strip output from tests that passed")
 parser.add_option("--fail-immediately", action="store_true",
@@ -66,13 +67,13 @@ if opts.perf_test_output:
         sys.exit(1)
 
 if opts.expected_failures:
-    expected_failures = subunithelper.read_test_regexes(opts.expected_failures)
+    expected_failures = subunithelper.read_test_regexes(*opts.expected_failures)
 else:
     expected_failures = {}
 
 
 if opts.flapping:
-    flapping = subunithelper.read_test_regexes(opts.flapping)
+    flapping = subunithelper.read_test_regexes(*opts.flapping)
 else:
     flapping = {}
 
diff --git a/selftest/flapping.d/README b/selftest/flapping.d/README
new file mode 100644
index 00000000000..cf80129f242
--- /dev/null
+++ b/selftest/flapping.d/README
@@ -0,0 +1,14 @@
+# Files in this directory contain lists of regular expressions
+# matching the names of tests that are that are flapping. In other
+# words, they sometimes succeed and sometimes fail, depending on
+# external factors.
+#
+# "make test" will not report failures or successes for tests listed here.
+#
+# DO NOT ADD TESTS HERE UNLESS THEY ARE ACTUALLY FLAPPING
+#
+# It is much better to add known failing tests to 'knownfail', so the
+# test system can warn when they actually start passing.
+#
+# Empty lines and lines begining with '#' are ignored.
+# Please don't add tests to this README!
diff --git a/selftest/knownfail.d/README b/selftest/knownfail.d/README
new file mode 100644
index 00000000000..d333231a21e
--- /dev/null
+++ b/selftest/knownfail.d/README
@@ -0,0 +1,8 @@
+# Files in this directory contain lists of regular expressions
+# matching the names of tests that are temporarily expected to fail.
+#
+# "make test" will not report failures for tests listed here and will consider
+# a successful run for any of these tests an error.
+#
+# Empty lines and lines begining with '#' are ignored.
+# Please don't add tests to this README!
diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py
index c17036defba..fab7d6f0b41 100644
--- a/selftest/subunithelper.py
+++ b/selftest/subunithelper.py
@@ -20,6 +20,7 @@ __all__ = ['parse_results']
 import datetime
 import re
 import sys
+import os
 from samba import subunit
 from samba.subunit.run import TestProtocolClient
 from samba.subunit import iso8601
@@ -228,21 +229,33 @@ class SubunitOps(TestProtocolClient,TestsuiteEnabledTestResult):
         self._stream.write(msg)
 
 
-def read_test_regexes(name):
+def read_test_regexes(*names):
     ret = {}
-    f = open(name, 'r')
-    try:
-        for l in f:
-            l = l.strip()
-            if l == "" or l[0] == "#":
-                continue
-            if "#" in l:
-                (regex, reason) = l.split("#", 1)
-                ret[regex.strip()] = reason.strip()
-            else:
-                ret[l] = None
-    finally:
-        f.close()
+    files = []
+    for name in names:
+        # if we are given a directory, we read all the files it contains
+        # (except the ones that end with "~").
+        if os.path.isdir(name):
+            files.extend([os.path.join(name, x)
+                          for x in os.listdir(name)
+                          if x[-1] != '~'])
+        else:
+            files.append(name)
+
+    for filename in files:
+        f = open(filename, 'r')
+        try:
+            for l in f:
+                l = l.strip()
+                if l == "" or l[0] == "#":
+                    continue
+                if "#" in l:
+                    (regex, reason) = l.split("#", 1)
+                    ret[regex.strip()] = reason.strip()
+                else:
+                    ret[l] = None
+        finally:
+            f.close()
     return ret
 
 
diff --git a/selftest/wscript b/selftest/wscript
index d8094af1cdc..9f1fd4d3d71 100644
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -121,7 +121,11 @@ def cmd_testonly(opt):
             env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit-json --prefix=${SELFTEST_PREFIX}'
         else:
             env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
-    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping'
+    env.FILTER_XFAIL = ('${PYTHON} -u ${srcdir}/selftest/filter-subunit '
+                        '--expected-failures=${srcdir}/selftest/knownfail '
+                        '--expected-failures=${srcdir}/selftest/knownfail.d '
+                        '--flapping=${srcdir}/selftest/flapping '
+                        '--flapping=${srcdir}/selftest/flapping.d')
 
     if Options.options.FAIL_IMMEDIATELY:
         env.FILTER_XFAIL += ' --fail-immediately'