selftest: Add separate command line for listing tests, allowing us of subunit-filter (which doesn't support subunit v2).

Change-Id: Icb877214a422c63fd38ef92425c7b4fe7ee19ad5
Signed-Off-By: Jelmer Vernooij <jelmer@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

6 files changed, 38 insertions, 17 deletions

diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index 34ae6056efd..ea63ee528b5 100755
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -574,10 +574,15 @@ sub read_testlist($)
 			$name =~ s/\n//g;
 			my $env = <IN>;
 			$env =~ s/\n//g;
+			my $loadlist;
+			if ($supports_loadlist) {
+				$loadlist = <IN>;
+				$loadlist =~ s/\n//g;
+			}
 			my $cmdline = <IN>;
 			$cmdline =~ s/\n//g;
 			if (should_run_test($name) == 1) {
-				push (@ret, [$name, $env, $cmdline, $supports_loadlist]);
+				push (@ret, [$name, $env, $cmdline, $loadlist]);
 			}
 		} else {
 			print;
@@ -937,11 +942,12 @@ $envvarstr
 	teardown_env($testenv_name);
 } elsif ($opt_list) {
 	foreach (@todo) {
-		my $cmd = $$_[2];
 		my $name = $$_[0];
 		my $envname = $$_[1];
+		my $cmd = $$_[2];
+		my $listcmd = $$_[3];
 
-		unless($cmd =~ /\$LISTOPT/) {
+		unless (defined($listcmd)) {
 			warn("Unable to list tests in $name");
 			# Rather than ignoring this testsuite altogether, just pretend the entire testsuite is
 			# a single "test".
@@ -949,14 +955,12 @@ $envvarstr
 			next;
 		}
 
-		$cmd =~ s/\$LISTOPT/--list/g;
-
-		system($cmd);
+		system($listcmd);
 
 		if ($? == -1) {
-			die("Unable to run $cmd: $!");
+			die("Unable to run $listcmd: $!");
 		} elsif ($? & 127) {
-			die(sprintf("%s died with signal %d, %s coredump\n", $cmd, ($? & 127),  ($? & 128) ? 'with' : 'without'));
+			die(sprintf("%s died with signal %d, %s coredump\n", $listcmd, ($? & 127),  ($? & 128) ? 'with' : 'without'));
 		}
 
 		my $exitcode = $? >> 8;
diff --git a/selftest/selftesthelpers.py b/selftest/selftesthelpers.py
index c66d92d4b3c..b2440054e60 100644
--- a/selftest/selftesthelpers.py
+++ b/selftest/selftesthelpers.py
@@ -82,6 +82,12 @@ if sub.returncode == 0:
     if sub.returncode == 0:
         tap2subunit = "tap2subunit"
 
+def to_subunit1(subunit_version):
+    if subunit_version == 1:
+        return ""
+    return " | " + subunit2to1
+
+
 def valgrindify(cmdline):
     """Run a command under valgrind, if $VALGRIND was set."""
     valgrind = os.getenv("VALGRIND")
@@ -90,7 +96,7 @@ def valgrindify(cmdline):
     return valgrind + " " + cmdline
 
 
-def plantestsuite(name, env, cmdline):
+def plantestsuite(name, env, cmdline, subunit_version=1):
     """Plan a test suite.
 
     :param name: Testsuite name
@@ -102,7 +108,9 @@ def plantestsuite(name, env, cmdline):
     print env
     if isinstance(cmdline, list):
         cmdline = " ".join(cmdline)
-    print "%s 2>&1 | " + add_prefix(name, env, "$LISTOPT" in cmdline)
+    if "$LISTOPT" in cmdline:
+        raise AssertionError("test %s supports --list, but not --load-list" % name)
+    print cmdline + " 2>&1 " + to_subunit1(subunit_version) + " | " + add_prefix(name, env)
 
 
 def add_prefix(prefix, env, support_list=False):
@@ -113,7 +121,7 @@ def add_prefix(prefix, env, support_list=False):
     return "%s/selftest/filter-subunit %s--fail-on-empty --prefix=\"%s.\" --suffix=\"(%s)\"" % (srcdir(), listopt, prefix, env)
 
 
-def plantestsuite_loadlist(name, env, cmdline):
+def plantestsuite_loadlist(name, env, cmdline, subunit_version=1):
     print "-- TEST-LOADLIST --"
     if env == "none":
         fullname = name
@@ -124,7 +132,10 @@ def plantestsuite_loadlist(name, env, cmdline):
     if isinstance(cmdline, list):
         cmdline = " ".join(cmdline)
     support_list = ("$LISTOPT" in cmdline)
-    print "%s $LOADLIST 2>&1 | %s" % (cmdline, add_prefix(name, env, support_list))
+    if not "$LISTOPT" in cmdline:
+        raise AssertionError("test %s supports --load-list, but not --list" % name)
+    print ("%s | %s" % (cmdline, add_prefix(name, env, support_list))).replace("$LISTOPT", "--list")
+    print cmdline + " $LOADLIST 2>&1 " + to_subunit1(subunit_version) + "| " + add_prefix(name, env, support_list)
 
 
 def skiptestsuite(name, reason):
diff --git a/selftest/testlist.py b/selftest/testlist.py
index 50ea31b62db..34c7a5d31c9 100644
--- a/selftest/testlist.py
+++ b/selftest/testlist.py
@@ -81,8 +81,12 @@ def read_testlist(inf, outf):
             supports_loadlist = l.startswith("-- TEST-LOADLIST")
             name = inf.readline().rstrip("\n")
             env = inf.readline().rstrip("\n")
+            if supports_loadlist:
+                loadlist = inf.readline().rstrip("\n")
+            else:
+                loadlist = None
             cmdline = inf.readline().rstrip("\n")
-            yield (name, env, cmdline, supports_loadlist)
+            yield (name, env, cmdline, loadlist)
         else:
             outf.write(l)
 
diff --git a/selftest/tests/test_testlist.py b/selftest/tests/test_testlist.py
index 7846eb1eff2..7007d6d35a2 100644
--- a/selftest/tests/test_testlist.py
+++ b/selftest/tests/test_testlist.py
@@ -67,14 +67,14 @@ class ReadTestlistTests(TestCase):
     def test_read_list(self):
         inf = StringIO("-- TEST --\nfoo\nbar\nbla\n")
         outf = StringIO()
-        self.assertEquals([('foo', 'bar', 'bla', False)],
+        self.assertEquals([('foo', 'bar', 'bla', None)],
                 list(read_testlist(inf, outf)))
         self.assertEquals("", outf.getvalue())
 
     def test_read_list_passes_through(self):
         inf = StringIO("MORENOISE\n-- TEST --\nfoo\nbar\nbla\nNOISE\n")
         outf = StringIO()
-        self.assertEquals([('foo', 'bar', 'bla', False)],
+        self.assertEquals([('foo', 'bar', 'bla', None)],
                 list(read_testlist(inf, outf)))
         self.assertEquals("MORENOISE\nNOISE\n", outf.getvalue())
 
@@ -143,6 +143,6 @@ class ReadTestListFileTests(TestCase):
             f.close()
         outf = StringIO()
         self.assertEquals(
-            [('data', 'env', 'cmd', False, False)],
+            [('data', 'env', 'cmd', None)],
             list(read_testlist_file(p, outf)))
         self.assertEquals("noise\n", outf.getvalue())
diff --git a/source4/scripting/bin/subunitrun b/source4/scripting/bin/subunitrun
index 99d53196eea..8d43f259a88 100755
--- a/source4/scripting/bin/subunitrun
+++ b/source4/scripting/bin/subunitrun
@@ -81,5 +81,7 @@ opts, args = parser.parse_args()
 if not getattr(opts, "listtests", False):
     lp = sambaopts.get_loadparm()
     samba.tests.cmdline_credentials = credopts.get_credentials(lp)
+if getattr(opts, 'load_list', None):
+    args.insert(0, "--load-list=%s" % opts.load_list)
 
 TestProgram(module=None, args=args, opts=subunitopts)
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index ef0d853584e..722e065fc30 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -436,7 +436,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex
     args += extra_args
     if name is None:
         name = module
-    plantestsuite(name, env, args)
+    plantestsuite_loadlist(name, env, args)
 
 planoldpythontestsuite("dc:local", "samba.tests.gensec", extra_args=['-U"$USERNAME%$PASSWORD"'])
 planoldpythontestsuite("none", "simple", extra_path=["%s/lib/tdb/python/tests" % srcdir()], name="tdb.python")