Testsuite: framework failure improvements (#11165)

* add framework failures to unexpected results list
* report errors in .T files as framework failures (show in summary)
* don't report missing tests when framework failures in .T files

2 files changed, 18 insertions, 9 deletions

diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index efd8b090e7..8a11f4478e 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -298,20 +298,26 @@ def cleanup_and_exit(exitcode):
     exit(exitcode)
 
 # First collect all the tests to be run
+t_files_ok = True
 for file in t_files:
     if_verbose(2, '====> Scanning %s' % file)
     newTestDir(tempdir, os.path.dirname(file))
     try:
         exec(open(file).read())
-    except Exception:
-        print('*** framework failure: found an error while executing ', file, ':')
-        t.n_framework_failures = t.n_framework_failures + 1
+    except Exception as e:
         traceback.print_exc()
+        framework_fail(file, '', str(e))
+        t_files_ok = False
 
-if config.only:
-    # See Note [Mutating config.only]
-    sys.stderr.write("ERROR: tests not found: {0}\n".format(list(config.only)))
-    cleanup_and_exit(1)
+for name in config.only:
+    if t_files_ok:
+        # See Note [Mutating config.only]
+        framework_fail(name, '', 'test not found')
+    else:
+        # Let user fix .T file errors before reporting on unfound tests.
+        # The reson the test can not be found is likely because of those
+        # .T file errors.
+        pass
 
 if config.list_broken:
     global brokens
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index e9548c75fd..6d4d77c70c 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1841,7 +1841,9 @@ def findTFiles(roots):
 def summary(t, file, short=False):
 
     file.write('\n')
-    printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
+    printUnexpectedTests(file,
+        [t.unexpected_passes, t.unexpected_failures,
+         t.unexpected_stat_failures, t.framework_failures])
 
     if short:
         # Only print the list of unexpected tests above.
@@ -1897,7 +1899,8 @@ def summary(t, file, short=False):
 
 def printUnexpectedTests(file, testInfoss):
     unexpected = {name for testInfos in testInfoss
-                       for (_, name, _, _) in testInfos}
+                       for (_, name, _, _) in testInfos
+                       if not name.endswith('.T')}
     if unexpected:
         file.write('Unexpected results from:\n')
         file.write('TEST="' + ' '.join(unexpected) + '"\n')