testsuite: Identify framework failures in testsuite summary

Currently the testsuite driver tells you how many tests failed due to a
framework failure but you need to manually grep through the testsuite
output to identify which ones.

Test Plan: Validate with, e.g., a timing out testcase

Reviewers: austin, thomie

Reviewed By: austin, thomie

Differential Revision: https://phabricator.haskell.org/D2026

GHC Trac Issues: #11165

1 files changed, 14 insertions, 0 deletions

diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 0e2ba49dbf..bba351602b 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -2265,6 +2265,10 @@ def summary(t, file, short=False):
         file.write('Unexpected stat failures:\n')
         printFailingTestInfosSummary(file, t.unexpected_stat_failures)
 
+    if t.n_framework_failures > 0:
+        file.write('Test framework failures:\n')
+        printFrameworkFailureSummary(file, t.framework_failures)
+
     if config.check_files_written:
         checkForFilesWrittenProblems(file)
 
@@ -2310,6 +2314,16 @@ def printFailingTestInfosSummary(file, testInfos):
                           ' (' + ','.join(testInfos[directory][test][reason]) + ')\n')
     file.write('\n')
 
+def printFrameworkFailureSummary(file, testInfos):
+    names = list(testInfos.keys())
+    names.sort()
+    maxNameLen = max(len(n) for n in names)
+    for name in names:
+        ways = testInfos[name]
+        file.write('   ' + name.ljust(maxNameLen + 2) + \
+                   ' (' + ','.join(ways) + ')\n')
+    file.write('\n')
+
 def modify_lines(s, f):
     s = '\n'.join([f(l) for l in s.splitlines()])
     if s and s[-1] != '\n':