bpo-30856: Update TestResult early, without buffering in _Outcome (GH-28180)

TestResult methods addFailure(), addError(), addSkip() and
addSubTest() are now called immediately after raising an exception
in test or finishing a subtest.  Previously they were called only
after finishing the test clean up.

5 files changed, 40 insertions, 29 deletions

diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py
index 156b252912..ee4c0b354f 100644
--- a/Lib/unittest/test/test_case.py
+++ b/Lib/unittest/test/test_case.py
@@ -197,8 +197,8 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
                 super(Foo, self).test()
                 raise RuntimeError('raised by Foo.test')
 
-        expected = ['startTest', 'setUp', 'test', 'tearDown',
-                    'addError', 'stopTest']
+        expected = ['startTest', 'setUp', 'test',
+                    'addError', 'tearDown', 'stopTest']
         Foo(events).run(result)
         self.assertEqual(events, expected)
 
@@ -216,7 +216,7 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
                 raise RuntimeError('raised by Foo.test')
 
         expected = ['startTestRun', 'startTest', 'setUp', 'test',
-                    'tearDown', 'addError', 'stopTest', 'stopTestRun']
+                    'addError', 'tearDown', 'stopTest', 'stopTestRun']
         Foo(events).run()
         self.assertEqual(events, expected)
 
@@ -236,8 +236,8 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
                 super(Foo, self).test()
                 self.fail('raised by Foo.test')
 
-        expected = ['startTest', 'setUp', 'test', 'tearDown',
-                    'addFailure', 'stopTest']
+        expected = ['startTest', 'setUp', 'test',
+                    'addFailure', 'tearDown', 'stopTest']
         Foo(events).run(result)
         self.assertEqual(events, expected)
 
@@ -252,7 +252,7 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
                 self.fail('raised by Foo.test')
 
         expected = ['startTestRun', 'startTest', 'setUp', 'test',
-                    'tearDown', 'addFailure', 'stopTest', 'stopTestRun']
+                    'addFailure', 'tearDown', 'stopTest', 'stopTestRun']
         events = []
         Foo(events).run()
         self.assertEqual(events, expected)
@@ -353,10 +353,10 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
     def test_run_call_order__subtests(self):
         events = []
         result = LoggingResult(events)
-        expected = ['startTest', 'setUp', 'test', 'tearDown',
+        expected = ['startTest', 'setUp', 'test',
                     'addSubTestFailure', 'addSubTestSuccess',
                     'addSubTestFailure', 'addSubTestFailure',
-                    'addSubTestSuccess', 'addError', 'stopTest']
+                    'addSubTestSuccess', 'addError', 'tearDown', 'stopTest']
         self._check_call_order__subtests(result, events, expected)
 
     def test_run_call_order__subtests_legacy(self):
@@ -364,8 +364,8 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
         # text execution stops after the first subtest failure.
         events = []
         result = LegacyLoggingResult(events)
-        expected = ['startTest', 'setUp', 'test', 'tearDown',
-                    'addFailure', 'stopTest']
+        expected = ['startTest', 'setUp', 'test',
+                    'addFailure', 'tearDown', 'stopTest']
         self._check_call_order__subtests(result, events, expected)
 
     def _check_call_order__subtests_success(self, result, events, expected_events):
@@ -386,9 +386,9 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
         result = LoggingResult(events)
         # The 6 subtest successes are individually recorded, in addition
         # to the whole test success.
-        expected = (['startTest', 'setUp', 'test', 'tearDown']
+        expected = (['startTest', 'setUp', 'test']
                     + 6 * ['addSubTestSuccess']
-                    + ['addSuccess', 'stopTest'])
+                    + ['tearDown', 'addSuccess', 'stopTest'])
         self._check_call_order__subtests_success(result, events, expected)
 
     def test_run_call_order__subtests_success_legacy(self):
@@ -413,8 +413,8 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
                     self.fail('failure')
                 self.fail('failure')
 
-        expected = ['startTest', 'setUp', 'test', 'tearDown',
-                    'addSubTestFailure', 'stopTest']
+        expected = ['startTest', 'setUp', 'test',
+                    'addSubTestFailure', 'tearDown', 'stopTest']
         Foo(events).run(result)
         self.assertEqual(events, expected)
 
diff --git a/Lib/unittest/test/test_functiontestcase.py b/Lib/unittest/test/test_functiontestcase.py
index c5f2bcbe74..4971729880 100644
--- a/Lib/unittest/test/test_functiontestcase.py
+++ b/Lib/unittest/test/test_functiontestcase.py
@@ -58,8 +58,8 @@ class Test_FunctionTestCase(unittest.TestCase):
         def tearDown():
             events.append('tearDown')
 
-        expected = ['startTest', 'setUp', 'test', 'tearDown',
-                    'addError', 'stopTest']
+        expected = ['startTest', 'setUp', 'test',
+                    'addError', 'tearDown', 'stopTest']
         unittest.FunctionTestCase(test, setUp, tearDown).run(result)
         self.assertEqual(events, expected)
 
@@ -84,8 +84,8 @@ class Test_FunctionTestCase(unittest.TestCase):
         def tearDown():
             events.append('tearDown')
 
-        expected = ['startTest', 'setUp', 'test', 'tearDown',
-                    'addFailure', 'stopTest']
+        expected = ['startTest', 'setUp', 'test',
+                    'addFailure', 'tearDown', 'stopTest']
         unittest.FunctionTestCase(test, setUp, tearDown).run(result)
         self.assertEqual(events, expected)
 
diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py
index 3b9da12776..c4a49b4875 100644
--- a/Lib/unittest/test/test_result.py
+++ b/Lib/unittest/test/test_result.py
@@ -816,7 +816,8 @@ class TestOutputBuffering(unittest.TestCase):
         self.assertEqual(str(test_case), description)
         self.assertIn('ValueError: bad cleanup2', formatted_exc)
         self.assertNotIn('TypeError', formatted_exc)
-        self.assertIn(expected_out, formatted_exc)
+        self.assertIn('\nStdout:\nset up\ndo cleanup2\n', formatted_exc)
+        self.assertNotIn('\ndo cleanup1\n', formatted_exc)
         test_case, formatted_exc = result.errors[1]
         self.assertEqual(str(test_case), description)
         self.assertIn('TypeError: bad cleanup1', formatted_exc)
@@ -847,13 +848,16 @@ class TestOutputBuffering(unittest.TestCase):
         self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
         self.assertNotIn('ValueError', formatted_exc)
         self.assertNotIn('TypeError', formatted_exc)
-        self.assertIn(expected_out, formatted_exc)
+        self.assertIn('\nStdout:\nset up\n', formatted_exc)
+        self.assertNotIn('\ndo cleanup2\n', formatted_exc)
+        self.assertNotIn('\ndo cleanup1\n', formatted_exc)
         test_case, formatted_exc = result.errors[1]
         self.assertEqual(str(test_case), description)
         self.assertIn('ValueError: bad cleanup2', formatted_exc)
         self.assertNotIn('ZeroDivisionError', formatted_exc)
         self.assertNotIn('TypeError', formatted_exc)
-        self.assertIn(expected_out, formatted_exc)
+        self.assertIn('\nStdout:\nset up\ndo cleanup2\n', formatted_exc)
+        self.assertNotIn('\ndo cleanup1\n', formatted_exc)
         test_case, formatted_exc = result.errors[2]
         self.assertEqual(str(test_case), description)
         self.assertIn('TypeError: bad cleanup1', formatted_exc)
@@ -887,13 +891,16 @@ class TestOutputBuffering(unittest.TestCase):
         self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
         self.assertNotIn('ValueError', formatted_exc)
         self.assertNotIn('TypeError', formatted_exc)
-        self.assertIn(expected_out, formatted_exc)
+        self.assertIn('\nStdout:\nset up\ntear down\n', formatted_exc)
+        self.assertNotIn('\ndo cleanup2\n', formatted_exc)
+        self.assertNotIn('\ndo cleanup1\n', formatted_exc)
         test_case, formatted_exc = result.errors[1]
         self.assertEqual(str(test_case), description)
         self.assertIn('ValueError: bad cleanup2', formatted_exc)
         self.assertNotIn('ZeroDivisionError', formatted_exc)
         self.assertNotIn('TypeError', formatted_exc)
-        self.assertIn(expected_out, formatted_exc)
+        self.assertIn('\nStdout:\nset up\ntear down\ndo cleanup2\n', formatted_exc)
+        self.assertNotIn('\ndo cleanup1\n', formatted_exc)
         test_case, formatted_exc = result.errors[2]
         self.assertEqual(str(test_case), description)
         self.assertIn('TypeError: bad cleanup1', formatted_exc)
diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py
index 6dc3cf6aee..bcf7538c26 100644
--- a/Lib/unittest/test/test_runner.py
+++ b/Lib/unittest/test/test_runner.py
@@ -78,7 +78,8 @@ class TestCleanUp(unittest.TestCase):
                 pass
 
         test = TestableTest('testNothing')
-        outcome = test._outcome = _Outcome()
+        result = unittest.TestResult()
+        outcome = test._outcome = _Outcome(result=result)
 
         CleanUpExc = Exception('foo')
         exc2 = Exception('bar')
@@ -94,10 +95,13 @@ class TestCleanUp(unittest.TestCase):
         self.assertFalse(test.doCleanups())
         self.assertFalse(outcome.success)
 
-        ((_, (Type1, instance1, _)),
-         (_, (Type2, instance2, _))) = reversed(outcome.errors)
-        self.assertEqual((Type1, instance1), (Exception, CleanUpExc))
-        self.assertEqual((Type2, instance2), (Exception, exc2))
+        (_, msg2), (_, msg1) = result.errors
+        self.assertIn('in cleanup1', msg1)
+        self.assertIn('raise CleanUpExc', msg1)
+        self.assertIn('Exception: foo', msg1)
+        self.assertIn('in cleanup2', msg2)
+        self.assertIn('raise exc2', msg2)
+        self.assertIn('Exception: bar', msg2)
 
     def testCleanupInRun(self):
         blowUp = False
diff --git a/Lib/unittest/test/test_skipping.py b/Lib/unittest/test/test_skipping.py
index 7cb9d33f5e..64ceeae37e 100644
--- a/Lib/unittest/test/test_skipping.py
+++ b/Lib/unittest/test/test_skipping.py
@@ -197,7 +197,7 @@ class Test_TestSkipping(unittest.TestCase):
         result = LoggingResult(events)
         test = Foo("test_skip_me")
         self.assertIs(test.run(result), result)
-        self.assertEqual(events, ['startTest', 'addSkip', 'addFailure', 'stopTest'])
+        self.assertEqual(events, ['startTest', 'addFailure', 'addSkip', 'stopTest'])
         self.assertEqual(result.skipped, [(test, "skip")])
 
     def test_skipping_and_fail_in_cleanup(self):