Non-deterministic test case

2 files changed, 99 insertions, 2 deletions

diff --git a/testtools/tests/samplecases.py b/testtools/tests/samplecases.py
index 557d575..556648d 100644
--- a/testtools/tests/samplecases.py
+++ b/testtools/tests/samplecases.py
@@ -6,6 +6,13 @@ These are primarily of use in testing the test framework.
 """
 
 from testtools import TestCase
+from testtools.matchers import (
+    AfterPreprocessing,
+    Contains,
+    Equals,
+    MatchesDict,
+    MatchesListwise,
+)
 
 
 class _NormalTest(TestCase):
@@ -31,6 +38,61 @@ class _TearDownFails(TestCase):
         1/0
 
 
+class _SetUpFailsOnGlobalState(TestCase):
+    """Fail to upcall setUp on first run. Fail to upcall tearDown after.
+
+    This simulates a test that fails to upcall in ``setUp`` if some global
+    state is broken, and fails to call ``tearDown`` at all.
+    """
+
+    first_run = True
+
+    def setUp(self):
+        if not self.first_run:
+            return
+        super(_SetUpFailsOnGlobalState, self).setUp()
+
+    def test_success(self):
+        pass
+
+    def tearDown(self):
+        if not self.first_run:
+            super(_SetUpFailsOnGlobalState, self).tearDown()
+        self.__class__.first_run = False
+
+    @classmethod
+    def make_scenario(cls):
+        case = cls('test_success')
+        return {
+            'case': case,
+            'expected_first_result': _test_error_traceback(
+                case, Contains('TestCase.tearDown was not called')),
+            'expected_second_result': _test_error_traceback(
+                case, Contains('TestCase.setUp was not called')),
+        }
+
+
+def _test_error_traceback(case, traceback_matcher):
+    """Match result log of single test that errored out.
+
+    ``traceback_matcher`` is applied to the text of the traceback.
+    """
+    return MatchesListwise([
+        Equals(('startTest', case)),
+        MatchesListwise([
+            Equals('addError'),
+            Equals(case),
+            MatchesDict({
+                'traceback': AfterPreprocessing(
+                    lambda x: x.as_text(),
+                    traceback_matcher,
+                )
+            })
+        ]),
+        Equals(('stopTest', case)),
+    ])
+
+
 """
 A list that can be used with testscenarios to test every deterministic sample
 case that we have.
@@ -41,3 +103,12 @@ deterministic_sample_cases_scenarios = [
     ('simple-failure-test', {'case': _NormalTest('test_failure')}),
     ('teardown-fails', {'case': _TearDownFails('test_success')}),
 ]
+
+
+"""
+A list that can be used with testscenarios to test every non-deterministic
+sample case that we have.
+"""
+nondeterministic_sample_cases_scenarios = [
+    ('setup-fails-global-state', _SetUpFailsOnGlobalState.make_scenario()),
+]
diff --git a/testtools/tests/test_testcase.py b/testtools/tests/test_testcase.py
index 5fbdbbc..ce16c19 100644
--- a/testtools/tests/test_testcase.py
+++ b/testtools/tests/test_testcase.py
@@ -54,7 +54,10 @@ from testtools.tests.helpers import (
     FullStackRunTest,
     LoggingResult,
     )
-from testtools.tests.samplecases import deterministic_sample_cases_scenarios
+from testtools.tests.samplecases import (
+    deterministic_sample_cases_scenarios,
+    nondeterministic_sample_cases_scenarios,
+)
 
 
 class TestPlaceHolder(TestCase):
@@ -1270,7 +1273,7 @@ class TestSetupTearDown(TestCase):
                 ELLIPSIS))
 
 
-class TestRunTwice(TestCase):
+class TestRunTwiceDeterminstic(TestCase):
     """Can we run the same test case twice?"""
 
     # XXX: Reviewer, please note that all of the other test cases in this
@@ -1292,6 +1295,29 @@ class TestRunTwice(TestCase):
         self.assertEqual(first_result._events, second_result._events)
 
 
+class TestRunTwiceNondeterministic(TestCase):
+    """Can we run the same test case twice?
+
+    Separate suite for non-deterministic tests, which require more complicated
+    assertions and scenarios.
+    """
+
+    run_tests_with = FullStackRunTest
+
+    scenarios = nondeterministic_sample_cases_scenarios
+
+    def test_runTwice(self):
+        test = self.case
+        first_result = ExtendedTestResult()
+        test.run(first_result)
+        second_result = ExtendedTestResult()
+        test.run(second_result)
+        self.expectThat(
+            first_result._events, self.expected_first_result)
+        self.assertThat(
+            second_result._events, self.expected_second_result)
+
+
 require_py27_minimum = skipIf(
     sys.version < '2.7',
     "Requires python 2.7 or greater"