Handle different SyntaxError output in testtools 2.5.0

testtools 2.5.0 only supports Python 3, and so switched from traceback2
to the standard library's traceback.  However, this includes the fix for
https://bugs.python.org/issue24695, and so doesn't produce a traceback
header when there's no traceback, in particular for `SyntaxError`
exceptions.  Adjust the tests to tolerate this.

See also https://github.com/zopefoundation/zope.testrunner/issues/125.

2 files changed, 98 insertions, 42 deletions

diff --git a/python/subunit/tests/__init__.py b/python/subunit/tests/__init__.py
index c6599f7..4c8b2ae 100644
--- a/python/subunit/tests/__init__.py
+++ b/python/subunit/tests/__init__.py
@@ -23,6 +23,7 @@ from testscenarios import generate_scenarios
 # Before the test module imports to avoid circularity.
 # For testing: different pythons have different str() implementations.
 _remote_exception_repr = "testtools.testresult.real._StringException"
+_remote_exception_repr_chunked = "34\r\n" + _remote_exception_repr + ": boo qux\n0\r\n"
 _remote_exception_str = "Traceback (most recent call last):\ntesttools.testresult.real._StringException"
 _remote_exception_str_chunked = "57\r\n" + _remote_exception_str + ": boo qux\n0\r\n"
 
diff --git a/python/subunit/tests/test_test_protocol.py b/python/subunit/tests/test_test_protocol.py
index b0691f2..70e3564 100644
--- a/python/subunit/tests/test_test_protocol.py
+++ b/python/subunit/tests/test_test_protocol.py
@@ -41,11 +41,12 @@ except ImportError:
         Python27TestResult,
         ExtendedTestResult,
         )
-from testtools.matchers import Contains
+from testtools.matchers import Contains, Equals, MatchesAny
 
 import subunit
 from subunit.tests import (
     _remote_exception_repr,
+    _remote_exception_repr_chunked,
     _remote_exception_str,
     _remote_exception_str_chunked,
     )
@@ -1194,6 +1195,11 @@ class TestIsolatedTestSuite(TestCase):
         self.assertEqual(self.SampleTestToIsolate.TEST, False)
 
 
+# A number of these tests produce different output depending on the
+# testtools version.  testtools < 2.5.0 used traceback2, which incorrectly
+# included the traceback header even for an exception with no traceback.
+# testtools 2.5.0 switched to the Python 3 standard library's traceback
+# module, which fixes this bug.  See https://bugs.python.org/issue24695.
 class TestTestProtocolClient(TestCase):
 
     def setUp(self):
@@ -1249,72 +1255,121 @@ class TestTestProtocolClient(TestCase):
         """Test addFailure on a TestProtocolClient."""
         self.protocol.addFailure(
             self.test, subunit.RemoteError(_u("boo qux")))
-        self.assertEqual(
-            self.io.getvalue(),
-            _b(('failure: %s [\n' + _remote_exception_str + ': boo qux\n]\n')
-            % self.test.id()))
+        self.assertThat(self.io.getvalue(), MatchesAny(
+            # testtools < 2.5.0
+            Equals(_b((
+                'failure: %s [\n' +
+                _remote_exception_str + ': boo qux\n' +
+                ']\n') % self.test.id())),
+            # testtools >= 2.5.0
+            Equals(_b((
+                'failure: %s [\n' +
+                _remote_exception_repr + ': boo qux\n' +
+                ']\n') % self.test.id()))))
 
     def test_add_failure_details(self):
         """Test addFailure on a TestProtocolClient with details."""
         self.protocol.addFailure(
             self.test, details=self.sample_tb_details)
-        self.assertEqual(
-            self.io.getvalue(),
-            _b(("failure: %s [ multipart\n"
-            "Content-Type: text/plain\n"
-            "something\n"
-            "F\r\nserialised\nform0\r\n"
-            "Content-Type: text/x-traceback;charset=utf8,language=python\n"
-            "traceback\n" + _remote_exception_str_chunked +
-            "]\n") % self.test.id()))
+        self.assertThat(self.io.getvalue(), MatchesAny(
+            # testtools < 2.5.0
+            Equals(_b((
+                "failure: %s [ multipart\n"
+                "Content-Type: text/plain\n"
+                "something\n"
+                "F\r\nserialised\nform0\r\n"
+                "Content-Type: text/x-traceback;charset=utf8,language=python\n"
+                "traceback\n" + _remote_exception_str_chunked +
+                "]\n") % self.test.id())),
+            # testtools >= 2.5.0
+            Equals(_b((
+                "failure: %s [ multipart\n"
+                "Content-Type: text/plain\n"
+                "something\n"
+                "F\r\nserialised\nform0\r\n"
+                "Content-Type: text/x-traceback;charset=utf8,language=python\n"
+                "traceback\n" + _remote_exception_repr_chunked +
+                "]\n") % self.test.id()))))
 
     def test_add_error(self):
         """Test stopTest on a TestProtocolClient."""
         self.protocol.addError(
             self.test, subunit.RemoteError(_u("phwoar crikey")))
-        self.assertEqual(
-            self.io.getvalue(),
-            _b(('error: %s [\n' +
-            _remote_exception_str + ": phwoar crikey\n"
-            "]\n") % self.test.id()))
+        self.assertThat(self.io.getvalue(), MatchesAny(
+            # testtools < 2.5.0
+            Equals(_b((
+                'error: %s [\n' +
+                _remote_exception_str + ": phwoar crikey\n"
+                "]\n") % self.test.id())),
+            # testtools >= 2.5.0
+            Equals(_b((
+                'error: %s [\n' +
+                _remote_exception_repr + ": phwoar crikey\n"
+                "]\n") % self.test.id()))))
 
     def test_add_error_details(self):
         """Test stopTest on a TestProtocolClient with details."""
         self.protocol.addError(
             self.test, details=self.sample_tb_details)
-        self.assertEqual(
-            self.io.getvalue(),
-            _b(("error: %s [ multipart\n"
-            "Content-Type: text/plain\n"
-            "something\n"
-            "F\r\nserialised\nform0\r\n"
-            "Content-Type: text/x-traceback;charset=utf8,language=python\n"
-            "traceback\n" + _remote_exception_str_chunked +
-            "]\n") % self.test.id()))
+        self.assertThat(self.io.getvalue(), MatchesAny(
+            # testtools < 2.5.0
+            Equals(_b((
+                "error: %s [ multipart\n"
+                "Content-Type: text/plain\n"
+                "something\n"
+                "F\r\nserialised\nform0\r\n"
+                "Content-Type: text/x-traceback;charset=utf8,language=python\n"
+                "traceback\n" + _remote_exception_str_chunked +
+                "]\n") % self.test.id())),
+            # testtools >= 2.5.0
+            Equals(_b((
+                "error: %s [ multipart\n"
+                "Content-Type: text/plain\n"
+                "something\n"
+                "F\r\nserialised\nform0\r\n"
+                "Content-Type: text/x-traceback;charset=utf8,language=python\n"
+                "traceback\n" + _remote_exception_repr_chunked +
+                "]\n") % self.test.id()))))
 
     def test_add_expected_failure(self):
         """Test addExpectedFailure on a TestProtocolClient."""
         self.protocol.addExpectedFailure(
             self.test, subunit.RemoteError(_u("phwoar crikey")))
-        self.assertEqual(
-            self.io.getvalue(),
-            _b(('xfail: %s [\n' +
-            _remote_exception_str + ": phwoar crikey\n"
-            "]\n") % self.test.id()))
+        self.assertThat(self.io.getvalue(), MatchesAny(
+            # testtools < 2.5.0
+            Equals(_b((
+                'xfail: %s [\n' +
+                _remote_exception_str + ": phwoar crikey\n"
+                "]\n") % self.test.id())),
+            # testtools >= 2.5.0
+            Equals(_b((
+                'xfail: %s [\n' +
+                _remote_exception_repr + ": phwoar crikey\n"
+                "]\n") % self.test.id()))))
 
     def test_add_expected_failure_details(self):
         """Test addExpectedFailure on a TestProtocolClient with details."""
         self.protocol.addExpectedFailure(
             self.test, details=self.sample_tb_details)
-        self.assertEqual(
-            self.io.getvalue(),
-            _b(("xfail: %s [ multipart\n"
-            "Content-Type: text/plain\n"
-            "something\n"
-            "F\r\nserialised\nform0\r\n"
-            "Content-Type: text/x-traceback;charset=utf8,language=python\n"
-            "traceback\n" + _remote_exception_str_chunked +
-            "]\n") % self.test.id()))
+        self.assertThat(self.io.getvalue(), MatchesAny(
+            # testtools < 2.5.0
+            Equals(_b((
+                "xfail: %s [ multipart\n"
+                "Content-Type: text/plain\n"
+                "something\n"
+                "F\r\nserialised\nform0\r\n"
+                "Content-Type: text/x-traceback;charset=utf8,language=python\n"
+                "traceback\n" + _remote_exception_str_chunked +
+                "]\n") % self.test.id())),
+            # testtools >= 2.5.0
+            Equals(_b((
+                "xfail: %s [ multipart\n"
+                "Content-Type: text/plain\n"
+                "something\n"
+                "F\r\nserialised\nform0\r\n"
+                "Content-Type: text/x-traceback;charset=utf8,language=python\n"
+                "traceback\n" + _remote_exception_repr_chunked +
+                "]\n") % self.test.id()))))
 
     def test_add_skip(self):
         """Test addSkip on a TestProtocolClient."""