Ensure test work as intended in all supported Python versions.

Take care to prevent false positives caused by varying error messages or
default behaviour. Extend test support to p3k.
Fixes [2972615].

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk/docutils@6268 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

5 files changed, 45 insertions, 21 deletions

diff --git a/test/test_functional.py b/test/test_functional.py
index 2eec17cc5..8b9defbf0 100755
--- a/test/test_functional.py
+++ b/test/test_functional.py
@@ -157,6 +157,14 @@ expected output and check it in:
         # Get output (automatically written to the output/ directory
         # by publish_file):
         output = docutils.core.publish_file(**params)
+        # ensure output is unicode
+        output_encoding = params.get('output_encoding', 'utf-8')
+        if sys.version_info < (3,0):
+            try:
+                output = output.decode(output_encoding)
+            except UnicodeDecodeError:
+                # failsafe (default for latex2e writer)
+                output = output.decode('latin1', 'replace')
         # Get the expected output *after* writing the actual output.
         no_expected = self.no_expected_template % {
             'exp': expected_path, 'out': params['destination_path']}
@@ -164,20 +172,23 @@ expected output and check it in:
         f = open(expected_path, 'rb')
         expected = f.read()
         f.close()
+        try:
+            expected = expected.decode(output_encoding)
+        except UnicodeDecodeError:
+            expected = expected.decode('latin1', 'replace')
+
         diff = self.expected_output_differs_template % {
             'exp': expected_path, 'out': params['destination_path']}
         try:
             self.assertEquals(output, expected, diff)
         except AssertionError:
-            if hasattr(difflib, 'unified_diff'):
-                # Generate diff if unified_diff available:
-                diff = ''.join(
-                    difflib.unified_diff(expected.decode('latin1').splitlines(1),
-                                         output.decode('latin1').splitlines(1),
-                                         expected_path,
-                                         params['destination_path']))
+            diff = ''.join(difflib.unified_diff(
+                expected.splitlines(True), output.splitlines(True),
+		expected_path, params['destination_path']))
+            if sys.version_info < (3,0):
+                diff = diff.encode('ascii', 'replace')
             print >>sys.stderr, '\n%s:' % (self,)
-            print >>sys.stderr, diff.encode('ascii', 'replace')
+            print >>sys.stderr, diff
             raise
         # Execute optional function containing extra tests:
         if '_test_more' in namespace:
diff --git a/test/test_parsers/test_rst/test_directives/include10.txt b/test/test_parsers/test_rst/test_directives/include10.txt
index d60fec6ad..ec7568f43 100644
--- a/test/test_parsers/test_rst/test_directives/include10.txt
+++ b/test/test_parsers/test_rst/test_directives/include10.txt
@@ -1,4 +1,4 @@
-.. |bad| unicode:: 0xFFFFFFFFF
+.. |bad| unicode:: 0x11111111
 
 hi
 -----
diff --git a/test/test_parsers/test_rst/test_directives/test_include.py b/test/test_parsers/test_rst/test_directives/test_include.py
index 76b0f9d60..e31567e52 100755
--- a/test/test_parsers/test_rst/test_directives/test_include.py
+++ b/test/test_parsers/test_rst/test_directives/test_include.py
@@ -35,6 +35,12 @@ include13rel = DocutilsTestSupport.utils.relative_path(None, include13)
 include_literal = os.path.join(mydir, 'include_literal.txt')
 utf_16_file = os.path.join(mydir, 'utf-16.csv')
 utf_16_file_rel = DocutilsTestSupport.utils.relative_path(None, utf_16_file)
+utf_16_error_str = ("UnicodeDecodeError: 'ascii' codec can't decode byte 0xfe "
+                    "in position 0: ordinal not in range(128)")
+if sys.version_info < (3,0):
+    utf_16_error_str = ("UnicodeError: Unable to decode input data.  "
+                        "Tried the following encodings: 'ascii'.\n"
+                        "            (%s)" % utf_16_error_str)
 nonexistent = os.path.join(os.path.dirname(states.__file__),
                            'include', 'nonexistent')
 nonexistent_rel = DocutilsTestSupport.utils.relative_path(
@@ -317,12 +323,11 @@ Include file is UTF-16-encoded, and is not valid ASCII.
     <system_message level="4" line="3" source="test data" type="SEVERE">
         <paragraph>
             Problem with "include" directive:
-            UnicodeError: Unable to decode input data.  Tried the following encodings: 'ascii'.
-            (UnicodeDecodeError: 'ascii' codec can't decode byte 0xfe in position 0: ordinal not in range(128))
+            %s
         <literal_block xml:space="preserve">
             .. include:: %s
                :encoding: ascii
-""" % utf_16_file_rel],
+""" % (utf_16_error_str, utf_16_file_rel)],
 # @@@ BUG with errors reported with incorrect "source" & "line":
 ["""\
 Testing errors in included file:
@@ -335,15 +340,15 @@ Testing errors in included file:
         Testing errors in included file:
     <system_message level="3" line="1" source="%(source)s" type="ERROR">
         <paragraph>
-            Invalid character code: 0xFFFFFFFFF
-            ValueError: unichr() arg not in range(0x110000) (wide Python build)
+            Invalid character code: 0x11111111
+            %(unichr_exception)s
         <literal_block xml:space="preserve">
-            unicode:: 0xFFFFFFFFF
+            unicode:: 0x11111111
     <system_message level="2" line="1" source="%(source)s" type="WARNING">
         <paragraph>
             Substitution definition "bad" empty or invalid.
         <literal_block xml:space="preserve">
-            .. |bad| unicode:: 0xFFFFFFFFF
+            .. |bad| unicode:: 0x11111111
     <section dupnames="hi" ids="hi">
         <title>
             hi
@@ -441,7 +446,7 @@ Testing errors in included file:
             no bottom       border
 """ % {'source': include10rel, 'nonexistent': nonexistent_rel,
        'unichr_exception':
-       DocutilsTestSupport.exception_data(unichr, int("0xFFFFFFFFF", 16))[2]
+       DocutilsTestSupport.exception_data(unichr, int("11111111", 16))[2]
       }],
 ["""\
 Include file with whitespace in the path:
diff --git a/test/test_parsers/test_rst/test_directives/test_raw.py b/test/test_parsers/test_rst/test_directives/test_raw.py
index d7112781b..fa7808112 100755
--- a/test/test_parsers/test_rst/test_directives/test_raw.py
+++ b/test/test_parsers/test_rst/test_directives/test_raw.py
@@ -22,6 +22,12 @@ mydir = 'test_parsers/test_rst/test_directives/'
 raw1 = os.path.join(mydir, 'raw1.txt')
 utf_16_file = os.path.join(mydir, 'utf-16.csv')
 utf_16_file_rel = DocutilsTestSupport.utils.relative_path(None, utf_16_file)
+utf_16_error_str = ("UnicodeDecodeError: 'ascii' codec can't decode byte 0xfe "
+                    "in position 0: ordinal not in range(128)")
+if sys.version_info < (3,0):
+    utf_16_error_str = ("UnicodeError: Unable to decode input data.  "
+                        "Tried the following encodings: 'ascii'.\n"
+                        "            (%s)" % utf_16_error_str)
 
 totest = {}
 
@@ -115,13 +121,12 @@ Raw input file is UTF-16-encoded, and is not valid ASCII.
     <system_message level="4" line="3" source="test data" type="SEVERE">
         <paragraph>
             Problem with "raw" directive:
-            UnicodeError: Unable to decode input data.  Tried the following encodings: \'ascii\'.
-            (UnicodeDecodeError: 'ascii' codec can't decode byte 0xfe in position 0: ordinal not in range(128))
+            %s
         <literal_block xml:space="preserve">
             .. raw:: html
                :file: %s
                :encoding: ascii
-""" % utf_16_file_rel],
+""" % (utf_16_error_str, utf_16_file_rel)],
 [u"""\
 .. raw:: html
    :encoding: utf-8
diff --git a/test/test_readers/test_python/test_functions.py b/test/test_readers/test_python/test_functions.py
index 3109edf0c..26b9db9c6 100755
--- a/test/test_readers/test_python/test_functions.py
+++ b/test/test_readers/test_python/test_functions.py
@@ -10,7 +10,10 @@ Tests for PySource Reader functions.
 
 import unittest
 from __init__ import DocutilsTestSupport
-from docutils.readers.python.moduleparser import trim_docstring
+try:
+    from docutils.readers.python.moduleparser import trim_docstring
+except ImportError: # `compiler` module dropped from py3k
+    pass
 
 
 class MiscTests(unittest.TestCase):