fix and re-organize error printing to sys.stderr

New class docutils.io.ErrorOutput:

Wrapper class for file-like error streams with
failsave de- and encoding of `str`, `bytes`, `unicode` and
`Exception` instances.

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk/docutils@7030 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

4 files changed, 148 insertions, 90 deletions

diff --git a/docutils/core.py b/docutils/core.py
index 0484a153f..91275f4a2 100644
--- a/docutils/core.py
+++ b/docutils/core.py
@@ -20,6 +20,7 @@ from docutils import __version__, __version_details__, SettingsSpec
 from docutils import frontend, io, utils, readers, writers
 from docutils.frontend import OptionParser
 from docutils.transforms import Transformer
+from docutils.io import ErrorOutput
 import docutils.readers.doctree
 
 class Publisher:
@@ -74,6 +75,8 @@ class Publisher:
         """An object containing Docutils settings as instance attributes.
         Set by `self.process_command_line()` or `self.get_settings()`."""
 
+        self._stderr = ErrorOutput()
+
     def set_reader(self, reader_name, parser, parser_name):
         """Set `self.reader` by name."""
         reader_class = readers.get_reader_class(reader_name)
@@ -230,23 +233,23 @@ class Publisher:
         if not self.document:
             return
         if self.settings.dump_settings:
-            print >>sys.stderr, '\n::: Runtime settings:'
-            print >>sys.stderr, pprint.pformat(self.settings.__dict__)
+            print >>self._stderr, '\n::: Runtime settings:'
+            print >>self._stderr, pprint.pformat(self.settings.__dict__)
         if self.settings.dump_internals:
-            print >>sys.stderr, '\n::: Document internals:'
-            print >>sys.stderr, pprint.pformat(self.document.__dict__)
+            print >>self._stderr, '\n::: Document internals:'
+            print >>self._stderr, pprint.pformat(self.document.__dict__)
         if self.settings.dump_transforms:
-            print >>sys.stderr, '\n::: Transforms applied:'
-            print >>sys.stderr, (' (priority, transform class, '
+            print >>self._stderr, '\n::: Transforms applied:'
+            print >>self._stderr, (' (priority, transform class, '
                                  'pending node details, keyword args)')
-            print >>sys.stderr, pprint.pformat(
+            print >>self._stderr, pprint.pformat(
                 [(priority, '%s.%s' % (xclass.__module__, xclass.__name__),
                   pending and pending.details, kwargs)
                  for priority, xclass, pending, kwargs
                  in self.document.transformer.applied])
         if self.settings.dump_pseudo_xml:
-            print >>sys.stderr, '\n::: Pseudo-XML:'
-            print >>sys.stderr, self.document.pformat().encode(
+            print >>self._stderr, '\n::: Pseudo-XML:'
+            print >>self._stderr, self.document.pformat().encode(
                 'raw_unicode_escape')
 
     def report_Exception(self, error):
@@ -255,8 +258,8 @@ class Publisher:
         elif isinstance(error, UnicodeEncodeError):
             self.report_UnicodeError(error)
         else:
-            print >>sys.stderr, '%s: %s' % (error.__class__.__name__, error)
-            print >>sys.stderr, ("""\
+            print >>self._stderr, '%s: %s' % (error.__class__.__name__, error)
+            print >>self._stderr, ("""\
 Exiting due to error.  Use "--traceback" to diagnose.
 Please report errors to <docutils-users@lists.sf.net>.
 Include "--traceback" output, Docutils version (%s [%s]),
@@ -265,13 +268,13 @@ command line used.""" % (__version__, __version_details__,
                          sys.version.split()[0]))
 
     def report_SystemMessage(self, error):
-        print >>sys.stderr, ('Exiting due to level-%s (%s) system message.'
+        print >>self._stderr, ('Exiting due to level-%s (%s) system message.'
                              % (error.level,
                                 utils.Reporter.levels[error.level]))
 
     def report_UnicodeError(self, error):
         data = error.object[error.start:error.end]
-        sys.stderr.write(
+        self._stderr.write(
             '%s: %s\n'
             '\n'
             'The specified output encoding (%s) cannot\n'
diff --git a/docutils/frontend.py b/docutils/frontend.py
index f8fdc06e4..0f78f18be 100644
--- a/docutils/frontend.py
+++ b/docutils/frontend.py
@@ -33,27 +33,13 @@ import sys
 import warnings
 import ConfigParser as CP
 import codecs
+import optparse
+from optparse import SUPPRESS_HELP
 import docutils
 import docutils.utils
 import docutils.nodes
-import optparse
-from optparse import SUPPRESS_HELP
+from docutils.io import locale_encoding, ErrorOutput
 
-# Guess the locale's encoding.
-# If no valid guess can be made, locale_encoding is set to `None`:
-try:
-    import locale # module missing in Jython
-except ImportError:
-    locale_encoding = None
-else:
-    locale_encoding = locale.getlocale()[1] or locale.getdefaultlocale()[1]
-    # locale.getpreferredencoding([do_setlocale=True|False])
-    # has side-effects | might return a wrong guess. 
-    # (cf. Update 1 in http://stackoverflow.com/questions/4082645/using-python-2-xs-locale-module-to-format-numbers-and-currency)
-    try:
-        codecs.lookup(locale_encoding)
-    except LookupError:
-        locale_encoding = None
 
 def store_multiple(option, opt, value, parser, *args, **kwargs):
     """
@@ -705,6 +691,9 @@ Skipping "%s" configuration file.
         self._files = []
         """List of paths of configuration files read."""
 
+        self._stderr = ErrorOutput()
+        """Wrapper around sys.stderr catching en-/decoding errors"""
+
     def read(self, filenames, option_parser):
         if type(filenames) in (str, unicode):
             filenames = [filenames]
@@ -717,9 +706,7 @@ Skipping "%s" configuration file.
             try:
                 CP.RawConfigParser.readfp(self, fp, filename)
             except UnicodeDecodeError:
-                msg = self.not_utf8_error % (filename, filename)
-                sys.stderr.write(msg.encode(sys.stderr.encoding or 'ascii',
-                                            'backslashreplace'))
+                self._stderr.write(self.not_utf8_error % (filename, filename))
                 fp.close()
                 continue
             fp.close()
diff --git a/docutils/io.py b/docutils/io.py
index b608eee62..16c5bfa4c 100644
--- a/docutils/io.py
+++ b/docutils/io.py
@@ -10,29 +10,28 @@ will exist for a variety of input/output mechanisms.
 __docformat__ = 'reStructuredText'
 
 import sys
-try:
-    import locale
-except ImportError:  # module missing in Jython
-    pass
 import re
 import codecs
 from docutils import TransformSpec
-from docutils._compat import b
+from docutils._compat import b, bytes
+
 
+# Guess the locale's encoding.
+# If no valid guess can be made, locale_encoding is set to `None`:
 try:
-    locale_encoding = locale.getlocale()[1]
-    defaultlocale_encoding = locale.getdefaultlocale()[1]
-except NameError:
+    import locale # module missing in Jython
+except ImportError:
     locale_encoding = None
-    defaultlocale_encoding = None
+else:
+    locale_encoding = locale.getlocale()[1] or locale.getdefaultlocale()[1]
+    # locale.getpreferredencoding([do_setlocale=True|False])
+    # has side-effects | might return a wrong guess.
+    # (cf. Update 1 in http://stackoverflow.com/questions/4082645/using-python-2-xs-locale-module-to-format-numbers-and-currency)
+    try:
+        codecs.lookup(locale_encoding or '') # None -> ''
+    except LookupError:
+        locale_encoding = None
 
-def _save_encode(s):
-    # Return `s` encoded catching exceptions.
-    # For debugging -> keep simple to avoid dependencies and errors.
-    if type(s) == str:
-        return s
-    return s.encode(sys.stderr.encoding or defaultlocale_encoding or 'ascii',
-                    'backslashreplace')
 
 class Input(TransformSpec):
 
@@ -104,7 +103,6 @@ class Input(TransformSpec):
                 # data that *IS* UTF-8:
                 encodings = ['utf-8',
                              locale_encoding,
-                             defaultlocale_encoding,
                              'latin-1', # fallback encoding
                             ]
         error = None
@@ -203,12 +201,80 @@ class Output(TransformSpec):
             return data.encode(self.encoding, self.error_handler)
 
 
+class ErrorOutput(object):
+    """
+    Wrapper class for file-like error streams with
+    failsave de- and encoding of `str`, `bytes`, `unicode` and
+    `Exception` instances.
+
+    Note that the output stream is not automatically closed.
+    Call the close() method if required.
+    """
+
+    def __init__(self, stream=None, encoding=None,
+                 encoding_errors='backslashreplace',
+                 decoding_errors='replace'):
+        """
+        :Parameters:
+            - `stream`: a file-like object (which is written to)
+            - `encoding`: `stream` text encoding. Guessed if None.
+            - `encoding_errors`: how to treat encoding errors.
+        """
+        if stream is None:
+            stream = sys.stderr
+        elif stream: # if `stream` is a file name, open it
+            if type(stream) is bytes:
+                stream = open(stream, 'w')
+            elif type(stream) is unicode:
+                stream = open(stream.encode(sys.getfilesystemencoding()), 'w')
+
+        self.stream = stream
+        """Where warning output is sent."""
+
+        self.encoding = (encoding or getattr(stream, 'encoding', None) or
+                         locale_encoding or 'ascii')
+        """The output character encoding."""
+
+        self.encoding_errors = encoding_errors
+        """Encoding error handler."""
+
+        self.decoding_errors = decoding_errors
+        """Decoding error handler."""
+
+    def write(self, data):
+        """`data` can be a `string`, `unicode`, or `Exception` instance.
+        """
+        if isinstance(data, Exception):
+            # Convert now to detect errors:
+            # In Python <= 2.6, unicode(<exception instance>)
+            # uses __str__ and fails with non-ASCII chars in arguments
+            try:
+                data = unicode(data)
+            except UnicodeError, err:
+                try:
+                    data = u', '.join(data.args)
+                except AttributeError:
+                    raise err
+                except UnicodeDecodeError:
+                    data = str(data)
+        try:
+            self.stream.write(data)
+        except UnicodeEncodeError:
+            self.stream.write(data.encode(self.encoding,
+                                               self.encoding_errors))
+        except TypeError: # in Python 3, stderr expects unicode
+            self.stream.write(data.decode(self.encoding,
+                                          self.decoding_errors))
+
+    def close(self):
+        self.stream.close()
+
+
 class FileInput(Input):
 
     """
     Input for single, simple file-like objects.
     """
-
     def __init__(self, source=None, source_path=None,
                  encoding=None, error_handler='strict',
                  autoclose=1, handle_io_errors=1, mode='rU'):
@@ -229,6 +295,8 @@ class FileInput(Input):
         Input.__init__(self, source, source_path, encoding, error_handler)
         self.autoclose = autoclose
         self.handle_io_errors = handle_io_errors
+        self._stderr = ErrorOutput()
+
         if source is None:
             if source_path:
                 # Specify encoding in Python 3
@@ -243,9 +311,9 @@ class FileInput(Input):
                 except IOError, error:
                     if not handle_io_errors:
                         raise
-                    print >>sys.stderr, '%s: %s' % (error.__class__.__name__,
-                                                    error)
-                    print >>sys.stderr, _save_encode(u'Unable to open source'
+                    print >>self._stderr, '%s: %s' % (
+                        error.__class__.__name__, error)
+                    print >>self._stderr, (u'Unable to open source'
                         u" file for reading ('%s'). Exiting." % source_path)
                     sys.exit(1)
             else:
@@ -290,8 +358,8 @@ class FileOutput(Output):
     """
 
     def __init__(self, destination=None, destination_path=None,
-                 encoding=None, error_handler='strict', autoclose=1,
-                 handle_io_errors=1):
+                 encoding=None, error_handler='strict', autoclose=True,
+                 handle_io_errors=True):
         """
         :Parameters:
             - `destination`: either a file-like object (which is written
@@ -300,19 +368,21 @@ class FileOutput(Output):
             - `destination_path`: a path to a file, which is opened and then
               written.
             - `autoclose`: close automatically after write (boolean); always
-              false if `sys.stdout` is the destination.
+              False if `sys.stdout` or `sys.stderr` is the destination.
         """
         Output.__init__(self, destination, destination_path,
                         encoding, error_handler)
-        self.opened = 1
+        self.opened = True
         self.autoclose = autoclose
         self.handle_io_errors = handle_io_errors
+        self._stderr = ErrorOutput()
         if destination is None:
             if destination_path:
-                self.opened = None
+                self.opened = False
             else:
                 self.destination = sys.stdout
-                self.autoclose = None
+        if destination in (sys.stdout, sys.stderr):
+            self.autoclose = False
         if not destination_path:
             try:
                 self.destination_path = self.destination.name
@@ -334,16 +404,16 @@ class FileOutput(Output):
         except IOError, error:
             if not self.handle_io_errors:
                 raise
-            print >>sys.stderr, '%s: %s' % (error.__class__.__name__, error)
-            print >>sys.stderr, _save_encode('Unable to open destination file'
-                " for writing ('%s').  Exiting." % self.destination_path)
+            print >>self._stderr, '%s: %s' % (error.__class__.__name__, error)
+            print >>self._stderr, (u'Unable to open destination file'
+                u" for writing ('%s').  Exiting." % self.destination_path)
             sys.exit(1)
         self.opened = 1
 
     def write(self, data):
         """Encode `data`, write it to a single file, and return it.
 
-        In Python 3, a (unicode) String is returned.
+        In Python 3, a (unicode) string is returned.
         """
         if sys.version_info >= (3,0):
             output = data # in py3k, write expects a (Unicode) string
@@ -360,7 +430,7 @@ class FileOutput(Output):
 
     def close(self):
         self.destination.close()
-        self.opened = None
+        self.opened = False
 
 
 class BinaryFileOutput(FileOutput):
@@ -373,11 +443,11 @@ class BinaryFileOutput(FileOutput):
         except IOError, error:
             if not self.handle_io_errors:
                 raise
-            print >>sys.stderr, '%s: %s' % (error.__class__.__name__, error)
-            print >>sys.stderr, _save_encode('Unable to open destination file'
-                " for writing ('%s').  Exiting." % self.destination_path)
+            print >>self._stderr, '%s: %s' % (error.__class__.__name__, error)
+            print >>self._stderr, (u'Unable to open destination file'
+                u" for writing ('%s').  Exiting." % self.destination_path)
             sys.exit(1)
-        self.opened = 1
+        self.opened = True
 
 
 class StringInput(Input):
diff --git a/docutils/statemachine.py b/docutils/statemachine.py
index 21ee7dfcf..decaaf17f 100644
--- a/docutils/statemachine.py
+++ b/docutils/statemachine.py
@@ -110,6 +110,7 @@ import sys
 import re
 import types
 import unicodedata
+from docutils.io import ErrorOutput
 
 class StateMachine:
 
@@ -168,6 +169,10 @@ class StateMachine:
         line changes.  Observers are called with one argument, ``self``.
         Cleared at the end of `run()`."""
 
+        self._stderr = ErrorOutput()
+        """Wrapper around sys.stderr catching en-/decoding errors"""
+
+
     def unlink(self):
         """Remove circular references to objects no longer required."""
         for state in self.states.values():
@@ -206,7 +211,7 @@ class StateMachine:
         self.line_offset = -1
         self.current_state = initial_state or self.initial_state
         if self.debug:
-            print >>sys.stderr, self._save_encode(
+            print >>self._stderr, (
                 u'\nStateMachine.run: input_lines (line_offset=%s):\n| %s'
                 % (self.line_offset, u'\n| '.join(self.input_lines)))
         transitions = None
@@ -214,7 +219,7 @@ class StateMachine:
         state = self.get_state()
         try:
             if self.debug:
-                print >>sys.stderr, ('\nStateMachine.run: bof transition')
+                print >>self._stderr, '\nStateMachine.run: bof transition'
             context, result = state.bof(context)
             results.extend(result)
             while 1:
@@ -224,7 +229,7 @@ class StateMachine:
                         if self.debug:
                             source, offset = self.input_lines.info(
                                 self.line_offset)
-                            print >>sys.stderr, self._save_encode(
+                            print >>self._stderr, (
                                 u'\nStateMachine.run: line (source=%r, '
                                 u'offset=%r):\n| %s'
                                 % (source, offset, self.line))
@@ -232,7 +237,7 @@ class StateMachine:
                             context, state, transitions)
                     except EOFError:
                         if self.debug:
-                            print >>sys.stderr, (
+                            print >>self._stderr, (
                                 '\nStateMachine.run: %s.eof transition'
                                 % state.__class__.__name__)
                         result = state.eof(context)
@@ -244,7 +249,7 @@ class StateMachine:
                     self.previous_line() # back up for another try
                     transitions = (exception.args[0],)
                     if self.debug:
-                        print >>sys.stderr, (
+                        print >>self._stderr, (
                               '\nStateMachine.run: TransitionCorrection to '
                               'state "%s", transition %s.'
                               % (state.__class__.__name__, transitions[0]))
@@ -257,7 +262,7 @@ class StateMachine:
                     else:
                         transitions = (exception.args[1],)
                     if self.debug:
-                        print >>sys.stderr, (
+                        print >>self._stderr, (
                               '\nStateMachine.run: StateCorrection to state '
                               '"%s", transition %s.'
                               % (next_state, transitions[0]))
@@ -281,7 +286,7 @@ class StateMachine:
         """
         if next_state:
             if self.debug and next_state != self.current_state:
-                print >>sys.stderr, (
+                print >>self._stderr, (
                     '\nStateMachine.get_state: Changing state from '
                     '"%s" to "%s" (input line %s).'
                     % (self.current_state, next_state,
@@ -438,7 +443,7 @@ class StateMachine:
             transitions =  state.transition_order
         state_correction = None
         if self.debug:
-            print >>sys.stderr, (
+            print >>self._stderr, (
                   '\nStateMachine.check_line: state="%s", transitions=%r.'
                   % (state.__class__.__name__, transitions))
         for name in transitions:
@@ -446,14 +451,14 @@ class StateMachine:
             match = pattern.match(self.line)
             if match:
                 if self.debug:
-                    print >>sys.stderr, (
+                    print >>self._stderr, (
                           '\nStateMachine.check_line: Matched transition '
                           '"%s" in state "%s".'
                           % (name, state.__class__.__name__))
                 return method(match, context, next_state)
         else:
             if self.debug:
-                print >>sys.stderr, (
+                print >>self._stderr, (
                       '\nStateMachine.check_line: No match in state "%s".'
                       % state.__class__.__name__)
             return state.no_match(context, transitions)
@@ -487,10 +492,10 @@ class StateMachine:
     def error(self):
         """Report error details."""
         type, value, module, line, function = _exception_data()
-        print >>sys.stderr, self._save_encode(u'%s: %s' % (type, value))
-        print >>sys.stderr, 'input line %s' % (self.abs_line_number())
-        print >>sys.stderr, self._save_encode(
-                u'module %s, line %s, function %s' % (module, line, function))
+        print >>self._stderr, u'%s: %s' % (type, value)
+        print >>self._stderr, 'input line %s' % (self.abs_line_number())
+        print >>self._stderr, (u'module %s, line %s, function %s' %
+                               (module, line, function))
 
     def attach_observer(self, observer):
         """
@@ -510,13 +515,6 @@ class StateMachine:
                 info = (None, None)
             observer(*info)
 
-    def _save_encode(self, s):
-        # Return `s` encoded catching exceptions.
-        # For debugging -> keep simple to avoid dependencies and errors.
-        if type(s) == str:
-            return s
-        # @ TODO: use 'utf-8' as fallback?
-        return s.encode(sys.stderr.encoding or 'ascii', 'backslashreplace')
 
 class State: