made xmlcharrefreplace the default output encoding error handler forblais_interruptus@3512

HTML and XML writer

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk/docutils@3512 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

6 files changed, 33 insertions, 10 deletions

diff --git a/HISTORY.txt b/HISTORY.txt
index 5cb822a0f..a76eb4abe 100644
--- a/HISTORY.txt
+++ b/HISTORY.txt
@@ -47,6 +47,13 @@ Changes Since 0.3.9
 * docutils/writers/html4css1.py:
 
   - Added support for image width and height units.
+  - Made ``xmlcharrefreplace`` the default output encoding error
+    handler.
+
+* docutils/writers/docutils_xml.py:
+
+  - Made ``xmlcharrefreplace`` the default output encoding error
+    handler.
 
 * docs/user/links.txt: Added to project; link list.
 
diff --git a/docs/dev/todo.txt b/docs/dev/todo.txt
index 481a99142..2727aeb43 100644
--- a/docs/dev/todo.txt
+++ b/docs/dev/todo.txt
@@ -122,10 +122,6 @@ General
   from "--generator", "--source-link", and "--date" etc.), allowing
   translations.
 
-* Need a Unicode to HTML entities codec for HTML writer?  No, the
-  "xmlcharrefreplace" output error handler is sufficient.
-  Make it the default for HTML & XML writers?
-
 * Add validation?  See http://pytrex.sourceforge.net, RELAX NG, pyRXP.
 
 * Ask Python-dev for opinions (GvR for a pronouncement) on special
diff --git a/docutils/writers/docutils_xml.py b/docutils/writers/docutils_xml.py
index 1b1164977..2362e6b78 100644
--- a/docutils/writers/docutils_xml.py
+++ b/docutils/writers/docutils_xml.py
@@ -39,6 +39,8 @@ class Writer(writers.Writer):
           {'dest': 'doctype_declaration', 'default': 1,
            'action': 'store_false', 'validator': frontend.validate_boolean}),))
 
+    settings_defaults = {'output_encoding_error_handler': 'xmlcharrefreplace'}
+
     config_section = 'docutils_xml writer'
     config_section_dependencies = ('writers',)
 
diff --git a/docutils/writers/html4css1.py b/docutils/writers/html4css1.py
index e5c1c261e..f77d6d8e8 100644
--- a/docutils/writers/html4css1.py
+++ b/docutils/writers/html4css1.py
@@ -108,6 +108,8 @@ class Writer(writers.Writer):
           ['--cloak-email-addresses'],
           {'action': 'store_true', 'validator': frontend.validate_boolean}),))
 
+    settings_defaults = {'output_encoding_error_handler': 'xmlcharrefreplace'}
+
     relative_path_settings = ('stylesheet_path',)
 
     config_section = 'html4css1 writer'
diff --git a/test/test_writers/test_docutils_xml.py b/test/test_writers/test_docutils_xml.py
index 1b9790c6a..f5cf18b3f 100755
--- a/test/test_writers/test_docutils_xml.py
+++ b/test/test_writers/test_docutils_xml.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
 # Author: Felix Wiemann
 # Contact: Felix_Wiemann@ososo.de
@@ -19,17 +20,16 @@ import docutils.core
 
 class DocutilsXMLTestCase(DocutilsTestSupport.StandardTestCase):
 
-    input = 'Test\n====\n\nSubsection\n----------\n\nTest\n\n----------\n\nTest.'
+    input = 'Test\n====\n\nSubsection\n----------\n\nTest\n\n----------\n\nTest. äöü€'
     xmldecl = '<?xml version="1.0" encoding="iso-8859-1"?>\n'
     doctypedecl = '<!DOCTYPE document PUBLIC "+//IDN docutils.sourceforge.net//DTD Docutils Generic//EN//XML" "http://docutils.sourceforge.net/docs/ref/docutils.dtd">\n'
     generatedby = '<!-- Generated by Docutils %s -->\n' % docutils.__version__
-    bodynormal = '<document ids="test" names="test" source="&lt;string&gt;"><title>Test</title><subtitle ids="subsection" names="subsection">Subsection</subtitle><paragraph>Test</paragraph><transition/><paragraph>Test.</paragraph></document>'
-    bodynormal = '<document ids="test" names="test" source="&lt;string&gt;"><title>Test</title><subtitle ids="subsection" names="subsection">Subsection</subtitle><paragraph>Test</paragraph><transition/><paragraph>Test.</paragraph></document>'
-    bodynewlines = '<document ids="test" names="test" source="&lt;string&gt;">\n<title>\nTest\n</title>\n<subtitle ids="subsection" names="subsection">\nSubsection\n</subtitle>\n<paragraph>\nTest\n</paragraph>\n<transition/>\n<paragraph>\nTest.\n</paragraph>\n</document>\n'
-    bodyindents = '<document ids="test" names="test" source="&lt;string&gt;">\n    <title>\n        Test\n    </title>\n    <subtitle ids="subsection" names="subsection">\n        Subsection\n    </subtitle>\n    <paragraph>\n        Test\n    </paragraph>\n    <transition/>\n    <paragraph>\n        Test.\n    </paragraph>\n</document>\n'
+    bodynormal = '<document ids="test" names="test" source="&lt;string&gt;"><title>Test</title><subtitle ids="subsection" names="subsection">Subsection</subtitle><paragraph>Test</paragraph><transition/><paragraph>Test. \xe4\xf6\xfc&#8364;</paragraph></document>'
+    bodynewlines = '<document ids="test" names="test" source="&lt;string&gt;">\n<title>\nTest\n</title>\n<subtitle ids="subsection" names="subsection">\nSubsection\n</subtitle>\n<paragraph>\nTest\n</paragraph>\n<transition/>\n<paragraph>\nTest. \xe4\xf6\xfc&#8364;\n</paragraph>\n</document>\n'
+    bodyindents = '<document ids="test" names="test" source="&lt;string&gt;">\n    <title>\n        Test\n    </title>\n    <subtitle ids="subsection" names="subsection">\n        Subsection\n    </subtitle>\n    <paragraph>\n        Test\n    </paragraph>\n    <transition/>\n    <paragraph>\n        Test. \xe4\xf6\xfc&#8364;\n    </paragraph>\n</document>\n'
 
     def test_publish(self):
-        settings = {'output_encoding': 'iso-8859-1'}
+        settings = {'input_encoding': 'utf8', 'output_encoding': 'iso-8859-1'}
         for settings['newlines'] in 0, 1:
             for settings['indents'] in 0, 1:
                 for settings['xml_declaration'] in 0, 1:
diff --git a/test/test_writers/test_html4css1.py b/test/test_writers/test_html4css1.py
index 70d3be772..1cc15bd5e 100755
--- a/test/test_writers/test_html4css1.py
+++ b/test/test_writers/test_html4css1.py
@@ -1,4 +1,5 @@
 #! /usr/bin/env python
+# -*- coding: utf-8 -*-
 
 # Author: reggie dugard
 # Contact: reggie@users.sourceforge.net
@@ -14,11 +15,16 @@ dictionaries (redundant), along with 'meta' and 'stylesheet' entries with
 standard values, and any entries with empty values.
 """
 
+from docutils import core
+
+import unittest
 from __init__ import DocutilsTestSupport
 
 def suite():
     s = DocutilsTestSupport.HtmlPublishPartsTestSuite()
     s.generateTests(totest)
+    import test_html4css1
+    s.addTest(unittest.defaultTestLoader.loadTestsFromModule(test_html4css1))
     return s
 
 
@@ -317,6 +323,16 @@ And even more stuff
 ])
 
 
+class EncodingTestCase(DocutilsTestSupport.StandardTestCase):
+
+    def test_xmlcharrefreplace(self):
+        # Test that xmlcharrefreplace is the default output encoding
+        # error handler.
+        self.assert_('\xe4\xf6\xfc&#8364;' in core.publish_string(
+            'äöü€', writer_name='html4css1',
+            settings_overrides={'output_encoding': 'latin1'}))
+
+
 if __name__ == '__main__':
     import unittest
     unittest.main(defaultTest='suite')