add --load-dtd option

This option tells itstool to load external DTDs when
parsing the document to be translated.  This prevents
errors when the document includes entity references defined
in those DTDs.

Note that externally-defined entity refs still cannot be used
in translated strings in the PO files.

Also note that this adds test cases that require either
network access or updating the local XML catalog to
including the DocBook DTDs.

Signed-off-by: Galen Charlton <gmc@esilibrary.com>

6 files changed, 82 insertions, 6 deletions

diff --git a/itstool.in b/itstool.in
index 3a21aaf..49d33bd 100755
--- a/itstool.in
+++ b/itstool.in
@@ -360,7 +360,7 @@ def fix_node_ns (node, nsdefs):
 
 
 class Document (object):
-    def __init__ (self, filename, messages):
+    def __init__ (self, filename, messages, load_dtd=False):
         self._xml_err = ''
         libxml2.registerErrorHandler(xml_error_catcher, self)
         try:
@@ -369,6 +369,8 @@ class Document (object):
             sys.stderr.write('Error: cannot open XML file %s\n' % filename)
             sys.exit(1)
         ctxt.lineNumbers(1)
+        if load_dtd:
+            ctxt.loadSubset(1)
         ctxt.replaceEntities(1)
         ctxt.parseDocument()
         self._filename = filename
@@ -1106,6 +1108,11 @@ if __name__ == '__main__':
                        dest='strict',
                        default=False,
                        help='Exit with error when PO files contain broken XML')
+    options.add_option('-d', '--load-dtd',
+                       action='store_true',
+                       dest='load_dtd',
+                       default=False,
+                       help='Load external DTDs used by input XML')
     options.add_option('-v', '--version',
                        action='store_true',
                        dest='version',
@@ -1120,7 +1127,7 @@ if __name__ == '__main__':
     if opts.merge is None and opts.join is None:
         messages = MessageList()
         for filename in args[1:]:
-            doc = Document(filename, messages)
+            doc = Document(filename, messages, opts.load_dtd)
             doc.apply_its_rules()
             if opts.itsfile is not None:
                 for itsfile in opts.itsfile:
@@ -1158,7 +1165,7 @@ if __name__ == '__main__':
             sys.exit(1)
         for filename in args[1:]:
             messages = MessageList()
-            doc = Document(filename, messages)
+            doc = Document(filename, messages, load_dtd=opts.load_dtd)
             doc.apply_its_rules()
             if opts.itsfile is not None:
                 for itsfile in opts.itsfile:
diff --git a/tests/IT-uses-external-dtds.ll.po b/tests/IT-uses-external-dtds.ll.po
new file mode 100644
index 0000000..aef9716
--- /dev/null
+++ b/tests/IT-uses-external-dtds.ll.po
@@ -0,0 +1,21 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2012-08-29 09:51-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. Put one translator per line, in the form NAME <EMAIL>, YEAR1, YEAR2
+msgctxt "_"
+msgid "translator-credits"
+msgstr ""
+
+#. (itstool) path: bookinfo/title
+#: IT-uses-external-dtds.xml:5
+msgid "The history of Leonard “Bones” McCoy"
+msgstr "La historia de Leonard “Bones” McCoy"
+
diff --git a/tests/IT-uses-external-dtds.ll.xml b/tests/IT-uses-external-dtds.ll.xml
new file mode 100644
index 0000000..e445a60
--- /dev/null
+++ b/tests/IT-uses-external-dtds.ll.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book lang="test">
+  <bookinfo id="startrek">
+    <title>La historia de Leonard “Bones” McCoy</title>
+  </bookinfo>
+</book>
diff --git a/tests/IT-uses-external-dtds.pot b/tests/IT-uses-external-dtds.pot
new file mode 100644
index 0000000..bc8afaf
--- /dev/null
+++ b/tests/IT-uses-external-dtds.pot
@@ -0,0 +1,21 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2012-08-29 09:51-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. Put one translator per line, in the form NAME <EMAIL>, YEAR1, YEAR2
+msgctxt "_"
+msgid "translator-credits"
+msgstr ""
+
+#. (itstool) path: bookinfo/title
+#: tests/IT-uses-external-dtds.xml:5
+msgid "The history of Leonard “Bones” McCoy"
+msgstr ""
+
diff --git a/tests/IT-uses-external-dtds.xml b/tests/IT-uses-external-dtds.xml
new file mode 100644
index 0000000..bb87ea8
--- /dev/null
+++ b/tests/IT-uses-external-dtds.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book>
+  <bookinfo id="startrek">
+    <title>The history of Leonard &ldquo;Bones&rdquo; McCoy</title>
+  </bookinfo>
+</book>
diff --git a/tests/run_tests.py b/tests/run_tests.py
index 5c108e5..10a725e 100644
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -30,10 +30,11 @@ class ItstoolTests(unittest.TestCase):
         result = self.run_command("diff -u %s %s %s" % (options, f1, f2))
         self.assertEqual(result['output'], "", result['output'])
 
-    def _test_pot_generation(self, start_file, reference_pot=None, expected_status=0):
+    def _test_pot_generation(self, start_file, reference_pot=None, expected_status=0, options=None):
         start_file_base = os.path.splitext(start_file)[0]
-        result = self.run_command("cd %(dir)s && python itstool_test -o %(out)s %(in)s" % {
+        result = self.run_command("cd %(dir)s && python itstool_test %(opt)s -o %(out)s %(in)s" % {
             'dir' : ITSTOOL_DIR,
+            'opt' : (options or ''),
             'out' : os.path.join('tests', "test.pot"),
             'in'  : os.path.join('tests', start_file),
         }, expected_status)
@@ -70,7 +71,7 @@ class ItstoolTests(unittest.TestCase):
 
     def _test_translation_process(self, start_file, expected_status=0, po_file=None, xml_file=None, options=None):
         start_file_base = os.path.splitext(start_file)[0]
-        self._test_pot_generation(start_file)
+        self._test_pot_generation(start_file, options=options)
 
         # Compile mo and merge
         if po_file is None:
@@ -176,6 +177,18 @@ class ItstoolTests(unittest.TestCase):
         res = self._test_pot_generation('IT-malformed.xml', expected_status=1)
         #self.assertTrue("libxml2.parserError" in res['errors'])
 
+    def test_IT_malformed(self):
+        """ Test that parsing XML requiring external DTD generates exception """
+        res = self._test_pot_generation('IT-uses-external-dtds.xml', expected_status=1)
+
+    def test_IT_malformed(self):
+        """ Test that parsing XML requiring external DTD generates exception """
+        res = self._test_pot_generation('IT-uses-external-dtds.xml', expected_status=0,
+                                        options='--load-dtd')
+
+    def test_IT_translate_with_external_dtds(self):
+        self._test_translation_process('IT-uses-external-dtds.xml', options='--load-dtd')
+
     def test_IT_join_1(self):
         res = self._test_translation_join('IT-join-1.xml', ('cs', 'de', 'fr'))