Merge branch 'master' into its-2-0

15 files changed, 224 insertions, 12 deletions

diff --git a/configure.ac b/configure.ac
index e2e86f4..bac146f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -10,6 +10,16 @@ DATADIR=`(
 )`
 AC_SUBST([DATADIR])
 
+py_module=libxml2
+AC_MSG_CHECKING(for python module $py_module)
+echo "import $py_module" | python - &>/dev/null
+if test $? -ne 0; then
+	AC_MSG_RESULT(not found)
+	AC_MSG_ERROR(Python module $py_module is needed to run this package)
+else
+	AC_MSG_RESULT(found)
+fi
+
 AC_CONFIG_FILES([
 Makefile
 itstool
diff --git a/itstool.in b/itstool.in
index b239a60..671254c 100755
--- a/itstool.in
+++ b/itstool.in
@@ -204,6 +204,10 @@ class Message (object):
         if re.sub('\s+', ' ', text).strip() != '':
             self._empty = False
 
+    def add_entity_ref (self, name):
+        self._message.append('&' + name + ';')
+        self._empty = False
+
     def add_placeholder (self, node):
         holder = Message.Placeholder(node)
         self._placeholders.append(holder)
@@ -221,9 +225,9 @@ class Message (object):
         if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)):
             self._message.append('')
         if node.ns() is not None and node.ns().name is not None:
-            self._message[-1] += ('<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
+            self._message[-1] += (u'<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
         else:
-            self._message[-1] += ('<%s' % unicode(node.name, 'utf-8'))
+            self._message[-1] += (u'<%s' % unicode(node.name, 'utf-8'))
         for prop in xml_attr_iter(node):
             name = prop.name
             if prop.ns() is not None:
@@ -243,9 +247,9 @@ class Message (object):
             if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)):
                 self._message.append('')
             if node.ns() is not None and node.ns().name is not None:
-                self._message[-1] += ('</%s:%s>' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
+                self._message[-1] += (u'</%s:%s>' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
             else:
-                self._message[-1] += ('</%s>' % unicode(node.name, 'utf-8'))
+                self._message[-1] += (u'</%s>' % unicode(node.name, 'utf-8'))
 
     def is_empty (self):
         return self._empty
@@ -440,7 +444,7 @@ class LocNote (object):
 
 
 class Document (object):
-    def __init__ (self, filename, messages):
+    def __init__ (self, filename, messages, load_dtd=False, keep_entities=False):
         self._xml_err = ''
         libxml2.registerErrorHandler(xml_error_catcher, self)
         try:
@@ -449,7 +453,14 @@ class Document (object):
             sys.stderr.write('Error: cannot open XML file %s\n' % filename)
             sys.exit(1)
         ctxt.lineNumbers(1)
-        ctxt.replaceEntities(1)
+        self._load_dtd = load_dtd
+        self._keep_entities = keep_entities
+        if load_dtd:
+            ctxt.loadSubset(1)
+        if keep_entities:
+            ctxt.replaceEntities(0)
+        else:
+            ctxt.replaceEntities(1)
         ctxt.parseDocument()
         self._filename = filename
         self._doc = ctxt.doc()
@@ -977,7 +988,11 @@ class Document (object):
                 nsdef = nsdef.next
         reg_ns(node, nss)
         nss['_'] = NS_BLANK
-        blurb = '<' + node.name
+        try:
+            blurb = node.doc.intSubset().serialize('utf-8')
+        except:
+            blurb = ''
+        blurb += '<' + node.name
         for nsname in nss.keys():
             if nsname is None:
                 blurb += ' xmlns="%s"' % nss[nsname]
@@ -985,6 +1000,8 @@ class Document (object):
                 blurb += ' xmlns:%s="%s"' % (nsname, nss[nsname])
         blurb += '>%s</%s>' % (trans.encode('utf-8'), node.name)
         ctxt = libxml2.createDocParserCtxt(blurb)
+        if self._load_dtd:
+            ctxt.loadSubset(1)
         ctxt.replaceEntities(0)
         ctxt.parseDocument()
         trnode = ctxt.doc().getRootElement()
@@ -1031,6 +1048,8 @@ class Document (object):
         if node.type in ('text', 'cdata') and msg is not None:
             msg.add_text(node.content)
             return
+        if node.type == 'entity_ref':
+            msg.add_entity_ref(node.name);
         if node.type != 'element':
             return
         if node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes':
@@ -1408,6 +1427,16 @@ if __name__ == '__main__':
                        dest='strict',
                        default=False,
                        help='Exit with error when PO files contain broken XML')
+    options.add_option('-d', '--load-dtd',
+                       action='store_true',
+                       dest='load_dtd',
+                       default=False,
+                       help='Load external DTDs used by input XML')
+    options.add_option('-k', '--keep-entities',
+                       action='store_true',
+                       dest='keep_entities',
+                       default=False,
+                       help='Keep entity reference unexpanded')
     options.add_option('-t', '--test',
                        dest='test',
                        default=None,
@@ -1427,7 +1456,7 @@ if __name__ == '__main__':
     if opts.merge is None and opts.join is None:
         messages = MessageList()
         for filename in args[1:]:
-            doc = Document(filename, messages)
+            doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities)
             doc.apply_its_rules(not(opts.nobuiltins))
             if opts.itsfile is not None:
                 for itsfile in opts.itsfile:
@@ -1469,7 +1498,7 @@ if __name__ == '__main__':
             sys.exit(1)
         for filename in args[1:]:
             messages = MessageList()
-            doc = Document(filename, messages)
+            doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities)
             doc.apply_its_rules(not(opts.nobuiltins))
             if opts.itsfile is not None:
                 for itsfile in opts.itsfile:
diff --git a/tests/IT-keep-entities-1.ll.po b/tests/IT-keep-entities-1.ll.po
new file mode 100644
index 0000000..780f4cc
--- /dev/null
+++ b/tests/IT-keep-entities-1.ll.po
@@ -0,0 +1,16 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2012-08-29 09:51-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. (itstool) path: bookinfo/title
+#: IT-keep-entities-1.xml:5
+msgid "The history of Leonard &ldquo;Bones&rdquo; McCoy"
+msgstr "La historia de Leonard &ldquo;Bones&rdquo; McCoy"
+
diff --git a/tests/IT-keep-entities-1.ll.xml b/tests/IT-keep-entities-1.ll.xml
new file mode 100644
index 0000000..31b70dc
--- /dev/null
+++ b/tests/IT-keep-entities-1.ll.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book>
+  <bookinfo id="startrek">
+    <title>La historia de Leonard &ldquo;Bones&rdquo; McCoy</title>
+  </bookinfo>
+</book>
diff --git a/tests/IT-keep-entities-1.pot b/tests/IT-keep-entities-1.pot
new file mode 100644
index 0000000..6df55c9
--- /dev/null
+++ b/tests/IT-keep-entities-1.pot
@@ -0,0 +1,16 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2012-08-29 09:51-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. (itstool) path: bookinfo/title
+#: tests/IT-keep-entities-1.xml:5
+msgid "The history of Leonard &ldquo;Bones&rdquo; McCoy"
+msgstr ""
+
diff --git a/tests/IT-keep-entities-1.xml b/tests/IT-keep-entities-1.xml
new file mode 100644
index 0000000..bb87ea8
--- /dev/null
+++ b/tests/IT-keep-entities-1.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book>
+  <bookinfo id="startrek">
+    <title>The history of Leonard &ldquo;Bones&rdquo; McCoy</title>
+  </bookinfo>
+</book>
diff --git a/tests/IT-keep-entities-2.ll.po b/tests/IT-keep-entities-2.ll.po
new file mode 100644
index 0000000..cab95b5
--- /dev/null
+++ b/tests/IT-keep-entities-2.ll.po
@@ -0,0 +1,21 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2013-09-16 12:04-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. (itstool) path: test/p
+#: tests/IT-keep-entities-2.xml:7
+msgid "&first;"
+msgstr "[&first;]"
+
+#. (itstool) path: test/p
+#: tests/IT-keep-entities-2.xml:8
+msgid "&second;"
+msgstr "[&second;]"
+
diff --git a/tests/IT-keep-entities-2.ll.xml b/tests/IT-keep-entities-2.ll.xml
new file mode 100644
index 0000000..95af179
--- /dev/null
+++ b/tests/IT-keep-entities-2.ll.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE test [
+<!ENTITY first "hello">
+<!ENTITY second SYSTEM "world.xml">
+]>
+<test>
+  <p>[&first;]</p>
+  <p>[&second;]</p>
+</test>
diff --git a/tests/IT-keep-entities-2.pot b/tests/IT-keep-entities-2.pot
new file mode 100644
index 0000000..40f755e
--- /dev/null
+++ b/tests/IT-keep-entities-2.pot
@@ -0,0 +1,21 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2013-09-16 12:04-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. (itstool) path: test/p
+#: tests/IT-keep-entities-2.xml:7
+msgid "&first;"
+msgstr ""
+
+#. (itstool) path: test/p
+#: tests/IT-keep-entities-2.xml:8
+msgid "&second;"
+msgstr ""
+
diff --git a/tests/IT-keep-entities-2.xml b/tests/IT-keep-entities-2.xml
new file mode 100644
index 0000000..0173e48
--- /dev/null
+++ b/tests/IT-keep-entities-2.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE test [
+<!ENTITY first "hello">
+<!ENTITY second SYSTEM "world.xml">
+]>
+<test>
+  <p>&first;</p>
+  <p>&second;</p>
+</test>
diff --git a/tests/IT-uses-external-dtds.ll.po b/tests/IT-uses-external-dtds.ll.po
new file mode 100644
index 0000000..3748474
--- /dev/null
+++ b/tests/IT-uses-external-dtds.ll.po
@@ -0,0 +1,16 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2012-08-29 09:51-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. (itstool) path: bookinfo/title
+#: IT-uses-external-dtds.xml:5
+msgid "The history of Leonard “Bones” McCoy"
+msgstr "La historia de Leonard “Bones” McCoy"
+
diff --git a/tests/IT-uses-external-dtds.ll.xml b/tests/IT-uses-external-dtds.ll.xml
new file mode 100644
index 0000000..3643188
--- /dev/null
+++ b/tests/IT-uses-external-dtds.ll.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book>
+  <bookinfo id="startrek">
+    <title>La historia de Leonard “Bones” McCoy</title>
+  </bookinfo>
+</book>
diff --git a/tests/IT-uses-external-dtds.pot b/tests/IT-uses-external-dtds.pot
new file mode 100644
index 0000000..8814cda
--- /dev/null
+++ b/tests/IT-uses-external-dtds.pot
@@ -0,0 +1,16 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2012-08-29 09:51-0400\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. (itstool) path: bookinfo/title
+#: tests/IT-uses-external-dtds.xml:5
+msgid "The history of Leonard “Bones” McCoy"
+msgstr ""
+
diff --git a/tests/IT-uses-external-dtds.xml b/tests/IT-uses-external-dtds.xml
new file mode 100644
index 0000000..bb87ea8
--- /dev/null
+++ b/tests/IT-uses-external-dtds.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book>
+  <bookinfo id="startrek">
+    <title>The history of Leonard &ldquo;Bones&rdquo; McCoy</title>
+  </bookinfo>
+</book>
diff --git a/tests/run_tests.py b/tests/run_tests.py
index 59738e8..6939ff6 100644
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -30,13 +30,17 @@ class ItstoolTests(unittest.TestCase):
         result = self.run_command("diff -u %s %s %s" % (options, f1, f2))
         self.assertEqual(result['output'], "", result['output'])
 
-    def _test_pot_generation(self, start_file, reference_pot=None, expected_status=0):
+    def _test_pot_generation(self, start_file, reference_pot=None, expected_status=0, options=None):
         start_file_base = os.path.splitext(start_file)[0]
-        result = self.run_command("cd %(dir)s && python itstool_test -n -o %(out)s %(in)s" % {
+        result = self.run_command("cd %(dir)s && python itstool_test %(opt)s -n -o %(out)s %(in)s" % {
             'dir' : ITSTOOL_DIR,
+            'opt' : (options or ''),
             'out' : os.path.join('tests', "test.pot"),
             'in'  : os.path.join('tests', start_file),
         }, expected_status)
+        # If we expected a failure, don't keep checking stuff
+        if expected_status != 0:
+            return result
         # If a reference pot file is present, test the output with this file
         if reference_pot is None:
             reference_pot = start_file_base + ".pot"
@@ -70,7 +74,7 @@ class ItstoolTests(unittest.TestCase):
 
     def _test_translation_process(self, start_file, expected_status=0, outputs=None, options=None):
         start_file_base = os.path.splitext(start_file)[0]
-        self._test_pot_generation(start_file)
+        self._test_pot_generation(start_file, options=options)
 
         if outputs is None:
             outputs = [("%s.ll.po" % start_file_base, "%s.ll.xml" % start_file_base, 'll')]
@@ -321,6 +325,23 @@ class ItstoolTests(unittest.TestCase):
         res = self._test_pot_generation('IT-malformed.xml', expected_status=1)
         #self.assertTrue("libxml2.parserError" in res['errors'])
 
+    def test_IT_translate_with_external_dtds_malformed(self):
+        """ Test that parsing XML requiring external DTD generates exception """
+        res = self._test_pot_generation('IT-uses-external-dtds.xml', expected_status=1)
+
+    def test_IT_translate_with_external_dtds(self):
+        self._test_translation_process('IT-uses-external-dtds.xml', options='--load-dtd')
+
+    # FIXME: It would be nice to be able to do this without loading the
+    # external subset, but libxml2 seems to verify entity references even
+    # if it doesn't do substitution.
+    def test_IT_keep_entities_1(self):
+        self._test_translation_process('IT-keep-entities-1.xml',
+                                       options='--load-dtd --keep-entities')
+
+    def test_IT_keep_entities_2(self):
+        self._test_translation_process('IT-keep-entities-2.xml', options='--keep-entities')
+
     def test_IT_join_1(self):
         self._test_translation_join('IT-join-1.xml', ('cs', 'de', 'fr'))