diff options
author | Shaun McCance <shaunm@gnome.org> | 2013-10-28 12:05:03 -0400 |
---|---|---|
committer | Shaun McCance <shaunm@gnome.org> | 2013-10-28 12:05:03 -0400 |
commit | d657543826789e263ac9c8861a722f2527b5b0a3 (patch) | |
tree | 81e78a2207c7320d1290b3c7757c7a54d216b560 | |
parent | e338a3e4463d6f8098ef76c45d44851fe62cb126 (diff) | |
parent | 2928d6f02a0f30415bd993490d1920fd990ce130 (diff) | |
download | itstool-d657543826789e263ac9c8861a722f2527b5b0a3.tar.gz |
Merge branch 'master' into its-2-0
-rw-r--r-- | configure.ac | 10 | ||||
-rwxr-xr-x | itstool.in | 47 | ||||
-rw-r--r-- | tests/IT-keep-entities-1.ll.po | 16 | ||||
-rw-r--r-- | tests/IT-keep-entities-1.ll.xml | 7 | ||||
-rw-r--r-- | tests/IT-keep-entities-1.pot | 16 | ||||
-rw-r--r-- | tests/IT-keep-entities-1.xml | 7 | ||||
-rw-r--r-- | tests/IT-keep-entities-2.ll.po | 21 | ||||
-rw-r--r-- | tests/IT-keep-entities-2.ll.xml | 9 | ||||
-rw-r--r-- | tests/IT-keep-entities-2.pot | 21 | ||||
-rw-r--r-- | tests/IT-keep-entities-2.xml | 9 | ||||
-rw-r--r-- | tests/IT-uses-external-dtds.ll.po | 16 | ||||
-rw-r--r-- | tests/IT-uses-external-dtds.ll.xml | 7 | ||||
-rw-r--r-- | tests/IT-uses-external-dtds.pot | 16 | ||||
-rw-r--r-- | tests/IT-uses-external-dtds.xml | 7 | ||||
-rw-r--r-- | tests/run_tests.py | 27 |
15 files changed, 224 insertions, 12 deletions
diff --git a/configure.ac b/configure.ac index e2e86f4..bac146f 100644 --- a/configure.ac +++ b/configure.ac @@ -10,6 +10,16 @@ DATADIR=`( )` AC_SUBST([DATADIR]) +py_module=libxml2 +AC_MSG_CHECKING(for python module $py_module) +echo "import $py_module" | python - &>/dev/null +if test $? -ne 0; then + AC_MSG_RESULT(not found) + AC_MSG_ERROR(Python module $py_module is needed to run this package) +else + AC_MSG_RESULT(found) +fi + AC_CONFIG_FILES([ Makefile itstool @@ -204,6 +204,10 @@ class Message (object): if re.sub('\s+', ' ', text).strip() != '': self._empty = False + def add_entity_ref (self, name): + self._message.append('&' + name + ';') + self._empty = False + def add_placeholder (self, node): holder = Message.Placeholder(node) self._placeholders.append(holder) @@ -221,9 +225,9 @@ class Message (object): if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): self._message.append('') if node.ns() is not None and node.ns().name is not None: - self._message[-1] += ('<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) + self._message[-1] += (u'<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) else: - self._message[-1] += ('<%s' % unicode(node.name, 'utf-8')) + self._message[-1] += (u'<%s' % unicode(node.name, 'utf-8')) for prop in xml_attr_iter(node): name = prop.name if prop.ns() is not None: @@ -243,9 +247,9 @@ class Message (object): if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): self._message.append('') if node.ns() is not None and node.ns().name is not None: - self._message[-1] += ('</%s:%s>' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) + self._message[-1] += (u'</%s:%s>' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) else: - self._message[-1] += ('</%s>' % unicode(node.name, 'utf-8')) + self._message[-1] += (u'</%s>' % unicode(node.name, 'utf-8')) def is_empty (self): return self._empty @@ -440,7 +444,7 @@ class LocNote (object): class Document (object): - def __init__ (self, filename, messages): + def __init__ (self, filename, messages, load_dtd=False, keep_entities=False): self._xml_err = '' libxml2.registerErrorHandler(xml_error_catcher, self) try: @@ -449,7 +453,14 @@ class Document (object): sys.stderr.write('Error: cannot open XML file %s\n' % filename) sys.exit(1) ctxt.lineNumbers(1) - ctxt.replaceEntities(1) + self._load_dtd = load_dtd + self._keep_entities = keep_entities + if load_dtd: + ctxt.loadSubset(1) + if keep_entities: + ctxt.replaceEntities(0) + else: + ctxt.replaceEntities(1) ctxt.parseDocument() self._filename = filename self._doc = ctxt.doc() @@ -977,7 +988,11 @@ class Document (object): nsdef = nsdef.next reg_ns(node, nss) nss['_'] = NS_BLANK - blurb = '<' + node.name + try: + blurb = node.doc.intSubset().serialize('utf-8') + except: + blurb = '' + blurb += '<' + node.name for nsname in nss.keys(): if nsname is None: blurb += ' xmlns="%s"' % nss[nsname] @@ -985,6 +1000,8 @@ class Document (object): blurb += ' xmlns:%s="%s"' % (nsname, nss[nsname]) blurb += '>%s</%s>' % (trans.encode('utf-8'), node.name) ctxt = libxml2.createDocParserCtxt(blurb) + if self._load_dtd: + ctxt.loadSubset(1) ctxt.replaceEntities(0) ctxt.parseDocument() trnode = ctxt.doc().getRootElement() @@ -1031,6 +1048,8 @@ class Document (object): if node.type in ('text', 'cdata') and msg is not None: msg.add_text(node.content) return + if node.type == 'entity_ref': + msg.add_entity_ref(node.name); if node.type != 'element': return if node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes': @@ -1408,6 +1427,16 @@ if __name__ == '__main__': dest='strict', default=False, help='Exit with error when PO files contain broken XML') + options.add_option('-d', '--load-dtd', + action='store_true', + dest='load_dtd', + default=False, + help='Load external DTDs used by input XML') + options.add_option('-k', '--keep-entities', + action='store_true', + dest='keep_entities', + default=False, + help='Keep entity reference unexpanded') options.add_option('-t', '--test', dest='test', default=None, @@ -1427,7 +1456,7 @@ if __name__ == '__main__': if opts.merge is None and opts.join is None: messages = MessageList() for filename in args[1:]: - doc = Document(filename, messages) + doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities) doc.apply_its_rules(not(opts.nobuiltins)) if opts.itsfile is not None: for itsfile in opts.itsfile: @@ -1469,7 +1498,7 @@ if __name__ == '__main__': sys.exit(1) for filename in args[1:]: messages = MessageList() - doc = Document(filename, messages) + doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities) doc.apply_its_rules(not(opts.nobuiltins)) if opts.itsfile is not None: for itsfile in opts.itsfile: diff --git a/tests/IT-keep-entities-1.ll.po b/tests/IT-keep-entities-1.ll.po new file mode 100644 index 0000000..780f4cc --- /dev/null +++ b/tests/IT-keep-entities-1.ll.po @@ -0,0 +1,16 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2012-08-29 09:51-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. (itstool) path: bookinfo/title +#: IT-keep-entities-1.xml:5 +msgid "The history of Leonard “Bones” McCoy" +msgstr "La historia de Leonard “Bones” McCoy" + diff --git a/tests/IT-keep-entities-1.ll.xml b/tests/IT-keep-entities-1.ll.xml new file mode 100644 index 0000000..31b70dc --- /dev/null +++ b/tests/IT-keep-entities-1.ll.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd"> +<book> + <bookinfo id="startrek"> + <title>La historia de Leonard “Bones” McCoy</title> + </bookinfo> +</book> diff --git a/tests/IT-keep-entities-1.pot b/tests/IT-keep-entities-1.pot new file mode 100644 index 0000000..6df55c9 --- /dev/null +++ b/tests/IT-keep-entities-1.pot @@ -0,0 +1,16 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2012-08-29 09:51-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. (itstool) path: bookinfo/title +#: tests/IT-keep-entities-1.xml:5 +msgid "The history of Leonard “Bones” McCoy" +msgstr "" + diff --git a/tests/IT-keep-entities-1.xml b/tests/IT-keep-entities-1.xml new file mode 100644 index 0000000..bb87ea8 --- /dev/null +++ b/tests/IT-keep-entities-1.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="US-ASCII"?> +<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd"> +<book> + <bookinfo id="startrek"> + <title>The history of Leonard “Bones” McCoy</title> + </bookinfo> +</book> diff --git a/tests/IT-keep-entities-2.ll.po b/tests/IT-keep-entities-2.ll.po new file mode 100644 index 0000000..cab95b5 --- /dev/null +++ b/tests/IT-keep-entities-2.ll.po @@ -0,0 +1,21 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2013-09-16 12:04-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. (itstool) path: test/p +#: tests/IT-keep-entities-2.xml:7 +msgid "&first;" +msgstr "[&first;]" + +#. (itstool) path: test/p +#: tests/IT-keep-entities-2.xml:8 +msgid "&second;" +msgstr "[&second;]" + diff --git a/tests/IT-keep-entities-2.ll.xml b/tests/IT-keep-entities-2.ll.xml new file mode 100644 index 0000000..95af179 --- /dev/null +++ b/tests/IT-keep-entities-2.ll.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE test [ +<!ENTITY first "hello"> +<!ENTITY second SYSTEM "world.xml"> +]> +<test> + <p>[&first;]</p> + <p>[&second;]</p> +</test> diff --git a/tests/IT-keep-entities-2.pot b/tests/IT-keep-entities-2.pot new file mode 100644 index 0000000..40f755e --- /dev/null +++ b/tests/IT-keep-entities-2.pot @@ -0,0 +1,21 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2013-09-16 12:04-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. (itstool) path: test/p +#: tests/IT-keep-entities-2.xml:7 +msgid "&first;" +msgstr "" + +#. (itstool) path: test/p +#: tests/IT-keep-entities-2.xml:8 +msgid "&second;" +msgstr "" + diff --git a/tests/IT-keep-entities-2.xml b/tests/IT-keep-entities-2.xml new file mode 100644 index 0000000..0173e48 --- /dev/null +++ b/tests/IT-keep-entities-2.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE test [ +<!ENTITY first "hello"> +<!ENTITY second SYSTEM "world.xml"> +]> +<test> + <p>&first;</p> + <p>&second;</p> +</test> diff --git a/tests/IT-uses-external-dtds.ll.po b/tests/IT-uses-external-dtds.ll.po new file mode 100644 index 0000000..3748474 --- /dev/null +++ b/tests/IT-uses-external-dtds.ll.po @@ -0,0 +1,16 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2012-08-29 09:51-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. (itstool) path: bookinfo/title +#: IT-uses-external-dtds.xml:5 +msgid "The history of Leonard “Bones” McCoy" +msgstr "La historia de Leonard “Bones” McCoy" + diff --git a/tests/IT-uses-external-dtds.ll.xml b/tests/IT-uses-external-dtds.ll.xml new file mode 100644 index 0000000..3643188 --- /dev/null +++ b/tests/IT-uses-external-dtds.ll.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd"> +<book> + <bookinfo id="startrek"> + <title>La historia de Leonard “Bones” McCoy</title> + </bookinfo> +</book> diff --git a/tests/IT-uses-external-dtds.pot b/tests/IT-uses-external-dtds.pot new file mode 100644 index 0000000..8814cda --- /dev/null +++ b/tests/IT-uses-external-dtds.pot @@ -0,0 +1,16 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2012-08-29 09:51-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. (itstool) path: bookinfo/title +#: tests/IT-uses-external-dtds.xml:5 +msgid "The history of Leonard “Bones” McCoy" +msgstr "" + diff --git a/tests/IT-uses-external-dtds.xml b/tests/IT-uses-external-dtds.xml new file mode 100644 index 0000000..bb87ea8 --- /dev/null +++ b/tests/IT-uses-external-dtds.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="US-ASCII"?> +<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd"> +<book> + <bookinfo id="startrek"> + <title>The history of Leonard “Bones” McCoy</title> + </bookinfo> +</book> diff --git a/tests/run_tests.py b/tests/run_tests.py index 59738e8..6939ff6 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -30,13 +30,17 @@ class ItstoolTests(unittest.TestCase): result = self.run_command("diff -u %s %s %s" % (options, f1, f2)) self.assertEqual(result['output'], "", result['output']) - def _test_pot_generation(self, start_file, reference_pot=None, expected_status=0): + def _test_pot_generation(self, start_file, reference_pot=None, expected_status=0, options=None): start_file_base = os.path.splitext(start_file)[0] - result = self.run_command("cd %(dir)s && python itstool_test -n -o %(out)s %(in)s" % { + result = self.run_command("cd %(dir)s && python itstool_test %(opt)s -n -o %(out)s %(in)s" % { 'dir' : ITSTOOL_DIR, + 'opt' : (options or ''), 'out' : os.path.join('tests', "test.pot"), 'in' : os.path.join('tests', start_file), }, expected_status) + # If we expected a failure, don't keep checking stuff + if expected_status != 0: + return result # If a reference pot file is present, test the output with this file if reference_pot is None: reference_pot = start_file_base + ".pot" @@ -70,7 +74,7 @@ class ItstoolTests(unittest.TestCase): def _test_translation_process(self, start_file, expected_status=0, outputs=None, options=None): start_file_base = os.path.splitext(start_file)[0] - self._test_pot_generation(start_file) + self._test_pot_generation(start_file, options=options) if outputs is None: outputs = [("%s.ll.po" % start_file_base, "%s.ll.xml" % start_file_base, 'll')] @@ -321,6 +325,23 @@ class ItstoolTests(unittest.TestCase): res = self._test_pot_generation('IT-malformed.xml', expected_status=1) #self.assertTrue("libxml2.parserError" in res['errors']) + def test_IT_translate_with_external_dtds_malformed(self): + """ Test that parsing XML requiring external DTD generates exception """ + res = self._test_pot_generation('IT-uses-external-dtds.xml', expected_status=1) + + def test_IT_translate_with_external_dtds(self): + self._test_translation_process('IT-uses-external-dtds.xml', options='--load-dtd') + + # FIXME: It would be nice to be able to do this without loading the + # external subset, but libxml2 seems to verify entity references even + # if it doesn't do substitution. + def test_IT_keep_entities_1(self): + self._test_translation_process('IT-keep-entities-1.xml', + options='--load-dtd --keep-entities') + + def test_IT_keep_entities_2(self): + self._test_translation_process('IT-keep-entities-2.xml', options='--keep-entities') + def test_IT_join_1(self): self._test_translation_join('IT-join-1.xml', ('cs', 'de', 'fr')) |