Merge branch 'master' into its-2-0

1 files changed, 38 insertions, 9 deletions

diff --git a/itstool.in b/itstool.in
index b239a60..671254c 100755
--- a/itstool.in
+++ b/itstool.in
@@ -204,6 +204,10 @@ class Message (object):
         if re.sub('\s+', ' ', text).strip() != '':
             self._empty = False
 
+    def add_entity_ref (self, name):
+        self._message.append('&' + name + ';')
+        self._empty = False
+
     def add_placeholder (self, node):
         holder = Message.Placeholder(node)
         self._placeholders.append(holder)
@@ -221,9 +225,9 @@ class Message (object):
         if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)):
             self._message.append('')
         if node.ns() is not None and node.ns().name is not None:
-            self._message[-1] += ('<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
+            self._message[-1] += (u'<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
         else:
-            self._message[-1] += ('<%s' % unicode(node.name, 'utf-8'))
+            self._message[-1] += (u'<%s' % unicode(node.name, 'utf-8'))
         for prop in xml_attr_iter(node):
             name = prop.name
             if prop.ns() is not None:
@@ -243,9 +247,9 @@ class Message (object):
             if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)):
                 self._message.append('')
             if node.ns() is not None and node.ns().name is not None:
-                self._message[-1] += ('</%s:%s>' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
+                self._message[-1] += (u'</%s:%s>' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8')))
             else:
-                self._message[-1] += ('</%s>' % unicode(node.name, 'utf-8'))
+                self._message[-1] += (u'</%s>' % unicode(node.name, 'utf-8'))
 
     def is_empty (self):
         return self._empty
@@ -440,7 +444,7 @@ class LocNote (object):
 
 
 class Document (object):
-    def __init__ (self, filename, messages):
+    def __init__ (self, filename, messages, load_dtd=False, keep_entities=False):
         self._xml_err = ''
         libxml2.registerErrorHandler(xml_error_catcher, self)
         try:
@@ -449,7 +453,14 @@ class Document (object):
             sys.stderr.write('Error: cannot open XML file %s\n' % filename)
             sys.exit(1)
         ctxt.lineNumbers(1)
-        ctxt.replaceEntities(1)
+        self._load_dtd = load_dtd
+        self._keep_entities = keep_entities
+        if load_dtd:
+            ctxt.loadSubset(1)
+        if keep_entities:
+            ctxt.replaceEntities(0)
+        else:
+            ctxt.replaceEntities(1)
         ctxt.parseDocument()
         self._filename = filename
         self._doc = ctxt.doc()
@@ -977,7 +988,11 @@ class Document (object):
                 nsdef = nsdef.next
         reg_ns(node, nss)
         nss['_'] = NS_BLANK
-        blurb = '<' + node.name
+        try:
+            blurb = node.doc.intSubset().serialize('utf-8')
+        except:
+            blurb = ''
+        blurb += '<' + node.name
         for nsname in nss.keys():
             if nsname is None:
                 blurb += ' xmlns="%s"' % nss[nsname]
@@ -985,6 +1000,8 @@ class Document (object):
                 blurb += ' xmlns:%s="%s"' % (nsname, nss[nsname])
         blurb += '>%s</%s>' % (trans.encode('utf-8'), node.name)
         ctxt = libxml2.createDocParserCtxt(blurb)
+        if self._load_dtd:
+            ctxt.loadSubset(1)
         ctxt.replaceEntities(0)
         ctxt.parseDocument()
         trnode = ctxt.doc().getRootElement()
@@ -1031,6 +1048,8 @@ class Document (object):
         if node.type in ('text', 'cdata') and msg is not None:
             msg.add_text(node.content)
             return
+        if node.type == 'entity_ref':
+            msg.add_entity_ref(node.name);
         if node.type != 'element':
             return
         if node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes':
@@ -1408,6 +1427,16 @@ if __name__ == '__main__':
                        dest='strict',
                        default=False,
                        help='Exit with error when PO files contain broken XML')
+    options.add_option('-d', '--load-dtd',
+                       action='store_true',
+                       dest='load_dtd',
+                       default=False,
+                       help='Load external DTDs used by input XML')
+    options.add_option('-k', '--keep-entities',
+                       action='store_true',
+                       dest='keep_entities',
+                       default=False,
+                       help='Keep entity reference unexpanded')
     options.add_option('-t', '--test',
                        dest='test',
                        default=None,
@@ -1427,7 +1456,7 @@ if __name__ == '__main__':
     if opts.merge is None and opts.join is None:
         messages = MessageList()
         for filename in args[1:]:
-            doc = Document(filename, messages)
+            doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities)
             doc.apply_its_rules(not(opts.nobuiltins))
             if opts.itsfile is not None:
                 for itsfile in opts.itsfile:
@@ -1469,7 +1498,7 @@ if __name__ == '__main__':
             sys.exit(1)
         for filename in args[1:]:
             messages = MessageList()
-            doc = Document(filename, messages)
+            doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities)
             doc.apply_its_rules(not(opts.nobuiltins))
             if opts.itsfile is not None:
                 for itsfile in opts.itsfile: