Port to Py3.3.

9 files changed, 130 insertions, 94 deletions

diff --git a/setup.py b/setup.py
index 80e3ca0..774d9d3 100644
--- a/setup.py
+++ b/setup.py
@@ -104,6 +104,7 @@ setup(name='zope.pagetemplate',
           untrusted=['zope.untrustedpython'],
       ),
       install_requires=['setuptools',
+                        'six',
                         'zope.interface',
                         'zope.component',
                         'zope.tales',
diff --git a/src/zope/pagetemplate/pagetemplate.py b/src/zope/pagetemplate/pagetemplate.py
index 055208b..33b0ee1 100644
--- a/src/zope/pagetemplate/pagetemplate.py
+++ b/src/zope/pagetemplate/pagetemplate.py
@@ -16,6 +16,7 @@
 HTML- and XML-based template objects using TAL, TALES, and METAL.
 """
 import sys
+import six
 from zope.tal.talparser import TALParser
 from zope.tal.htmltalparser import HTMLTALParser
 from zope.tal.talgenerator import TALGenerator
@@ -30,7 +31,6 @@ from zope.interface import implementer
 from zope.interface import provider
 
 _default_options = {}
-_error_start = '<!-- Page Template Diagnostics'
 
 
 class StringIO(list):
@@ -73,6 +73,9 @@ class PageTemplate(object):
         to perform the rendering.
     """
 
+    _error_start = '<!-- Page Template Diagnostics'
+    _error_end = '-->'
+    _newline = '\n'
 
     content_type = 'text/html'
     expand = 1
@@ -143,17 +146,28 @@ class PageTemplate(object):
             except:
                 return ('Macro expansion failed', '%s: %s' % sys.exc_info()[:2])
 
+    def _convert(self, string, text):
+        """Adjust the string type to the type of text"""
+        if isinstance(text, six.binary_type):
+            return string.encode('utf-8')
+        else:
+            return string
+
     def write(self, text):
         # We accept both, since the text can either come from a file (and the
         # parser will take care of the encoding) or from a TTW template, in
         # which case we already have unicode.
-        assert isinstance(text, (str, unicode))
+        assert isinstance(text, (six.string_types, six.binary_type))
+
+        def bs(s):
+            """Bytes or str"""
+            return self._convert(s, text)
 
-        if text.startswith(_error_start):
-            errend = text.find('-->')
+        if text.startswith(bs(self._error_start)):
+            errend = text.find(bs(self._error_end))
             if errend >= 0:
                 text = text[errend + 3:]
-                if text[:1] == "\n":
+                if text[:1] == bs(self._newline):
                     text = text[1:]
         if self._text != text:
             self._text = text
@@ -165,6 +179,9 @@ class PageTemplate(object):
     def read(self, request=None):
         """Gets the source, sometimes with macros expanded."""
         self._cook_check()
+        def bs(s):
+            """Bytes or str"""
+            return self._convert(s, self._text)
         if not self._v_errors:
             if not self.expand:
                 return self._text
@@ -175,13 +192,13 @@ class PageTemplate(object):
                 context = self.pt_getContext(self, request)
                 return self.pt_render(context, source=1)
             except:
-                return ('%s\n Macro expansion failed\n %s\n-->\n%s' %
-                        (_error_start, "%s: %s" % sys.exc_info()[:2],
-                         self._text) )
+                return (bs('%s\n Macro expansion failed\n %s\n-->\n' %
+                           (_error_start, "%s: %s" % sys.exc_info()[:2])) +
+                        self._text)
 
-        return ('%s\n %s\n-->\n%s' % (_error_start,
-                                      '\n'.join(self._v_errors),
-                                      self._text))
+        return bs('%s\n %s\n-->\n' % (self._error_start,
+                                      '\n'.join(self._v_errors))) + \
+               self._text
 
     def pt_source_file(self):
         """To be overridden."""
diff --git a/src/zope/pagetemplate/pagetemplatefile.py b/src/zope/pagetemplate/pagetemplatefile.py
index b8bdb06..74b099f 100644
--- a/src/zope/pagetemplate/pagetemplatefile.py
+++ b/src/zope/pagetemplate/pagetemplatefile.py
@@ -23,13 +23,14 @@ import sys
 import re
 import logging
 
+import six
 from zope.pagetemplate.pagetemplate import PageTemplate
 
 DEFAULT_ENCODING = "utf-8"
 
 meta_pattern = re.compile(
-    r'\s*<meta\s+http-equiv=["\']?Content-Type["\']?'
-    r'\s+content=["\']?([^;]+);\s*charset=([^"\']+)["\']?\s*/?\s*>\s*',
+    br'\s*<meta\s+http-equiv=["\']?Content-Type["\']?'
+    br'\s+content=["\']?([^;]+);\s*charset=([^"\']+)["\']?\s*/?\s*>\s*',
     re.IGNORECASE)
 
 def package_home(gdict):
@@ -41,6 +42,10 @@ class PageTemplateFile(PageTemplate):
 
     _v_last_read = 0
 
+    #_error_start = b'<!-- Page Template Diagnostics'
+    #_error_end = b'-->'
+    #_newline = b'\n'
+
     def __init__(self, filename, _prefix=None):
         path = self.get_path_from_prefix(_prefix)
         self.filename = os.path.join(path, filename)
@@ -59,14 +64,15 @@ class PageTemplateFile(PageTemplate):
     def _prepare_html(self, text):
         match = meta_pattern.search(text)
         if match is not None:
-            type_, encoding = match.groups()
+            type_, encoding = (x.decode('utf-8') for x in match.groups())
             # TODO: Shouldn't <meta>/<?xml?> stripping
             # be in PageTemplate.__call__()?
-            text = meta_pattern.sub("", text)
+            text = meta_pattern.sub(b"", text)
         else:
             type_ = None
             encoding = DEFAULT_ENCODING
-        return unicode(text, encoding), type_
+        text = text.decode(encoding)
+        return text, type_
 
     def _read_file(self):
         __traceback_info__ = self.filename
@@ -77,13 +83,8 @@ class PageTemplateFile(PageTemplate):
             f.close()
             raise
         type_ = sniff_type(text)
-        if type_ == "text/xml":
-            text += f.read()
-        else:
-            # For HTML, we really want the file read in text mode:
-            f.close()
-            f = open(self.filename)
-            text = f.read()
+        text += f.read()
+        if type_ != "text/xml":
             text, type_ = self._prepare_html(text)
         f.close()
         return text, type_
@@ -114,12 +115,12 @@ class PageTemplateFile(PageTemplate):
         raise TypeError("non-picklable object")
 
 XML_PREFIXES = [
-    "<?xml",                      # ascii, utf-8
-    "\xef\xbb\xbf<?xml",          # utf-8 w/ byte order mark
-    "\0<\0?\0x\0m\0l",            # utf-16 big endian
-    "<\0?\0x\0m\0l\0",            # utf-16 little endian
-    "\xfe\xff\0<\0?\0x\0m\0l",    # utf-16 big endian w/ byte order mark
-    "\xff\xfe<\0?\0x\0m\0l\0",    # utf-16 little endian w/ byte order mark
+    b"<?xml",                      # ascii, utf-8
+    b"\xef\xbb\xbf<?xml",          # utf-8 w/ byte order mark
+    b"\0<\0?\0x\0m\0l",            # utf-16 big endian
+    b"<\0?\0x\0m\0l\0",            # utf-16 little endian
+    b"\xfe\xff\0<\0?\0x\0m\0l",    # utf-16 big endian w/ byte order mark
+    b"\xff\xfe<\0?\0x\0m\0l\0",    # utf-16 little endian w/ byte order mark
     ]
 
 XML_PREFIX_MAX_LENGTH = max(map(len, XML_PREFIXES))
diff --git a/src/zope/pagetemplate/tests/input/checkpathalt.html b/src/zope/pagetemplate/tests/input/checkpathalt.html
index 3026cda..1407d24 100644
--- a/src/zope/pagetemplate/tests/input/checkpathalt.html
+++ b/src/zope/pagetemplate/tests/input/checkpathalt.html
@@ -11,7 +11,7 @@
    <p tal:attributes="name y/z | nil">Z</p>
    <p tal:attributes="name y/z | nothing">Z</p>
 
-   <p tal:on-error="python:str(error.value[0])" tal:content="a/b | c/d">Z</p>
+   <p tal:on-error="python:str(error.value.args[0])" tal:content="a/b | c/d">Z</p>
    </div>
 </body>
 </html>
diff --git a/src/zope/pagetemplate/tests/test_basictemplate.py b/src/zope/pagetemplate/tests/test_basictemplate.py
index 4f4d8c5..f42ea1a 100644
--- a/src/zope/pagetemplate/tests/test_basictemplate.py
+++ b/src/zope/pagetemplate/tests/test_basictemplate.py
@@ -68,8 +68,8 @@ class BasicTemplateTests(unittest.TestCase):
         self.t.write("<tal:block define='a string:foo'>xyz")
         try:
             self.t.pt_render({})
-        except zope.pagetemplate.pagetemplate.PTRuntimeError, e:
-            self.assertEquals(
+        except zope.pagetemplate.pagetemplate.PTRuntimeError as e:
+            self.assertEqual(
                 str(e),
                 "['Compilation failed', 'zope.tal.taldefs.TALError:"
                 " TAL attributes on <tal:block> require explicit"
@@ -208,11 +208,11 @@ class BasicTemplateTests(unittest.TestCase):
 
         # test with HTML parser
         self.t.pt_edit(text, 'text/html')
-        self.assertEquals(self.t().strip(), text)
+        self.assertEqual(self.t().strip(), text)
 
         # test with XML parser
         self.t.pt_edit(text, 'text/xml')
-        self.assertEquals(self.t().strip(), text)
+        self.assertEqual(self.t().strip(), text)
 
 def test_suite():
     return unittest.makeSuite(BasicTemplateTests)
diff --git a/src/zope/pagetemplate/tests/test_engine.py b/src/zope/pagetemplate/tests/test_engine.py
index 761bce9..58fe97e 100644
--- a/src/zope/pagetemplate/tests/test_engine.py
+++ b/src/zope/pagetemplate/tests/test_engine.py
@@ -13,8 +13,12 @@
 ##############################################################################
 """Doc tests for the pagetemplate's 'engine' module
 """
+import doctest
+import re
 import unittest
 import zope.pagetemplate.engine
+from zope.testing.renormalizing import RENormalizing
+
 
 class DummyNamespace(object):
 
@@ -80,10 +84,21 @@ class ZopePythonExprTests(unittest.TestCase):
         self.assertRaises(NameError, expr, DummyContext())
 
 
+
 def test_suite():
-    from doctest import DocTestSuite
+
+    checker = RENormalizing([
+        # Python 3 includes module name in exceptions
+        (re.compile(r"zope.security.interfaces.ForbiddenAttribute"),
+         "ForbiddenAttribute"),
+        (re.compile(r"<class 'zope.security._proxy._Proxy'>"),
+         "<type 'zope.security._proxy._Proxy'>"),
+        (re.compile(r"<class 'list'>"), "<type 'list'>"),
+    ])
+
     suite = unittest.TestSuite()
-    suite.addTest(DocTestSuite('zope.pagetemplate.engine'))
+    suite.addTest(doctest.DocTestSuite('zope.pagetemplate.engine',
+                                       checker=checker))
     suite.addTest(unittest.makeSuite(EngineTests))
     if zope.pagetemplate.engine.HAVE_UNTRUSTED:
         suite.addTest(unittest.makeSuite(ZopePythonExprTests))
diff --git a/src/zope/pagetemplate/tests/test_ptfile.py b/src/zope/pagetemplate/tests/test_ptfile.py
index 4bf9ac9..f7a0457 100644
--- a/src/zope/pagetemplate/tests/test_ptfile.py
+++ b/src/zope/pagetemplate/tests/test_ptfile.py
@@ -17,6 +17,7 @@ import os
 import tempfile
 import unittest
 
+import six
 from zope.pagetemplate.pagetemplatefile import PageTemplateFile
 
 
@@ -42,78 +43,78 @@ class TypeSniffingTestCase(unittest.TestCase):
 
     def test_sniffer_xml_ascii(self):
         self.check_content_type(
-            "<?xml version='1.0' encoding='ascii'?><doc/>",
+            b"<?xml version='1.0' encoding='ascii'?><doc/>",
             "text/xml")
         self.check_content_type(
-            "<?xml\tversion='1.0' encoding='ascii'?><doc/>",
+            b"<?xml\tversion='1.0' encoding='ascii'?><doc/>",
             "text/xml")
 
     def test_sniffer_xml_utf8(self):
         # w/out byte order mark
         self.check_content_type(
-            "<?xml version='1.0' encoding='utf-8'?><doc/>",
+            b"<?xml version='1.0' encoding='utf-8'?><doc/>",
             "text/xml")
         self.check_content_type(
-            "<?xml\tversion='1.0' encoding='utf-8'?><doc/>",
+            b"<?xml\tversion='1.0' encoding='utf-8'?><doc/>",
             "text/xml")
         # with byte order mark
         self.check_content_type(
-            "\xef\xbb\xbf<?xml version='1.0' encoding='utf-8'?><doc/>",
+            b"\xef\xbb\xbf<?xml version='1.0' encoding='utf-8'?><doc/>",
             "text/xml")
         self.check_content_type(
-            "\xef\xbb\xbf<?xml\tversion='1.0' encoding='utf-8'?><doc/>",
+            b"\xef\xbb\xbf<?xml\tversion='1.0' encoding='utf-8'?><doc/>",
             "text/xml")
 
     def test_sniffer_xml_utf16_be(self):
         # w/out byte order mark
         self.check_content_type(
-            "\0<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
-            "\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
-            "\0<\0d\0o\0c\0/\0>",
+            b"\0<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
+            b"\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
+            b"\0<\0d\0o\0c\0/\0>",
             "text/xml")
         self.check_content_type(
-            "\0<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
-            "\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
-            "\0<\0d\0o\0c\0/\0>",
+            b"\0<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
+            b"\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
+            b"\0<\0d\0o\0c\0/\0>",
             "text/xml")
         # with byte order mark
         self.check_content_type(
-            "\xfe\xff"
-            "\0<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
-            "\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
-            "\0<\0d\0o\0c\0/\0>",
+            b"\xfe\xff"
+            b"\0<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
+            b"\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
+            b"\0<\0d\0o\0c\0/\0>",
             "text/xml")
         self.check_content_type(
-            "\xfe\xff"
-            "\0<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
-            "\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
-            "\0<\0d\0o\0c\0/\0>",
+            b"\xfe\xff"
+            b"\0<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'"
+            b"\0 \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>"
+            b"\0<\0d\0o\0c\0/\0>",
             "text/xml")
 
     def test_sniffer_xml_utf16_le(self):
         # w/out byte order mark
         self.check_content_type(
-            "<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
-            " \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
-            "<\0d\0o\0c\0/\0>\n",
+            b"<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
+            b" \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
+            b"<\0d\0o\0c\0/\0>\n",
             "text/xml")
         self.check_content_type(
-            "<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
-            " \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
-            "<\0d\0o\0c\0/\0>\0",
+            b"<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
+            b" \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
+            b"<\0d\0o\0c\0/\0>\0",
             "text/xml")
         # with byte order mark
         self.check_content_type(
-            "\xff\xfe"
-            "<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
-            " \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
-            "<\0d\0o\0c\0/\0>\0",
+            b"\xff\xfe"
+            b"<\0?\0x\0m\0l\0 \0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
+            b" \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
+            b"<\0d\0o\0c\0/\0>\0",
             "text/xml")
         self.check_content_type(
-            "\xff\xfe"
-            "<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
-            " \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
-            "<\0d\0o\0c\0/\0>\0",
+            b"\xff\xfe"
+            b"<\0?\0x\0m\0l\0\t\0v\0e\0r\0s\0i\0o\0n\0=\0'\01\0.\0000\0'\0"
+            b" \0e\0n\0c\0o\0d\0i\0n\0g\0=\0'\0u\0t\0f\0-\08\0'\0?\0>\0"
+            b"<\0d\0o\0c\0/\0>\0",
             "text/xml")
 
     HTML_PUBLIC_ID = "-//W3C//DTD HTML 4.01 Transitional//EN"
@@ -121,11 +122,11 @@ class TypeSniffingTestCase(unittest.TestCase):
 
     def test_sniffer_html_ascii(self):
         self.check_content_type(
-            "<!DOCTYPE html [ SYSTEM '%s' ]><html></html>"
-            % self.HTML_SYSTEM_ID,
+            ("<!DOCTYPE html [ SYSTEM '%s' ]><html></html>"
+            % self.HTML_SYSTEM_ID).encode("utf-8"),
             "text/html")
         self.check_content_type(
-            "<html><head><title>sample document</title></head></html>",
+            b"<html><head><title>sample document</title></head></html>",
             "text/html")
 
     # TODO: This reflects a case that simply isn't handled by the
@@ -137,43 +138,43 @@ class TypeSniffingTestCase(unittest.TestCase):
 
     def test_html_default_encoding(self):
         pt = self.get_pt(
-            "<html><head><title>"
+            b"<html><head><title>"
             # 'Test' in russian (utf-8)
-            "\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82"
-            "</title></head></html>")
+            b"\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82"
+            b"</title></head></html>")
         rendered = pt()
-        self.failUnless(isinstance(rendered, unicode))
-        self.failUnlessEqual(rendered.strip(),
+        self.assertTrue(isinstance(rendered, six.text_type))
+        self.assertEqual(rendered.strip(),
             u"<html><head><title>"
             u"\u0422\u0435\u0441\u0442"
             u"</title></head></html>")
 
     def test_html_encoding_by_meta(self):
         pt = self.get_pt(
-            "<html><head><title>"
+            b"<html><head><title>"
             # 'Test' in russian (windows-1251)
-            "\xd2\xe5\xf1\xf2"
-            '</title><meta http-equiv="Content-Type"'
-            ' content="text/html; charset=windows-1251">'
-            "</head></html>")
+            b"\xd2\xe5\xf1\xf2"
+            b'</title><meta http-equiv="Content-Type"'
+            b' content="text/html; charset=windows-1251">'
+            b"</head></html>")
         rendered = pt()
-        self.failUnless(isinstance(rendered, unicode))
-        self.failUnlessEqual(rendered.strip(),
+        self.assertTrue(isinstance(rendered, six.text_type))
+        self.assertEqual(rendered.strip(),
             u"<html><head><title>"
             u"\u0422\u0435\u0441\u0442"
             u"</title></head></html>")
 
     def test_xhtml(self):
         pt = self.get_pt(
-            "<html><head><title>"
+            b"<html><head><title>"
             # 'Test' in russian (windows-1251)
-            "\xd2\xe5\xf1\xf2"
-            '</title><meta http-equiv="Content-Type"'
-            ' content="text/html; charset=windows-1251"/>'
-            "</head></html>")
+            b"\xd2\xe5\xf1\xf2"
+            b'</title><meta http-equiv="Content-Type"'
+            b' content="text/html; charset=windows-1251"/>'
+            b"</head></html>")
         rendered = pt()
-        self.failUnless(isinstance(rendered, unicode))
-        self.failUnlessEqual(rendered.strip(),
+        self.assertTrue(isinstance(rendered, six.text_type))
+        self.assertEqual(rendered.strip(),
             u"<html><head><title>"
             u"\u0422\u0435\u0441\u0442"
             u"</title></head></html>")
diff --git a/src/zope/pagetemplate/tests/util.py b/src/zope/pagetemplate/tests/util.py
index 113cd81..47c84b3 100644
--- a/src/zope/pagetemplate/tests/util.py
+++ b/src/zope/pagetemplate/tests/util.py
@@ -13,6 +13,7 @@
 ##############################################################################
 """Utilities
 """
+from __future__ import print_function
 import os
 import re
 import sys
@@ -70,7 +71,7 @@ def nicerange(lo, hi):
 
 def dump(tag, x, lo, hi):
     for i in xrange(lo, hi):
-        print '%s %s' % (tag, x[i]),
+        print('%s %s' % (tag, x[i]), end=' ')
 
 def check_html(s1, s2):
     s1 = normalize_html(s1)
diff --git a/tox.ini b/tox.ini
index d8b6a3b..0c459e6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,7 @@
 [tox]
 envlist =
     # py32 needs a py3k-compatible zope.security at the very least
-    py26,py27
+    py26,py27,py33
 
 [testenv]
 deps =