Merge branch 'next'

Conflicts:
	cheetah/Template.py
	cheetah/Tests/Performance.py
	cheetah/Version.py

7 files changed, 125 insertions, 83 deletions

diff --git a/CHANGES b/CHANGES
index 56c1d07..46c0b0e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,14 @@
 
+2.4.0 (October 15th, 2009)
+  - Fix a major performance regression in Template.__init__()
+  - More graceful handling of unicode when calling .respond() to render a template
+  - Minor code updates
+
+2.3.0 (October 15th, 2009) (loosely equivalent to 2.4.0)
+  - Fix a major performance regression in Template.__init__()
+  - More graceful handling of unicode when calling .respond() to render a template
+  - Minor code updates
+
 2.2.2 (September 10th, 2009)
   - Prevent _namemapper.c from segfaulting when PyImport_ImportModule fails for some reason (Bogdano Arendartchuk <debogdano@gmail.com>)
   - Removal of the contrib/markdown module (in favor of a setuptools dependency)
diff --git a/cheetah/DummyTransaction.py b/cheetah/DummyTransaction.py
index 26d2ea7..f84ade4 100644
--- a/cheetah/DummyTransaction.py
+++ b/cheetah/DummyTransaction.py
@@ -8,6 +8,7 @@ Warning: This may be deprecated in the future, please do not rely on any
 specific DummyTransaction or DummyResponse behavior
 '''
 
+import logging
 import types
 
 class DummyResponseFailure(Exception):
@@ -24,7 +25,25 @@ class DummyResponse(object):
 
     def flush(self):
         pass
-        
+
+    def safeConvert(self, chunk):
+        # Exceptionally gross, but the safest way
+        # I've found to ensure I get a legit unicode object
+        if not chunk:
+            return u''
+        if isinstance(chunk, unicode):
+            return chunk
+        try:
+            return chunk.decode('utf-8', 'strict')
+        except UnicodeDecodeError:
+            try:
+                return chunk.decode('latin-1', 'strict')
+            except UnicodeDecodeError:
+                return chunk.decode('ascii', 'ignore')
+        except AttributeError:
+            return unicode(chunk, errors='ignore')
+        return chunk
+
     def write(self, value):
         self._outputChunks.append(value)
 
@@ -34,18 +53,12 @@ class DummyResponse(object):
 
     def getvalue(self, outputChunks=None):
         chunks = outputChunks or self._outputChunks
-        try: 
-            return ''.join(chunks)
+        try:
+            return u''.join(chunks)
         except UnicodeDecodeError, ex:
-            nonunicode = [c for c in chunks if not isinstance(c, unicode)]
-            raise DummyResponseFailure('''Looks like you're trying to mix encoded strings with Unicode strings
-            (most likely utf-8 encoded ones)
-
-            This can happen if you're using the `EncodeUnicode` filter, or if you're manually
-            encoding strings as utf-8 before passing them in on the searchList (possible offenders: 
-            %s) 
-            (%s)''' % (nonunicode, ex))
-
+            logging.debug('Trying to work around a UnicodeDecodeError in getvalue()')
+            logging.debug('...perhaps you could fix "%s" while you\'re debugging')
+            return ''.join((self.safeConvert(c) for c in chunks))
 
     def writelines(self, *lines):
         ## not used
diff --git a/cheetah/Template.py b/cheetah/Template.py
index c903460..a8889d2 100644
--- a/cheetah/Template.py
+++ b/cheetah/Template.py
@@ -95,6 +95,7 @@ def hashDict(d):
         hashedList.append((k,v))
     return hash(tuple(hashedList))
 
+
 ################################################################################
 ## MODULE GLOBALS AND CONSTANTS
 
@@ -579,19 +580,12 @@ class Template(Servlet):
                           preprocessors=[ dict(tokens='@ %', searchList=[...]) ] )
             
         """
-        ##################################################           
-        ## normalize and validate args 
-        N = types.NoneType; S = types.StringType; U = types.UnicodeType
-        D = types.DictType; F = types.FileType
-        C = types.ClassType;  M = types.ModuleType
-        I = types.IntType; B = types.BooleanType
         errmsg = "arg '%s' must be %s"
 
-        t = type(source)
-        if not (t is N or t is S or t is U):
+        if not isinstance(source, (types.NoneType, basestring)):
             raise TypeError(errmsg % ('source', 'string or None'))
-        t = type(file)
-        if not (t is N or t is S or t is U or t is F):
+
+        if not isinstance(file, (types.NoneType, basestring, types.FileType)):
             raise TypeError(errmsg %
                             ('file', 'string, file-like object, or None'))
 
@@ -599,25 +593,25 @@ class Template(Servlet):
             baseclass = klass._CHEETAH_defaultBaseclassForTemplates
         if isinstance(baseclass, Template):
             baseclass = baseclass.__class__
-        t = type(baseclass)
-        if not (t is N or t is S or t is C or t is type):
+
+        if not isinstance(baseclass, (types.NoneType, basestring, types.ClassType, types.TypeType)):
             raise TypeError(errmsg % ('baseclass', 'string, class or None'))
 
         if cacheCompilationResults is Unspecified:
             cacheCompilationResults = klass._CHEETAH_cacheCompilationResults
-        t = type(cacheCompilationResults)
-        if not (t is I or t is B):
+
+        if not isinstance(cacheCompilationResults, (int, bool)):
             raise TypeError(errmsg % ('cacheCompilationResults', 'boolean'))
 
         if useCache is Unspecified:
             useCache = klass._CHEETAH_useCompilationCache
-        t = type(useCache)
-        if not (t is I or t is B):
+
+        if not isinstance(useCache, (int, bool)):
             raise TypeError(errmsg % ('useCache', 'boolean'))
 
         if compilerSettings is Unspecified:
             compilerSettings = klass._getCompilerSettings(source, file) or {}
-        if type(compilerSettings) is not D:
+        if not isinstance(compilerSettings, dict):
             raise TypeError(errmsg % ('compilerSettings', 'dictionary'))
 
         if compilerClass is Unspecified:
@@ -627,16 +621,15 @@ class Template(Servlet):
 
         if keepRefToGeneratedCode is Unspecified:
             keepRefToGeneratedCode = klass._CHEETAH_keepRefToGeneratedCode
-        t = type(keepRefToGeneratedCode)
-        if not (t is I or t is B):
+
+        if not isinstance(keepRefToGeneratedCode, (int, bool)):
             raise TypeError(errmsg % ('keepReftoGeneratedCode', 'boolean'))
 
-        t = type(moduleName)
-        if not (t is N or t is S):
+        if not isinstance(moduleName, (types.NoneType, basestring)):
             raise TypeError(errmsg % ('moduleName', 'string or None'))
         __orig_file__ = None
         if not moduleName:
-            if file and type(file) in StringTypes:
+            if file and isinstance(file, basestring):
                 moduleName = convertTmplPathToModuleName(file)
                 __orig_file__ = file
             else:
@@ -644,15 +637,15 @@ class Template(Servlet):
 
         if className is Unspecified:
             className = klass._CHEETAH_defaultClassNameForTemplates
-        t = type(className)
-        if not (t is N or t is S):
+
+        if not isinstance(className, (types.NoneType, basestring)):
             raise TypeError(errmsg % ('className', 'string or None'))
         className = className or moduleName
 
         if mainMethodName is Unspecified:
             mainMethodName = klass._CHEETAH_defaultMainMethodNameForTemplates
-        t = type(mainMethodName)
-        if not (t is N or t is S):
+
+        if not isinstance(mainMethodName, (types.NoneType, basestring)):
             raise TypeError(errmsg % ('mainMethodName', 'string or None'))
 
         if moduleGlobals is Unspecified:
@@ -660,15 +653,15 @@ class Template(Servlet):
 
         if cacheModuleFilesForTracebacks is Unspecified:
             cacheModuleFilesForTracebacks = klass._CHEETAH_cacheModuleFilesForTracebacks
-        t = type(cacheModuleFilesForTracebacks)
-        if not (t is I or t is B):
+
+        if not isinstance(cacheModuleFilesForTracebacks, (int, bool)):
             raise TypeError(errmsg %
                             ('cacheModuleFilesForTracebacks', 'boolean'))
 
         if cacheDirForModuleFiles is Unspecified:
             cacheDirForModuleFiles = klass._CHEETAH_cacheDirForModuleFiles
-        t = type(cacheDirForModuleFiles)
-        if not (t is N or t is S):
+
+        if not isinstance(cacheDirForModuleFiles, (types.NoneType, basestring)):
             raise TypeError(errmsg %
                             ('cacheDirForModuleFiles', 'string or None'))
 
@@ -683,9 +676,9 @@ class Template(Servlet):
         baseclassValue = None
         baseclassName = None
         if baseclass:
-            if type(baseclass) in StringTypes:
+            if isinstance(baseclass, basestring):
                 baseclassName = baseclass
-            elif type(baseclass) in (ClassType, type):
+            elif isinstance(baseclass, (types.ClassType, types.TypeType)):
                 # @@TR: should soft-code this
                 baseclassName = 'CHEETAH_dynamicallyAssignedBaseClass_'+baseclass.__name__
                 baseclassValue = baseclass
@@ -1142,46 +1135,41 @@ class Template(Servlet):
 
           Do NOT mess with the args _globalSetVars or _preBuiltSearchList!
 
-        """
-        
-        ##################################################           
-        ## Verify argument keywords and types
 
-        S = types.StringType; U = types.UnicodeType
-        L = types.ListType;   T = types.TupleType
-        D = types.DictType;   F = types.FileType
-        C = types.ClassType;  M = types.ModuleType
-        N = types.NoneType
+        """
         errmsg = "arg '%s' must be %s"
         errmsgextra = errmsg + "\n%s"
 
-        t = type(source)
-        if not (t is N or t is S or t is U):
+        if not isinstance(source, (types.NoneType, basestring)):
             raise TypeError(errmsg % ('source', 'string or None'))
-        t = type(file)
-        if not (t is N or t is S or t is U or t is F):
+
+        if not isinstance(source, (types.NoneType, basestring, types.FileType)):
             raise TypeError(errmsg %
                             ('file', 'string, file open for reading, or None'))
-        t = type(filter)
-        if not (t is S or (t is C and issubclass(filter, Filters.Filter)) or
-                t is type):
+
+        if not isinstance(filter, (basestring, types.TypeType)) and not \
+                (isinstance(filter, types.ClassType) and issubclass(filter, Filters.Filter)):
             raise TypeError(errmsgextra %
                             ('filter', 'string or class',
                              '(if class, must be subclass of Cheetah.Filters.Filter)'))
-        t = type(filtersLib)
-        if not (t is S or t is M):
+        if not isinstance(filtersLib, (basestring, types.ModuleType)):
             raise TypeError(errmsgextra %
                             ('filtersLib', 'string or module',
                              '(if module, must contain subclasses of Cheetah.Filters.Filter)'))
-        t = type(errorCatcher)
-        if not (t is N or t is S or
-                (t is C and issubclass(errorCatcher, ErrorCatchers.ErrorCatcher)) or
-                t is type):
-            raise TypeError(errmsgextra %
+
+        if not errorCatcher is None:
+            err = True
+            if isinstance(errorCatcher, (basestring, types.TypeType)):
+                err = False
+            if isinstance(errorCatcher, types.ClassType) and \
+                    issubclass(errorCatcher, ErrorCatchers.ErrorCatcher): 
+                err = False
+            if err:
+                raise TypeError(errmsgextra %
                             ('errorCatcher', 'string, class or None',
                              '(if class, must be subclass of Cheetah.ErrorCatchers.ErrorCatcher)'))
         if compilerSettings is not Unspecified:
-            if type(compilerSettings) is not D:
+            if not isinstance(compilerSettings, types.DictType):
                 raise TypeError(errmsg %
                                 ('compilerSettings', 'dictionary'))
         
@@ -1216,7 +1204,7 @@ class Template(Servlet):
         if searchList:
             for namespace in searchList:
                 if isinstance(namespace, dict):
-                    intersection = Reserved_SearchList & set(namespace.keys())
+                    intersection = self.Reserved_SearchList & set(namespace.keys())
                     warn = False
                     if intersection:
                         warn = True
@@ -1525,7 +1513,7 @@ class Template(Servlet):
         self._fileMtime = None
         self._fileDirName = None
         self._fileBaseName = None
-        if file and type(file) in StringTypes:
+        if file and isinstance(file, basestring):
             file = self.serverSidePath(file)
             self._fileMtime = os.path.getmtime(file)
             self._fileDirName, self._fileBaseName = os.path.split(file)
@@ -1831,7 +1819,7 @@ class Template(Servlet):
         return dic
 
 T = Template   # Short and sweet for debugging at the >>> prompt.
-Reserved_SearchList = set(dir(Template))
+Template.Reserved_SearchList = set(dir(Template))
 
 def genParserErrorFromPythonException(source, file, generatedPyCode, exception):
 
diff --git a/cheetah/Tests/CheetahWrapper.py b/cheetah/Tests/CheetahWrapper.py
index ca7b0ae..e152e68 100644
--- a/cheetah/Tests/CheetahWrapper.py
+++ b/cheetah/Tests/CheetahWrapper.py
@@ -12,7 +12,6 @@ Besides unittest usage, recognizes the following command-line options:
         Show the output of each subcommand.  (Normally suppressed.)
 '''
 import os
-import popen2
 import re                                     # Used by listTests.
 import shutil
 import sys
@@ -22,6 +21,18 @@ import unittest
 from optparse import OptionParser
 from Cheetah.CheetahWrapper import CheetahWrapper  # Used by NoBackup.
 
+try:
+    from subprocess import Popen, PIPE, STDOUT
+    class Popen4(Popen):
+        def __init__(self, cmd, bufsize=-1):
+            super(Popen4, self).__init__(cmd, bufsize=bufsize,
+                                         shell=True, close_fds=True,
+                                         stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+            self.tochild = self.stdin
+            self.fromchild = self.stdout
+            self.childerr = self.stderr
+except ImportError:
+    from popen2 import Popen4
 
 DELETE = True # True to clean up after ourselves, False for debugging.
 OUTPUT = False # Normally False, True for debugging.
@@ -152,7 +163,7 @@ Found %(result)r"""
         return rc, output
 
     def assertPosixSubprocess(self, cmd):
-        process = popen2.Popen4(cmd)
+        process = Popen4(cmd)
         process.tochild.close()
         output = process.fromchild.read()
         status = process.wait()
diff --git a/cheetah/Tests/Performance.py b/cheetah/Tests/Performance.py
index 213a02b..8101e85 100644
--- a/cheetah/Tests/Performance.py
+++ b/cheetah/Tests/Performance.py
@@ -1,8 +1,5 @@
 #!/usr/bin/env python
 
-import Cheetah.NameMapper 
-import Cheetah.Template
-
 import hotshot
 import hotshot.stats
 import os
@@ -12,6 +9,9 @@ import unittest
 from test import pystone
 import time
 
+import Cheetah.NameMapper 
+import Cheetah.Template
+
 # This can be turned on with the `--debug` flag when running the test
 # and will cause the tests to all just dump out how long they took
 # insteasd of asserting on duration
@@ -86,6 +86,7 @@ class DynamicTemplatePerformanceTest(unittest.TestCase):
 class PerformanceTest(unittest.TestCase):
     iterations = 100000
     display = False
+    save = False
 
     def runTest(self):
         self.prof = hotshot.Profile('%s.prof' % self.__class__.__name__)
@@ -100,9 +101,12 @@ class PerformanceTest(unittest.TestCase):
             print '>>> %s (%d iterations) ' % (self.__class__.__name__,
                     self.iterations)
             stats = hotshot.stats.load('%s.prof' % self.__class__.__name__)
-            stats.strip_dirs()
+            #stats.strip_dirs()
             stats.sort_stats('time', 'calls')
-            stats.print_stats(40)
+            stats.print_stats(50)
+
+        if not self.save:
+            os.unlink('%s.prof' % self.__class__.__name__)
 
 class DynamicMethodCompilationTest(PerformanceTest):
     def performanceSample(self):
@@ -119,6 +123,23 @@ class DynamicMethodCompilationTest(PerformanceTest):
         template = template()
         value = template.testMethod()
 
+
+class BunchOfWriteCalls(PerformanceTest):
+    iterations = 1000
+    def performanceSample(self):
+        template = '''
+            #import sys
+            #import os
+            #for i in xrange(1000)
+                $i
+            #end for
+        '''
+        template = Cheetah.Template.Template.compile(template, 
+            keepRefToGeneratedCode=False)
+        template = template()
+        value = template.respond()
+        del value
+
 class DynamicSimpleCompilationTest(PerformanceTest):
     def performanceSample(self):
         template = '''
diff --git a/cheetah/Tests/Unicode.py b/cheetah/Tests/Unicode.py
index e4499d9..d627503 100644
--- a/cheetah/Tests/Unicode.py
+++ b/cheetah/Tests/Unicode.py
@@ -168,15 +168,14 @@ class Unicode_in_SearchList_Test(CommandLineTest):
             'adjective' : u'\u0e22\u0e34\u0e19\u0e14\u0e35\u0e15\u0e49\u0e2d\u0e19\u0e23\u0e31\u0e1a'}])
         assert template.respond()
 
-    def test_ErrorReporting(self):
+    def test_Thai_utf8(self):
         utf8 = '\xe0\xb8\xa2\xe0\xb8\xb4\xe0\xb8\x99\xe0\xb8\x94\xe0\xb8\xb5\xe0\xb8\x95\xe0\xb9\x89\xe0\xb8\xad\xe0\xb8\x99\xe0\xb8\xa3\xe0\xb8\xb1\xe0\xb8\x9a'
 
         source = '''This is $adjective'''
         template = self.createAndCompile(source)
         assert template and issubclass(template, Template)
         template = template(searchList=[{'adjective' : utf8}])
-        self.failUnlessRaises(DummyTransaction.DummyResponseFailure, template.respond)
-
+        assert template.respond()
 
 
 if __name__ == '__main__':
diff --git a/cheetah/Version.py b/cheetah/Version.py
index eaf9c46..7fdf82f 100644
--- a/cheetah/Version.py
+++ b/cheetah/Version.py
@@ -1,5 +1,5 @@
-Version = '2.2.3'
-VersionTuple = (2, 2, 3,'final', 0)
+Version = '2.4.0'
+VersionTuple = (2, 4, 0, 'final', 0)
 
 MinCompatibleVersion = '2.0rc6'
 MinCompatibleVersionTuple = (2,0,0,'candidate',6)