From 87cb61133ea9f1b53b811d534e3fae73e41c5e9a Mon Sep 17 00:00:00 2001 From: "R. Tyler Ballance" Date: Fri, 16 Oct 2009 15:15:08 -0700 Subject: Add Template.__unicode__() to return unicode() objects, while Template.__str__() returns encoded str() objects Per my discussion in #cheetah on IRC with mikeb@ regarding the following issue: https://bugzilla.redhat.com/show_bug.cgi?id=529332 This, in addition to recent patches to cheetah/DummyTransaction.py should alleviate migration issues for users still passing a mishmash of unicode()/str() objects into Templates. __str__() should return a str() object, whereas __unicode__() should return a unicode() object. No-op the EncodeUnicode filter when it encounters a unicode() object. --- cheetah/Filters.py | 2 +- cheetah/Template.py | 25 ++++++++++++++++++++++++- cheetah/Tests/Unicode.py | 16 ++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cheetah/Filters.py b/cheetah/Filters.py index dd65f28..452afc5 100644 --- a/cheetah/Filters.py +++ b/cheetah/Filters.py @@ -58,7 +58,7 @@ class EncodeUnicode(Filter): >>> print t """ if isinstance(val, unicode): - return val.encode(encoding) + return val if val is None: return '' return str(val) diff --git a/cheetah/Template.py b/cheetah/Template.py index a8889d2..ec92208 100644 --- a/cheetah/Template.py +++ b/cheetah/Template.py @@ -994,22 +994,45 @@ class Template(Servlet): mainMethName = getattr(concreteTemplateClass,mainMethNameAttr, None) if mainMethName: def __str__(self): + rc = getattr(self, mainMethName)() + if isinstance(rc, unicode): + return rc.encode('utf-8') + return rc + def __unicode__(self): return getattr(self, mainMethName)() elif (hasattr(concreteTemplateClass, 'respond') and concreteTemplateClass.respond!=Servlet.respond): def __str__(self): + rc = self.respond() + if isinstance(rc, unicode): + return rc.encode('utf-8') + return rc + def __unicode__(self): return self.respond() else: def __str__(self): + rc = None + if hasattr(self, mainMethNameAttr): + rc = getattr(self,mainMethNameAttr)() + elif hasattr(self, 'respond'): + rc = self.respond() + else: + rc = super(self.__class__, self).__str__() + if isinstance(rc, unicode): + return rc.encode('utf-8') + return rc + def __unicode__(self): if hasattr(self, mainMethNameAttr): return getattr(self,mainMethNameAttr)() elif hasattr(self, 'respond'): return self.respond() else: - return super(self.__class__, self).__str__() + return super(self.__class__, self).__unicode__() __str__ = new.instancemethod(__str__, None, concreteTemplateClass) + __unicode__ = new.instancemethod(__unicode__, None, concreteTemplateClass) setattr(concreteTemplateClass, '__str__', __str__) + setattr(concreteTemplateClass, '__unicode__', __unicode__) _addCheetahPlumbingCodeToClass = classmethod(_addCheetahPlumbingCodeToClass) diff --git a/cheetah/Tests/Unicode.py b/cheetah/Tests/Unicode.py index d627503..12c00ac 100644 --- a/cheetah/Tests/Unicode.py +++ b/cheetah/Tests/Unicode.py @@ -150,6 +150,22 @@ $someUnicodeString""" a = unicode(template).encode("utf-8") self.assertEquals("Bébé", a) +class EncodeUnicodeCompatTest(unittest.TestCase): + """ + Taken initially from Red Hat's bugzilla #529332 + https://bugzilla.redhat.com/show_bug.cgi?id=529332 + """ + def runTest(self): + t = Template("""Foo ${var}""", filter='EncodeUnicode') + t.var = u"Text with some non-ascii characters: åäö" + + rc = t.respond() + assert isinstance(rc, unicode), ('Template.respond() should return unicode', rc) + + rc = str(t) + assert isinstance(rc, str), ('Template.__str__() should return a UTF-8 encoded string', rc) + + class Unicode_in_SearchList_Test(CommandLineTest): def test_BasicASCII(self): source = '''This is $adjective''' -- cgit v1.2.1 From 4213bbe9856d9c33a35ed2d7d5dba1ca16472015 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Fri, 16 Oct 2009 19:52:41 -0400 Subject: remove unnecessary encoding in the default filters, now that everything is a unicode object internally There is no longer any value in having the filters return encoded str objects, since they need to be decoded back to unicode before they can be combined with the rest of the template text (which is managed as a unicode object). This patch maintains API compatibility but fundamentally changes the behavior of some of the filters. RawOrEncodedUnicode and EncodeUnicode should be deprecated and removed from the API. Signed-off-by: R. Tyler Ballance --- cheetah/Filters.py | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/cheetah/Filters.py b/cheetah/Filters.py index 452afc5..d452439 100644 --- a/cheetah/Filters.py +++ b/cheetah/Filters.py @@ -29,40 +29,19 @@ class Filter(object): if val is None: return u'' if isinstance(val, unicode): - if encoding: - return val.encode(encoding) - else: - return val + # ignore the encoding and return the unicode object + return val else: try: - return str(val) - except UnicodeEncodeError: return unicode(val) - return u'' + except UnicodeDecodeError: + # we could put more fallbacks here, but we'll just pass the str + # on and let DummyTransaction worry about it + return str(val) RawOrEncodedUnicode = Filter -class EncodeUnicode(Filter): - def filter(self, val, - encoding='utf8', - str=str, - **kw): - """Encode Unicode strings, by default in UTF-8. - - >>> import Cheetah.Template - >>> t = Cheetah.Template.Template(''' - ... $myvar - ... ${myvar, encoding='utf16'} - ... ''', searchList=[{'myvar': u'Asni\xe8res'}], - ... filter='EncodeUnicode') - >>> print t - """ - if isinstance(val, unicode): - return val - if val is None: - return '' - return str(val) - +EncodeUnicode = Filter class Markdown(EncodeUnicode): ''' -- cgit v1.2.1 From a02a09a79c501ba82f3ce176632cec893d78c9d1 Mon Sep 17 00:00:00 2001 From: "R. Tyler Ballance" Date: Sat, 24 Oct 2009 17:31:27 -0700 Subject: tweak changes for release --- CHANGES | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 46c0b0e..1aff910 100644 --- a/CHANGES +++ b/CHANGES @@ -1,13 +1,15 @@ -2.4.0 (October 15th, 2009) +2.4.0 (October 24th, 2009) - Fix a major performance regression in Template.__init__() - More graceful handling of unicode when calling .respond() to render a template - Minor code updates + - Update the default filter (thanks mikeb!) -2.3.0 (October 15th, 2009) (loosely equivalent to 2.4.0) +2.3.0 (October 24th, 2009) (loosely equivalent to 2.4.0) - Fix a major performance regression in Template.__init__() - More graceful handling of unicode when calling .respond() to render a template - Minor code updates + - Update the default filter (thanks mikeb!) 2.2.2 (September 10th, 2009) - Prevent _namemapper.c from segfaulting when PyImport_ImportModule fails for some reason (Bogdano Arendartchuk ) -- cgit v1.2.1 From 7b1c2ad9f4e5830b5c88d95715014bb4044c090c Mon Sep 17 00:00:00 2001 From: "R. Tyler Ballance" Date: Sat, 24 Oct 2009 17:47:31 -0700 Subject: Update manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 266004d..f1d1938 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include MANIFEST.in *.py *.cfg TODO CHANGES LICENSE README examples docs bin +include MANIFEST.in *.py *.cfg TODO CHANGES LICENSE README.markdown examples docs bin recursive-include cheetah *.py *.tmpl *.txt recursive-include bin * recursive-include docs * -- cgit v1.2.1