[svn] Add encoding support. All processing is now done with unicode strings.

1 files changed, 4 insertions, 3 deletions

diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py
index 6347ab88..247f986d 100644
--- a/tests/test_examplefiles.py
+++ b/tests/test_examplefiles.py
@@ -38,11 +38,12 @@ for fn in os.listdir(os.path.join(testdir, 'examplefiles')):
     def test(self, lx=lx, absfn=absfn):
         text = file(absfn, 'U').read()
         text = text.strip('\n') + '\n'
-        ntext = ''
+        text = text.decode('latin1')
+        ntext = []
         for type, val in lx.get_tokens(text):
-            ntext += val
+            ntext.append(val)
             self.failIf(type == Error, 'lexer generated error token for '+absfn)
-        if ntext != text:
+        if u''.join(ntext) != text:
             self.fail('round trip failed for '+absfn)
 
     setattr(ExampleFileTest, 'test_%i' % lfd, test)