fix: Unicode errors with Python 3.2+ POST parameters. (fix #344, fix #349)

3 files changed, 22 insertions, 16 deletions

diff --git a/bottle.py b/bottle.py
index 72bea26..ca70659 100644
--- a/bottle.py
+++ b/bottle.py
@@ -116,6 +116,7 @@ def touni(s, enc='utf8', err='strict'):
 tonat = touni if py3k else tob
 
 # 3.2 fixes cgi.FieldStorage to accept bytes (which makes a lot of sense).
+#     but defaults to utf-8 (which is not always true)
 # 3.1 needs a workaround.
 NCTextIOWrapper = None
 if (3,0,0) < py < (3,2,0):
@@ -1078,11 +1079,13 @@ class BaseRequest(object):
         safe_env = {'QUERY_STRING':''} # Build a safe environment for cgi
         for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'):
             if key in self.environ: safe_env[key] = self.environ[key]
+        args = dict(fp=self.body, environ=safe_env, keep_blank_values=True)
+        if py >= (3,2,0):
+            args['encoding'] = 'ISO-8859-1'
         if NCTextIOWrapper:
-            fb = NCTextIOWrapper(self.body, encoding='ISO-8859-1', newline='\n')
-        else:
-            fb = self.body
-        data = cgi.FieldStorage(fp=fb, environ=safe_env, keep_blank_values=True)
+            args['fp'] = NCTextIOWrapper(args['fp'], encoding='ISO-8859-1',
+                                         newline='\n')
+        data = cgi.FieldStorage(**args)
         for item in (data.list or [])[:self.MAX_PARAMS]:
             post[item.name] = item if item.filename else item.value
         return post
diff --git a/test/test_environ.py b/test/test_environ.py
index 6e441e9..0195b34 100755
--- a/test/test_environ.py
+++ b/test/test_environ.py
@@ -4,7 +4,7 @@
 import unittest
 import sys, os.path
 import bottle
-from bottle import request, response, tob, touni, json_dumps, _e
+from bottle import request, response, tob, touni, tonat, json_dumps, _e
 import tools
 import wsgiref.util
 import threading
@@ -145,17 +145,20 @@ class TestRequest(unittest.TestCase):
 
     def test_get(self):
         """ Environ: GET data """ 
-        request = BaseRequest({'QUERY_STRING':'a=a&a=1&b=b&c=c'})
+        qs = tonat(tob('a=a&a=1&b=b&c=c&cn=瓶'), 'latin1')
+        request = BaseRequest({'QUERY_STRING':qs})
         self.assertTrue('a' in request.query)
         self.assertTrue('b' in request.query)
         self.assertEqual(['a','1'], request.query.getall('a'))
         self.assertEqual(['b'], request.query.getall('b'))
         self.assertEqual('1', request.query['a'])
         self.assertEqual('b', request.query['b'])
+        self.assertEqual(tonat(tob('瓶'), 'latin1'), request.query['cn'])
+        self.assertEqual(touni('瓶'), request.query.cn)
         
     def test_post(self):
         """ Environ: POST data """ 
-        sq = tob('a=a&a=1&b=b&c=&d')
+        sq = tob('a=a&a=1&b=b&c=&d&cn=瓶')
         e = {}
         wsgiref.util.setup_testing_defaults(e)
         e['wsgi.input'].write(sq)
@@ -171,6 +174,8 @@ class TestRequest(unittest.TestCase):
         self.assertEqual('b', request.POST['b'])
         self.assertEqual('', request.POST['c'])
         self.assertEqual('', request.POST['d'])
+        self.assertEqual(tonat(tob('瓶'), 'latin1'), request.POST['cn'])
+        self.assertEqual(touni('瓶'), request.POST.cn)
 
     def test_bodypost(self):
         sq = tob('foobar')
diff --git a/test/test_formsdict.py b/test/test_formsdict.py
index 7f2cf2e..4fde616 100644
--- a/test/test_formsdict.py
+++ b/test/test_formsdict.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# '瓶' means "Bottle"
 
 import unittest
 from bottle import FormsDict, touni, tob
@@ -6,10 +7,9 @@ from bottle import FormsDict, touni, tob
 class TestFormsDict(unittest.TestCase):
     def test_attr_access(self):
         """ FomsDict.attribute returs string values as unicode. """
-        data = tob('äöü')
-        d = FormsDict(py2=data, py3=data.decode('latin1'))
-        self.assertEqual(touni('äöü'), d.py2)
-        self.assertEqual(touni('äöü'), d.py3)
+        d = FormsDict(py2=tob('瓶'), py3=tob('瓶').decode('latin1'))
+        self.assertEqual(touni('瓶'), d.py2)
+        self.assertEqual(touni('瓶'), d.py3)
 
     def test_attr_missing(self):
         """ FomsDict.attribute returs u'' on missing keys. """
@@ -18,15 +18,13 @@ class TestFormsDict(unittest.TestCase):
 
     def test_attr_unicode_error(self):
         """ FomsDict.attribute returs u'' on UnicodeError. """
-        d = FormsDict(latin=touni('äöü').encode('latin1'))
+        d = FormsDict(latin=touni('öäüß').encode('latin1'))
         self.assertEqual(touni(''), d.latin)
         d.input_encoding = 'latin1'
-        self.assertEqual(touni('äöü'), d.latin)
+        self.assertEqual(touni('öäüß'), d.latin)
 
     def test_decode_method(self):
-        """ FomsDict.attribute returs u'' on UnicodeError. """
-        data = tob('äöü')
-        d = FormsDict(py2=data, py3=data.decode('latin1'))
+        d = FormsDict(py2=tob('瓶'), py3=tob('瓶').decode('latin1'))
         d = d.decode()
         self.assertFalse(d.recode_unicode)
         self.assertTrue(hasattr(list(d.keys())[0], 'encode'))