Enforce unicode json output for jsonutils.load[s]()

simplejson module applies some optimizations on ASCII-only unicode
strings which result in non-unicode json output. See details at [1]
and [2]. To make sure we always return consistent json output no
matter which json implementation is used, we should explicitly convert
the input for json.load[s]() to unicode.

If user wants to pass a file object of non UTF-8 encoding to
json.load[s](), she must also specify this encoding as an argument. To
support this scenario too, we've added 'encoding' argument to
jsonutils.load[s]() implementation.

Made all present JSON tests to run on both supported json library
implementations.

Added explicit dependency for simplejson to be able to test different
implementations in unit tests. Distributors still running Python 2.6
are recommended but not required to install simplejson.

Related-Bug: 1314129

[1]: https://code.djangoproject.com/ticket/11742
[2]: https://code.google.com/p/simplejson/issues/detail?id=40

Change-Id: Ic815ca3df94c33edec9104172048b2cd94b92e3f

2 files changed, 54 insertions, 9 deletions

diff --git a/openstack/common/jsonutils.py b/openstack/common/jsonutils.py
index f00a801..1a35e04 100644
--- a/openstack/common/jsonutils.py
+++ b/openstack/common/jsonutils.py
@@ -31,6 +31,7 @@ This module provides a few things:
 '''
 
 
+import codecs
 import datetime
 import functools
 import inspect
@@ -52,6 +53,7 @@ import six.moves.xmlrpc_client as xmlrpclib
 
 from openstack.common import gettextutils
 from openstack.common import importutils
+from openstack.common import strutils
 from openstack.common import timeutils
 
 netaddr = importutils.try_import("netaddr")
@@ -166,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs):
     return json.dumps(value, default=default, **kwargs)
 
 
-def loads(s):
-    return json.loads(s)
+def loads(s, encoding='utf-8'):
+    return json.loads(strutils.safe_decode(s, encoding))
 
 
-def load(fp):
-    return json.load(fp)
+def load(fp, encoding='utf-8'):
+    return json.load(codecs.getreader(encoding)(fp))
 
 
 try:
diff --git a/tests/unit/test_jsonutils.py b/tests/unit/test_jsonutils.py
index dfdb625..90931a0 100644
--- a/tests/unit/test_jsonutils.py
+++ b/tests/unit/test_jsonutils.py
@@ -14,9 +14,12 @@
 #    under the License.
 
 import datetime
+import json
 
+import mock
 import netaddr
 from oslotest import base as test_base
+import simplejson
 import six
 import six.moves.xmlrpc_client as xmlrpclib
 
@@ -24,17 +27,57 @@ from openstack.common import gettextutils
 from openstack.common import jsonutils
 
 
-class JSONUtilsTestCase(test_base.BaseTestCase):
+class JSONUtilsTestMixin(object):
+
+    json_impl = None
+
+    def setUp(self):
+        super(JSONUtilsTestMixin, self).setUp()
+        self.json_patcher = mock.patch.object(
+            jsonutils, 'json', self.json_impl)
+        self.json_impl_mock = self.json_patcher.start()
+
+    def tearDown(self):
+        self.json_patcher.stop()
+        super(JSONUtilsTestMixin, self).tearDown()
 
     def test_dumps(self):
-        self.assertEqual(jsonutils.dumps({'a': 'b'}), '{"a": "b"}')
+        self.assertEqual('{"a": "b"}', jsonutils.dumps({'a': 'b'}))
 
     def test_loads(self):
-        self.assertEqual(jsonutils.loads('{"a": "b"}'), {'a': 'b'})
+        self.assertEqual({'a': 'b'}, jsonutils.loads('{"a": "b"}'))
+
+    def test_loads_unicode(self):
+        self.assertIsInstance(jsonutils.loads(b'"foo"'), six.text_type)
+        self.assertIsInstance(jsonutils.loads(u'"foo"'), six.text_type)
+
+        # 'test' in Ukrainian
+        i18n_str_unicode = u'"\u0442\u0435\u0441\u0442"'
+        self.assertIsInstance(jsonutils.loads(i18n_str_unicode), six.text_type)
+
+        i18n_str = i18n_str_unicode.encode('utf-8')
+        self.assertIsInstance(jsonutils.loads(i18n_str), six.text_type)
 
     def test_load(self):
-        x = six.StringIO('{"a": "b"}')
-        self.assertEqual(jsonutils.load(x), {'a': 'b'})
+
+        jsontext = u'{"a": "\u0442\u044d\u0441\u0442"}'
+        expected = {u'a': u'\u0442\u044d\u0441\u0442'}
+
+        for encoding in ('utf-8', 'cp1251'):
+            fp = six.BytesIO(jsontext.encode(encoding))
+            result = jsonutils.load(fp, encoding=encoding)
+            self.assertEqual(expected, result)
+            for key, val in result.items():
+                self.assertIsInstance(key, six.text_type)
+                self.assertIsInstance(val, six.text_type)
+
+
+class JSONUtilsTestJson(JSONUtilsTestMixin, test_base.BaseTestCase):
+    json_impl = json
+
+
+class JSONUtilsTestSimpleJson(JSONUtilsTestMixin, test_base.BaseTestCase):
+    json_impl = simplejson
 
 
 class ToPrimitiveTestCase(test_base.BaseTestCase):