Fix #184 threaded import issue, prep v3.12.0v3.12.0

9 files changed, 102 insertions, 87 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 9b98658..9c66080 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,10 @@
+Version 3.12.0 released 2017-11-05
+
+* Fix threaded import race condition
+  https://github.com/simplejson/simplejson/issues/184
+* Move RawJSON implementation to simplejson.raw_json module
+* Move JSONDecodeError implementation to simplejson.errors module
+
 Version 3.11.1 released 2017-06-19
 
 * Fix issue with item_sort_key when speedups are available, and add
diff --git a/conf.py b/conf.py
index 44451a8..374da34 100644
--- a/conf.py
+++ b/conf.py
@@ -42,9 +42,9 @@ copyright = '2017, Bob Ippolito'
 # other places throughout the built documents.
 #
 # The short X.Y version.
-version = '3.11'
+version = '3.12'
 # The full version, including alpha/beta/rc tags.
-release = '3.11.1'
+release = '3.12.0'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
diff --git a/setup.py b/setup.py
index aaad55b..70e2391 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ from distutils.errors import CCompilerError, DistutilsExecError, \
     DistutilsPlatformError
 
 IS_PYPY = hasattr(sys, 'pypy_translation_info')
-VERSION = '3.11.1'
+VERSION = '3.12.0'
 DESCRIPTION = "Simple, fast, extensible JSON encoder/decoder for Python"
 
 with open('README.rst', 'r') as f:
diff --git a/simplejson/__init__.py b/simplejson/__init__.py
index bd144ff..2f14262 100644
--- a/simplejson/__init__.py
+++ b/simplejson/__init__.py
@@ -97,20 +97,21 @@ Using simplejson.tool from the shell to validate and pretty-print::
     Expecting property name: line 1 column 3 (char 2)
 """
 from __future__ import absolute_import
-__version__ = '3.11.1'
+__version__ = '3.12.0'
 __all__ = [
     'dump', 'dumps', 'load', 'loads',
     'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
-    'OrderedDict', 'simple_first',
+    'OrderedDict', 'simple_first', 'RawJSON'
 ]
 
 __author__ = 'Bob Ippolito <bob@redivi.com>'
 
 from decimal import Decimal
 
-from .scanner import JSONDecodeError
+from .errors import JSONDecodeError
+from .raw_json import RawJSON
 from .decoder import JSONDecoder
-from .encoder import JSONEncoder, JSONEncoderForHTML, RawJSON
+from .encoder import JSONEncoder, JSONEncoderForHTML
 def _import_OrderedDict():
     import collections
     try:
diff --git a/simplejson/_speedups.c b/simplejson/_speedups.c
index 36dcf91..7b46d8a 100644
--- a/simplejson/_speedups.c
+++ b/simplejson/_speedups.c
@@ -263,17 +263,10 @@ moduleinit(void);
 
 #define MIN_EXPANSION 6
 
-static PyObject* RawJSONType;
+static PyObject* RawJSONType = NULL;
 static int
 is_raw_json(PyObject *obj)
 {
-    if (RawJSONType == NULL) {
-        PyObject *encoder_module = PyImport_ImportModule("simplejson.encoder");
-        RawJSONType = PyObject_GetAttrString(encoder_module, "RawJSON");
-        Py_DECREF(encoder_module);
-        if (RawJSONType == NULL)
-            return 0;
-    }
     return PyObject_IsInstance(obj, RawJSONType) ? 1 : 0;
 }
 
@@ -785,22 +778,12 @@ bail:
     return NULL;
 }
 
+/* Use JSONDecodeError exception to raise a nice looking ValueError subclass */
+static PyObject *JSONDecodeError = NULL;
 static void
 raise_errmsg(char *msg, PyObject *s, Py_ssize_t end)
 {
-    /* Use JSONDecodeError exception to raise a nice looking ValueError subclass */
-    static PyObject *JSONDecodeError = NULL;
-    PyObject *exc;
-    if (JSONDecodeError == NULL) {
-        PyObject *scanner = PyImport_ImportModule("simplejson.scanner");
-        if (scanner == NULL)
-            return;
-        JSONDecodeError = PyObject_GetAttrString(scanner, "JSONDecodeError");
-        Py_DECREF(scanner);
-        if (JSONDecodeError == NULL)
-            return;
-    }
-    exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end);
+    PyObject *exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end);
     if (exc) {
         PyErr_SetObject(JSONDecodeError, exc);
         Py_DECREF(exc);
@@ -3349,6 +3332,17 @@ static struct PyModuleDef moduledef = {
 };
 #endif
 
+PyObject *
+import_dependency(char *module_name, char *attr_name)
+{
+    PyObject *module = PyImport_ImportModule(module_name);
+    if (module == NULL)
+        return NULL;
+    PyObject *rval = PyObject_GetAttrString(module, attr_name);
+    Py_DECREF(module);
+    return rval;
+}
+
 static PyObject *
 moduleinit(void)
 {
@@ -3367,6 +3361,12 @@ moduleinit(void)
     PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType);
     Py_INCREF((PyObject*)&PyEncoderType);
     PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType);
+    RawJSONType = import_dependency("simplejson.raw_json", "RawJSON");
+    if (RawJSONType == NULL)
+        return NULL;
+    JSONDecodeError = import_dependency("simplejson.errors", "JSONDecodeError");
+    if (JSONDecodeError == NULL)
+        return NULL;
     return m;
 }
 
diff --git a/simplejson/encoder.py b/simplejson/encoder.py
index b5c3141..c87468a 100644
--- a/simplejson/encoder.py
+++ b/simplejson/encoder.py
@@ -14,7 +14,8 @@ def _import_speedups():
         return None, None
 c_encode_basestring_ascii, c_make_encoder = _import_speedups()
 
-from simplejson.decoder import PosInf
+from .decoder import PosInf
+from .raw_json import RawJSON
 
 #ESCAPE = re.compile(ur'[\x00-\x1f\\"\b\f\n\r\t\u2028\u2029]')
 # This is required because u() will mangle the string and ur'' isn't valid
@@ -39,14 +40,6 @@ for i in [0x2028, 0x2029]:
 
 FLOAT_REPR = repr
 
-class RawJSON(object):
-    """Wrap an encoded JSON document for direct embedding in the output
-
-    """
-    def __init__(self, encoded_json):
-        self.encoded_json = encoded_json
-
-
 def encode_basestring(s, _PY3=PY3, _q=u('"')):
     """Return a JSON representation of a Python string
 
diff --git a/simplejson/errors.py b/simplejson/errors.py
new file mode 100644
index 0000000..b97ab1e
--- /dev/null
+++ b/simplejson/errors.py
@@ -0,0 +1,53 @@
+"""Error classes used by simplejson
+"""
+__all__ = ['JSONDecodeError']
+
+
+def linecol(doc, pos):
+    lineno = doc.count('\n', 0, pos) + 1
+    if lineno == 1:
+        colno = pos + 1
+    else:
+        colno = pos - doc.rindex('\n', 0, pos)
+    return lineno, colno
+
+
+def errmsg(msg, doc, pos, end=None):
+    lineno, colno = linecol(doc, pos)
+    msg = msg.replace('%r', repr(doc[pos:pos + 1]))
+    if end is None:
+        fmt = '%s: line %d column %d (char %d)'
+        return fmt % (msg, lineno, colno, pos)
+    endlineno, endcolno = linecol(doc, end)
+    fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
+    return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
+
+
+class JSONDecodeError(ValueError):
+    """Subclass of ValueError with the following additional properties:
+
+    msg: The unformatted error message
+    doc: The JSON document being parsed
+    pos: The start index of doc where parsing failed
+    end: The end index of doc where parsing failed (may be None)
+    lineno: The line corresponding to pos
+    colno: The column corresponding to pos
+    endlineno: The line corresponding to end (may be None)
+    endcolno: The column corresponding to end (may be None)
+
+    """
+    # Note that this exception is used from _speedups
+    def __init__(self, msg, doc, pos, end=None):
+        ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
+        self.msg = msg
+        self.doc = doc
+        self.pos = pos
+        self.end = end
+        self.lineno, self.colno = linecol(doc, pos)
+        if end is not None:
+            self.endlineno, self.endcolno = linecol(doc, end)
+        else:
+            self.endlineno, self.endcolno = None, None
+
+    def __reduce__(self):
+        return self.__class__, (self.msg, self.doc, self.pos, self.end)
diff --git a/simplejson/raw_json.py b/simplejson/raw_json.py
new file mode 100644
index 0000000..2071a70
--- /dev/null
+++ b/simplejson/raw_json.py
@@ -0,0 +1,9 @@
+"""Implementation of RawJSON
+"""
+
+class RawJSON(object):
+    """Wrap an encoded JSON document for direct embedding in the output
+
+    """
+    def __init__(self, encoded_json):
+        self.encoded_json = encoded_json
diff --git a/simplejson/scanner.py b/simplejson/scanner.py
index 5abed35..85e385e 100644
--- a/simplejson/scanner.py
+++ b/simplejson/scanner.py
@@ -1,9 +1,10 @@
 """JSON token scanner
 """
 import re
+from .errors import JSONDecodeError
 def _import_c_make_scanner():
     try:
-        from simplejson._speedups import make_scanner
+        from ._speedups import make_scanner
         return make_scanner
     except ImportError:
         return None
@@ -15,55 +16,6 @@ NUMBER_RE = re.compile(
     r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
     (re.VERBOSE | re.MULTILINE | re.DOTALL))
 
-class JSONDecodeError(ValueError):
-    """Subclass of ValueError with the following additional properties:
-
-    msg: The unformatted error message
-    doc: The JSON document being parsed
-    pos: The start index of doc where parsing failed
-    end: The end index of doc where parsing failed (may be None)
-    lineno: The line corresponding to pos
-    colno: The column corresponding to pos
-    endlineno: The line corresponding to end (may be None)
-    endcolno: The column corresponding to end (may be None)
-
-    """
-    # Note that this exception is used from _speedups
-    def __init__(self, msg, doc, pos, end=None):
-        ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
-        self.msg = msg
-        self.doc = doc
-        self.pos = pos
-        self.end = end
-        self.lineno, self.colno = linecol(doc, pos)
-        if end is not None:
-            self.endlineno, self.endcolno = linecol(doc, end)
-        else:
-            self.endlineno, self.endcolno = None, None
-
-    def __reduce__(self):
-        return self.__class__, (self.msg, self.doc, self.pos, self.end)
-
-
-def linecol(doc, pos):
-    lineno = doc.count('\n', 0, pos) + 1
-    if lineno == 1:
-        colno = pos + 1
-    else:
-        colno = pos - doc.rindex('\n', 0, pos)
-    return lineno, colno
-
-
-def errmsg(msg, doc, pos, end=None):
-    lineno, colno = linecol(doc, pos)
-    msg = msg.replace('%r', repr(doc[pos:pos + 1]))
-    if end is None:
-        fmt = '%s: line %d column %d (char %d)'
-        return fmt % (msg, lineno, colno, pos)
-    endlineno, endcolno = linecol(doc, end)
-    fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
-    return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
-
 
 def py_make_scanner(context):
     parse_object = context.parse_object