Merge pull request #11277 from eric-wieser/fix-older-ctypes-compat

BUG: Work around past and present PEP3118 issues in ctypes

3 files changed, 144 insertions, 15 deletions

diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py
index 8c6596d13..9990bacf0 100644
--- a/numpy/core/_internal.py
+++ b/numpy/core/_internal.py
@@ -756,3 +756,16 @@ def _ufunc_doc_signature_formatter(ufunc):
         out_args=out_args,
         kwargs=kwargs
     )
+
+
+def _is_from_ctypes(obj):
+    # determine if an object comes from ctypes, in order to work around
+    # a bug in the buffer protocol for those objects, bpo-10746
+    try:
+        # ctypes class are new-style, so have an __mro__. This probably fails
+        # for ctypes classes with multiple inheritance.
+        ctype_base = type(obj).__mro__[-2]
+        # right now, they're part of the _ctypes module
+        return 'ctypes' in ctype_base.__module__
+    except Exception:
+        return False
diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c
index cdef899a0..7367902cc 100644
--- a/numpy/core/src/multiarray/ctors.c
+++ b/numpy/core/src/multiarray/ctors.c
@@ -11,6 +11,7 @@
 
 #include "npy_config.h"
 
+#include "npy_import.h"
 #include "npy_pycompat.h"
 #include "multiarraymodule.h"
 
@@ -1302,7 +1303,51 @@ PyArray_New(PyTypeObject *subtype, int nd, npy_intp *dims, int type_num,
 }
 
 
-/* Steals a reference to the memory view */
+NPY_NO_EXPORT PyArray_Descr *
+_dtype_from_buffer_3118(PyObject *memoryview)
+{
+    PyArray_Descr *descr;
+    Py_buffer *view = PyMemoryView_GET_BUFFER(memoryview);
+    if (view->format != NULL) {
+        descr = _descriptor_from_pep3118_format(view->format);
+        if (descr == NULL) {
+            return NULL;
+        }
+    }
+    else {
+        /* If no format is specified, just assume a byte array
+         * TODO: void would make more sense here, as it wouldn't null
+         *       terminate.
+         */
+        descr = PyArray_DescrNewFromType(NPY_STRING);
+        descr->elsize = view->itemsize;
+    }
+    return descr;
+}
+
+
+/*
+ * Call the python _is_from_ctypes
+ */
+NPY_NO_EXPORT int
+_is_from_ctypes(PyObject *obj) {
+    PyObject *ret_obj;
+    static PyObject *py_func = NULL;
+
+    npy_cache_import("numpy.core._internal", "_is_from_ctypes", &py_func);
+
+    if (py_func == NULL) {
+        return -1;
+    }
+    ret_obj = PyObject_CallFunctionObjArgs(py_func, obj, NULL);
+    if (ret_obj == NULL) {
+        return -1;
+    }
+
+    return PyObject_IsTrue(ret_obj);
+}
+
+
 NPY_NO_EXPORT PyObject *
 _array_from_buffer_3118(PyObject *memoryview)
 {
@@ -1315,27 +1360,75 @@ _array_from_buffer_3118(PyObject *memoryview)
     npy_intp shape[NPY_MAXDIMS], strides[NPY_MAXDIMS];
 
     view = PyMemoryView_GET_BUFFER(memoryview);
-    if (view->format != NULL) {
-        descr = _descriptor_from_pep3118_format(view->format);
-        if (descr == NULL) {
-            goto fail;
+    nd = view->ndim;
+    descr = _dtype_from_buffer_3118(memoryview);
+
+    if (descr == NULL) {
+        return NULL;
+    }
+
+    /* Sanity check */
+    if (descr->elsize != view->itemsize) {
+        /* Ctypes has bugs in its PEP3118 implementation, which we need to
+         * work around.
+         *
+         * bpo-10746
+         * bpo-32780
+         * bpo-32782
+         *
+         * Note that even if the above are fixed in master, we have to drop the
+         * early patch versions of python to actually make use of the fixes.
+         */
+
+        int is_ctypes = _is_from_ctypes(view->obj);
+        if (is_ctypes < 0) {
+            /* This error is not useful */
+            PyErr_WriteUnraisable(view->obj);
+            is_ctypes = 0;
         }
 
-        /* Sanity check */
-        if (descr->elsize != view->itemsize) {
+        if (!is_ctypes) {
+            /* This object has no excuse for a broken PEP3118 buffer */
             PyErr_SetString(
                     PyExc_RuntimeError,
                     "Item size computed from the PEP 3118 buffer format "
                     "string does not match the actual item size.");
-            goto fail;
+            Py_DECREF(descr);
+            return NULL;
+        }
+
+        if (PyErr_Warn(
+                    PyExc_RuntimeWarning,
+                    "A builtin ctypes object gave a PEP3118 format "
+                    "string that does not match its itemsize, so a "
+                    "best-guess will be made of the data type. "
+                    "Newer versions of python may behave correctly.") < 0) {
+            Py_DECREF(descr);
+            return NULL;
+        }
+
+        /* Thankfully, np.dtype(ctypes_type) works in most cases.
+         * For an array input, this produces a dtype containing all the
+         * dimensions, so the array is now 0d.
+         */
+        nd = 0;
+        descr = (PyArray_Descr *)PyObject_CallFunctionObjArgs(
+                (PyObject *)&PyArrayDescr_Type, Py_TYPE(view->obj), NULL);
+        if (descr == NULL) {
+            return NULL;
+        }
+        if (descr->elsize != view->len) {
+            PyErr_SetString(
+                    PyExc_RuntimeError,
+                    "For the given ctypes object, neither the item size "
+                    "computed from the PEP 3118 buffer format nor from "
+                    "converting the type to a np.dtype matched the actual "
+                    "size. This is a bug both in python and numpy");
+            Py_DECREF(descr);
+            return NULL;
         }
-    }
-    else {
-        descr = PyArray_DescrNewFromType(NPY_STRING);
-        descr->elsize = view->itemsize;
     }
 
-    nd = view->ndim;
     if (view->shape != NULL) {
         int k;
         if (nd > NPY_MAXDIMS || nd < 0) {
@@ -1379,13 +1472,12 @@ _array_from_buffer_3118(PyObject *memoryview)
             &PyArray_Type, descr,
             nd, shape, strides, view->buf,
             flags, NULL, memoryview);
-    Py_DECREF(memoryview);
     return r;
 
+
 fail:
     Py_XDECREF(r);
     Py_XDECREF(descr);
-    Py_DECREF(memoryview);
     return NULL;
 
 }
@@ -1506,6 +1598,7 @@ PyArray_GetArrayParamsFromObject(PyObject *op,
         }
         else {
             PyObject *arr = _array_from_buffer_3118(memoryview);
+            Py_DECREF(memoryview);
             if (arr == NULL) {
                 return -1;
             }
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index 1bca46bf7..05dd2efd1 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -6562,6 +6562,29 @@ class TestNewBufferProtocol(object):
             ValueError, "format string",
             np.array, m)
 
+    def test_ctypes_integer_via_memoryview(self):
+        # gh-11150, due to bpo-10746
+        for c_integer in {ctypes.c_int, ctypes.c_long, ctypes.c_longlong}:
+            value = c_integer(42)
+            with warnings.catch_warnings(record=True) as w:
+                warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
+                np.asarray(value)
+
+    def test_ctypes_struct_via_memoryview(self):
+        # gh-10528
+        class foo(ctypes.Structure):
+            _fields_ = [('a', ctypes.c_uint8), ('b', ctypes.c_uint32)]
+        f = foo(a=1, b=2)
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
+            arr = np.asarray(f)
+
+        assert_equal(arr['a'], 1)
+        assert_equal(arr['b'], 2)
+        f.a = 3
+        assert_equal(arr['a'], 3)
+
 
 class TestArrayAttributeDeletion(object):