Issue #3329: Add new APIs to customize memory allocators

* Add a new PyMemAllocators structure
* New functions:

  - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree(): GIL-free memory
    allocator functions
  - PyMem_GetRawAllocators(), PyMem_SetRawAllocators()
  - PyMem_GetAllocators(), PyMem_SetAllocators()
  - PyMem_SetupDebugHooks()
  - _PyObject_GetArenaAllocators(), _PyObject_SetArenaAllocators()

* Add unit test for PyMem_Malloc(0) and PyObject_Malloc(0)
* Add unit test for new get/set allocators functions
* PyObject_Malloc() now falls back on PyMem_Malloc() instead of malloc() if
  size is bigger than SMALL_REQUEST_THRESHOLD, and PyObject_Realloc() falls
  back on PyMem_Realloc() instead of realloc()
* PyMem_Malloc() and PyMem_Realloc() now always call malloc() and realloc(),
  instead of calling PyObject_Malloc() and PyObject_Realloc() in debug mode

1 files changed, 71 insertions, 20 deletions

diff --git a/Include/pymem.h b/Include/pymem.h
index 10b5bea5eb..34d93186a3 100644
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -11,6 +11,40 @@
 extern "C" {
 #endif
 
+typedef struct {
+    /* user context passed as the first argument to the 3 functions */
+    void *ctx;
+
+    /* allocate memory */
+    void* (*malloc) (void *ctx, size_t size);
+
+    /* allocate memory or resize a memory buffer */
+    void* (*realloc) (void *ctx, void *ptr, size_t new_size);
+
+    /* release memory */
+    void (*free) (void *ctx, void *ptr);
+} PyMemAllocators;
+
+/* Raw memory allocators, system functions: malloc(), realloc(), free().
+
+   These functions are thread-safe, the GIL does not need to be held. */
+
+/* Get internal functions of PyMem_RawMalloc(), PyMem_RawRealloc() and
+   PyMem_RawFree(). *ctx_p is an arbitrary user value. */
+PyAPI_FUNC(void) PyMem_GetRawAllocators(PyMemAllocators *allocators);
+
+/* Set internal functions of PyMem_RawMalloc(), PyMem_RawRealloc() and
+   PyMem_RawFree(). ctx is an arbitrary user value.
+
+   PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
+   functions do no call original functions anymore. */
+PyAPI_FUNC(void) PyMem_SetRawAllocators(PyMemAllocators *allocators);
+
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+
+
 /* BEWARE:
 
    Each interface exports both functions and macros.  Extension modules should
@@ -49,21 +83,11 @@ extern "C" {
    performed on failure (no exception is set, no warning is printed, etc).
 */
 
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyMem_Free(void *);
-
-/* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
-   no longer supported. They used to call PyErr_NoMemory() on failure. */
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
 /* Macros. */
-#ifdef PYMALLOC_DEBUG
-/* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC		_PyMem_DebugMalloc
-#define PyMem_REALLOC		_PyMem_DebugRealloc
-#define PyMem_FREE		_PyMem_DebugFree
-
-#else	/* ! PYMALLOC_DEBUG */
 
 /* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
    for malloc(0), which would be treated as an error. Some platforms
@@ -71,13 +95,9 @@ PyAPI_FUNC(void) PyMem_Free(void *);
    pymalloc. To solve these problems, allocate an extra byte. */
 /* Returns NULL to indicate error if a negative size or size larger than
    Py_ssize_t can represent is supplied.  Helps prevents security holes. */
-#define PyMem_MALLOC(n)		((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
-				: malloc((n) ? (n) : 1))
-#define PyMem_REALLOC(p, n)	((size_t)(n) > (size_t)PY_SSIZE_T_MAX  ? NULL \
-				: realloc((p), (n) ? (n) : 1))
-#define PyMem_FREE		free
-
-#endif	/* PYMALLOC_DEBUG */
+#define PyMem_MALLOC(n)         PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n)     PyMem_Realloc(p, n)
+#define PyMem_FREE(p)           PyMem_Free(p)
 
 /*
  * Type-oriented memory interface
@@ -115,6 +135,37 @@ PyAPI_FUNC(void) PyMem_Free(void *);
 #define PyMem_Del		PyMem_Free
 #define PyMem_DEL		PyMem_FREE
 
+/* Get internal functions of PyMem_Malloc(), PyMem_Realloc()
+   and PyMem_Free() */
+PyAPI_FUNC(void) PyMem_GetAllocators(PyMemAllocators *allocators);
+
+/* Set internal functions of PyMem_Malloc(), PyMem_Realloc() and PyMem_Free().
+
+   malloc(ctx, 0) and realloc(ctx, ptr, 0) must not return NULL: it would be
+   treated as an error.
+
+   PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
+   functions do no call original functions anymore. */
+PyAPI_FUNC(void) PyMem_SetAllocators(PyMemAllocators *allocators);
+
+/* Setup hooks to detect bugs in the following Python memory allocator
+   functions:
+
+   - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
+   - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
+   - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
+
+   Newly allocated memory is filled with the byte 0xCB, freed memory is filled
+   with the byte 0xDB. Additionnal checks:
+
+   - detect API violations, ex: PyObject_Free() called on a buffer allocated
+     by PyMem_Malloc()
+   - detect write before the start of the buffer (buffer underflow)
+   - detect write after the end of the buffer (buffer overflow)
+
+   The function does nothing if Python is not compiled is debug mode. */
+PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
+
 #ifdef __cplusplus
 }
 #endif