Handle zero-length sublist correctly in Python -> SQL array conversion.

If PLySequence_ToArray came across a zero-length sublist, it'd compute
the overall array size as zero, possibly leading to a memory clobber.
(This would likely qualify as a security bug, were it not that plpython
is an untrusted language already.)

I think there are other corner-case issues in this code as well, notably
that the error messages don't match the core code and for some ranges
of array sizes you'd get "invalid memory alloc request size" rather than
the intended message about array size.

Really this code has no business doing its own array size calculation
at all, so remove the faulty code in favor of using ArrayGetNItems().

Per bug #17912 from Alexander Lakhin.  Bug seems to have come in with
commit 94aceed31, so back-patch to all supported branches.

Discussion: https://postgr.es/m/17912-82ceed78731d9cdc@postgresql.org

1 files changed, 7 insertions, 16 deletions

diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c
index 7018c9d404..864b5f1765 100644
--- a/src/pl/plpython/plpy_typeio.c
+++ b/src/pl/plpython/plpy_typeio.c
@@ -1136,7 +1136,7 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
 	int			i;
 	Datum	   *elems;
 	bool	   *nulls;
-	int64		len;
+	int			len;
 	int			ndim;
 	int			dims[MAXDIM];
 	int			lbs[MAXDIM];
@@ -1155,7 +1155,6 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
 	 * Determine the number of dimensions, and their sizes.
 	 */
 	ndim = 0;
-	len = 1;
 
 	Py_INCREF(plrv);
 
@@ -1174,17 +1173,6 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
 		if (dims[ndim] < 0)
 			PLy_elog(ERROR, "could not determine sequence length for function return value");
 
-		if (dims[ndim] > MaxAllocSize)
-			ereport(ERROR,
-					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
-					 errmsg("array size exceeds the maximum allowed")));
-
-		len *= dims[ndim];
-		if (len > MaxAllocSize)
-			ereport(ERROR,
-					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
-					 errmsg("array size exceeds the maximum allowed")));
-
 		if (dims[ndim] == 0)
 		{
 			/* empty sequence */
@@ -1214,15 +1202,18 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
 					 errmsg("return value of function with array return type is not a Python sequence")));
 
 		ndim = 1;
-		len = dims[0] = PySequence_Length(plrv);
+		dims[0] = PySequence_Length(plrv);
 	}
 
+	/* Allocate space for work arrays, after detecting array size overflow */
+	len = ArrayGetNItems(ndim, dims);
+	elems = palloc(sizeof(Datum) * len);
+	nulls = palloc(sizeof(bool) * len);
+
 	/*
 	 * Traverse the Python lists, in depth-first order, and collect all the
 	 * elements at the bottom level into 'elems'/'nulls' arrays.
 	 */
-	elems = palloc(sizeof(Datum) * len);
-	nulls = palloc(sizeof(bool) * len);
 	currelem = 0;
 	PLySequence_ToArray_recurse(arg->u.array.elm, plrv,
 								dims, ndim, 0,