Implement support for the new PyLong struct layout in Py3.12a7. (GH-5353)

See https://github.com/python/cpython/pull/102464

4 files changed, 133 insertions, 116 deletions

diff --git a/Cython/Utility/Builtins.c b/Cython/Utility/Builtins.c
index cf0c69e62..5cc786c4b 100644
--- a/Cython/Utility/Builtins.c
+++ b/Cython/Utility/Builtins.c
@@ -234,7 +234,7 @@ static PyObject *__Pyx_PyLong_AbsNeg(PyObject *num);/*proto*/
 
 #define __Pyx_PyNumber_Absolute(x) \
     ((likely(PyLong_CheckExact(x))) ? \
-         (likely(Py_SIZE(x) >= 0) ? (Py_INCREF(x), (x)) : __Pyx_PyLong_AbsNeg(x)) : \
+         (likely(__Pyx_PyLong_IsNonNeg(x)) ? (Py_INCREF(x), (x)) : __Pyx_PyLong_AbsNeg(x)) : \
          PyNumber_Absolute(x))
 
 #else
diff --git a/Cython/Utility/Optimize.c b/Cython/Utility/Optimize.c
index 9ec7bda37..a16fb6ac9 100644
--- a/Cython/Utility/Optimize.c
+++ b/Cython/Utility/Optimize.c
@@ -979,14 +979,12 @@ static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject
 #endif
     if (likely(PyLong_CheckExact(exp))) {
         #if CYTHON_USE_PYLONG_INTERNALS
-        const Py_ssize_t size = Py_SIZE(exp);
-        // tuned to optimise branch prediction
-        if (likely(size == 1)) {
-            shiftby = __Pyx_PyLong_Digits(exp)[0];
-        } else if (size == 0) {
+        if (__Pyx_PyLong_IsZero(exp)) {
             return PyInt_FromLong(1L);
-        } else if (unlikely(size < 0)) {
+        } else if (__Pyx_PyLong_IsNeg(exp)) {
             goto fallback;
+        } else if (__Pyx_PyLong_IsCompact(exp)) {
+            shiftby = __Pyx_PyLong_CompactValueUnsigned(exp);
         } else {
             shiftby = PyLong_AsSsize_t(exp);
         }
@@ -1062,21 +1060,18 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els
     if (likely(PyLong_CheckExact({{pyval}}))) {
         int unequal;
         unsigned long uintval;
-        Py_ssize_t size = Py_SIZE({{pyval}});
+        Py_ssize_t size = __Pyx_PyLong_DigitCount({{pyval}});
         const digit* digits = __Pyx_PyLong_Digits({{pyval}});
         if (intval == 0) {
-            // == 0  =>  Py_SIZE(pyval) == 0
-            {{return_compare('size', '0', c_op)}}
+            {{return_compare('__Pyx_PyLong_IsZero(%s)' % pyval, '1', c_op)}}
         } else if (intval < 0) {
-            // < 0  =>  Py_SIZE(pyval) < 0
-            if (size >= 0)
+            if (__Pyx_PyLong_IsNonNeg({{pyval}}))
                 {{return_false if op == 'Eq' else return_true}};
             // both are negative => can use absolute values now.
             intval = -intval;
-            size = -size;
         } else {
             // > 0  =>  Py_SIZE(pyval) > 0
-            if (size <= 0)
+            if (__Pyx_PyLong_IsNeg({{pyval}}))
                 {{return_false if op == 'Eq' else return_true}};
         }
         // After checking that the sign is the same (and excluding 0), now compare the absolute values.
@@ -1242,20 +1237,15 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval,
         PY_LONG_LONG ll{{ival}}, llx;
 #endif
         {{endif}}
-        const digit* digits = __Pyx_PyLong_Digits({{pyval}});
-        const Py_ssize_t size = Py_SIZE({{pyval}});
         {{if c_op == '&'}}
         // special case for &-ing arbitrarily large numbers with known single digit operands
         if ((intval & PyLong_MASK) == intval) {
-            long result = 0;
-            if(likely(size)) {
-                result = intval & (likely(size>0) ? digits[0] : (PyLong_MASK - digits[0] + 1));
-            }
+            long result = intval & (long) __Pyx_PyLong_CompactValue({{pyval}});
             return PyLong_FromLong(result);
         }
         {{endif}}
         // special cases for 0: + - * % / // | ^ & >> <<
-        if (unlikely(size == 0)) {
+        if (unlikely(__Pyx_PyLong_IsZero({{pyval}}))) {
             {{if order == 'CObj' and c_op in '%/'}}
             // division by zero!
             {{zerodiv_check('0')}}
@@ -1277,10 +1267,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval,
             {{endif}}
         }
         // handle most common case first to avoid indirect branch and optimise branch prediction
-        if (likely(__Pyx_sst_abs(size) <= 1)) {
-            {{ival}} = likely(size) ? digits[0] : 0;
-            if (size == -1) {{ival}} = -{{ival}};
+        if (likely(__Pyx_PyLong_IsCompact({{pyval}}))) {
+            {{ival}} = __Pyx_PyLong_CompactValue({{pyval}});
         } else {
+            const digit* digits = __Pyx_PyLong_Digits({{pyval}});
+            const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount({{pyval}});
             switch (size) {
                 {{for _size in range(2, 5)}}
                 {{for _case in (-_size, _size)}}
@@ -1337,7 +1328,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval,
                 x += ((x != 0) & ((x ^ b) < 0)) * b;
             {{elif op == 'TrueDivide'}}
                 if ((8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53)))
-                        || __Pyx_sst_abs(size) <= 52 / PyLong_SHIFT) {
+                        || __Pyx_PyLong_DigitCount({{pyval}}) <= 52 / PyLong_SHIFT) {
                     return PyFloat_FromDouble((double)a / (double)b);
                 }
                 return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2);
@@ -1497,46 +1488,50 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv
 
     if (likely(PyLong_CheckExact({{pyval}}))) {
         #if CYTHON_USE_PYLONG_INTERNALS
-        const digit* digits = __Pyx_PyLong_Digits({{pyval}});
-        const Py_ssize_t size = Py_SIZE({{pyval}});
-        switch (size) {
-            case  0: {{fval}} = 0.0; {{zerodiv_check(fval)}} break;
-            case -1: {{fval}} = -(double) digits[0]; break;
-            case  1: {{fval}} = (double) digits[0]; break;
-            {{for _size in (2, 3, 4)}}
-            case -{{_size}}:
-            case {{_size}}:
-                if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT && ((8 * sizeof(unsigned long) < 53) || ({{_size-1}} * PyLong_SHIFT < 53))) {
-                    {{fval}} = (double) {{pylong_join(_size, 'digits')}};
-                    // let CPython do its own float rounding from 2**53 on (max. consecutive integer in double float)
-                    if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) ((PY_LONG_LONG)1 << 53))) {
-                        if (size == {{-_size}})
-                            {{fval}} = -{{fval}};
-                        break;
+        if (__Pyx_PyLong_IsZero({{pyval}})) {
+            {{fval}} = 0.0;
+            {{zerodiv_check(fval)}}
+        } else if (__Pyx_PyLong_IsCompact({{pyval}})) {
+            {{fval}} = (double) __Pyx_PyLong_CompactValue({{pyval}});
+        } else {
+            const digit* digits = __Pyx_PyLong_Digits({{pyval}});
+            const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount({{pyval}});
+            switch (size) {
+                {{for _size in (2, 3, 4)}}
+                case -{{_size}}:
+                case {{_size}}:
+                    if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT && ((8 * sizeof(unsigned long) < 53) || ({{_size-1}} * PyLong_SHIFT < 53))) {
+                        {{fval}} = (double) {{pylong_join(_size, 'digits')}};
+                        // let CPython do its own float rounding from 2**53 on (max. consecutive integer in double float)
+                        if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) ((PY_LONG_LONG)1 << 53))) {
+                            if (size == {{-_size}})
+                                {{fval}} = -{{fval}};
+                            break;
+                        }
                     }
-                }
-                // Fall through if size doesn't fit safely into a double anymore.
-                // It may not be obvious that this is a safe fall-through given the "fval < 2**53"
-                // check above.  However, the number of digits that CPython uses for a given PyLong
-                // value is minimal, and together with the "(size-1) * SHIFT < 53" check above,
-                // this should make it safe.
-                CYTHON_FALLTHROUGH;
-            {{endfor}}
-            default:
+                    // Fall through if size doesn't fit safely into a double anymore.
+                    // It may not be obvious that this is a safe fall-through given the "fval < 2**53"
+                    // check above.  However, the number of digits that CPython uses for a given PyLong
+                    // value is minimal, and together with the "(size-1) * SHIFT < 53" check above,
+                    // this should make it safe.
+                    CYTHON_FALLTHROUGH;
+                {{endfor}}
+                default:
         #endif
         {{if op in ('Eq', 'Ne')}}
-            return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}(
-                PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}}));
+                    return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}(
+                        PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}}));
         {{else}}
-            {{fval}} = PyLong_AsDouble({{pyval}});
-            if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL;
-            {{if zerodiv_check(fval)}}
-            #if !CYTHON_USE_PYLONG_INTERNALS
-            {{zerodiv_check(fval)}}
-            #endif
-            {{endif}}
+                    {{fval}} = PyLong_AsDouble({{pyval}});
+                    if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL;
+                    {{if zerodiv_check(fval)}}
+                    #if !CYTHON_USE_PYLONG_INTERNALS
+                    {{zerodiv_check(fval)}}
+                    #endif
+                    {{endif}}
         {{endif}}
         #if CYTHON_USE_PYLONG_INTERNALS
+            }
         }
         #endif
     } else {
diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c
index 5d757fd35..553585987 100644
--- a/Cython/Utility/StringTools.c
+++ b/Cython/Utility/StringTools.c
@@ -1061,11 +1061,11 @@ static CYTHON_INLINE int __Pyx_PyByteArray_AppendObject(PyObject* bytearray, PyO
     } else
 #endif
 #if CYTHON_USE_PYLONG_INTERNALS
-    if (likely(PyLong_CheckExact(value)) && likely(Py_SIZE(value) == 1 || Py_SIZE(value) == 0)) {
-        if (Py_SIZE(value) == 0) {
+    if (likely(PyLong_CheckExact(value)) && likely(__Pyx_PyLong_IsCompact(value))) {
+        if (__Pyx_PyLong_IsZero(value)) {
             ival = 0;
         } else {
-            ival = __Pyx_PyLong_Digits(value)[0];
+            ival = __Pyx_PyLong_CompactValue(value);
             if (unlikely(ival > 255)) goto bad_range;
         }
     } else
diff --git a/Cython/Utility/TypeConversion.c b/Cython/Utility/TypeConversion.c
index 750fc0cfa..52b8c26d2 100644
--- a/Cython/Utility/TypeConversion.c
+++ b/Cython/Utility/TypeConversion.c
@@ -130,6 +130,33 @@ static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*);
 // string conversion work the same in all circumstances).
 
 #if CYTHON_USE_PYLONG_INTERNALS
+  #if PY_VERSION_HEX >= 0x030C00A7
+  #define __Pyx_PyLong_Sign(x)  (((PyLongObject*)x)->long_value.lv_tag & 3)
+  #define __Pyx_PyLong_IsNeg(x)  ((__Pyx_PyLong_Sign(x) & 2) != 0)
+  #define __Pyx_PyLong_IsNonNeg(x)  (!__Pyx_PyLong_IsNeg(x))
+  #define __Pyx_PyLong_IsZero(x)  (__Pyx_PyLong_Sign(x) & 1)
+  #define __Pyx_PyLong_IsPos(x)  (__Pyx_PyLong_Sign(x) == 0)
+  #define __Pyx_PyLong_IsCompact(x)  (((PyLongObject*)x)->long_value.lv_tag < (2 << 3))  // (2 << NON_SIZE_BITS)
+  #define __Pyx_PyLong_CompactValue(x)  ((1 - __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0])
+  #define __Pyx_PyLong_CompactValueUnsigned(x)  (__Pyx_PyLong_Digits(x)[0])
+  #define __Pyx_PyLong_DigitCount(x)  (((PyLongObject*)x)->long_value.lv_tag >> 3)  // (>> NON_SIZE_BITS)
+  #define __Pyx_PyLong_SignedDigitCount(x)  \
+        ((1 - ((PyLongObject*)x)->long_value.lv_tag & 3) * (((PyLongObject*)x)->long_value.lv_tag >> 3))  // (>> NON_SIZE_BITS)
+  #else  // Py < 3.12
+  #define __Pyx_PyLong_IsNeg(x)  (Py_SIZE(x) < 0)
+  #define __Pyx_PyLong_IsNonNeg(x)  (Py_SIZE(x) >= 0)
+  #define __Pyx_PyLong_IsZero(x)  (Py_SIZE(x) == 0)
+  #define __Pyx_PyLong_IsPos(x)  (Py_SIZE(x) > 0)
+  #define __Pyx_PyLong_IsCompact(x)  (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1)
+  #define __Pyx_PyLong_CompactValue(x)  \
+        ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0]))
+  #define __Pyx_PyLong_CompactValueUnsigned(x)  ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0])
+  #define __Pyx_PyLong_DigitCount(x)  __Pyx_sst_abs(Py_SIZE(x))
+  #define __Pyx_PyLong_SignedDigitCount(x)  Py_SIZE(x)
+  #endif
+
+  typedef sdigit  __Pyx_compact_pylong;
+
   #if PY_VERSION_HEX >= 0x030C00A5
   #define __Pyx_PyLong_Digits(x)  (((PyLongObject*)x)->long_value.ob_digit)
   #else
@@ -413,14 +440,12 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) {
 #endif
   if (likely(PyLong_CheckExact(b))) {
     #if CYTHON_USE_PYLONG_INTERNALS
-    const digit* digits = __Pyx_PyLong_Digits(b);
-    const Py_ssize_t size = Py_SIZE(b);
     // handle most common case first to avoid indirect branch and optimise branch prediction
-    if (likely(__Pyx_sst_abs(size) <= 1)) {
-        ival = likely(size) ? digits[0] : 0;
-        if (size == -1) ival = -ival;
-        return ival;
+    if (likely(__Pyx_PyLong_IsCompact(b))) {
+        return __Pyx_PyLong_CompactValue(b);
     } else {
+      const digit* digits = __Pyx_PyLong_Digits(b);
+      const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b);
       switch (size) {
          {{for _size in (2, 3, 4)}}
          {{for _case in (_size, -_size)}}
@@ -486,24 +511,12 @@ static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj) {
     double val;
     if (PyLong_CheckExact(obj)) {
 #if CYTHON_USE_PYLONG_INTERNALS
-        const digit* digits = __Pyx_PyLong_Digits(obj);
-        switch (Py_SIZE(obj)) {
-            case 0:
-                val = 0.0;
-                goto no_error;
-            // single digit PyLong values always cast safely to double
-            case 1:
-                val = (double) digits[0];
-                goto no_error;
-            case -1:
-                val = (double) - (sdigit) digits[0];
-                goto no_error;
-            default:
-                val = PyLong_AsDouble(obj);
+        if (likely(__Pyx_PyLong_IsCompact(obj))) {
+            val = (double) __Pyx_PyLong_CompactValue(obj);
+            goto no_error;
         }
-#else
-        val = PyLong_AsDouble(obj);
 #endif
+        val = PyLong_AsDouble(obj);
     } else if (PyUnicode_CheckExact(obj)) {
         val = __Pyx_PyUnicode_AsDouble(obj);
     } else if (PyBytes_CheckExact(obj)) {
@@ -976,24 +989,31 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) {
     if (likely(PyLong_Check(x))) {
         if (is_unsigned) {
 #if CYTHON_USE_PYLONG_INTERNALS
-            const digit* digits = __Pyx_PyLong_Digits(x);
-            switch (Py_SIZE(x)) {
-                case  0: return ({{TYPE}}) 0;
-                case  1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, digits[0])
-                {{for _size in (2, 3, 4)}}
-                case {{_size}}:
-                    if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) {
-                        if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) {
-                            __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}})
-                        } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) {
-                            return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}};
+            if (unlikely(__Pyx_PyLong_IsNeg(x))) {
+                goto raise_neg_overflow;
+            //} else if (__Pyx_PyLong_IsZero(x)) {
+            //    return ({{TYPE}}) 0;
+            } else if (__Pyx_PyLong_IsCompact(x)) {
+                __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_pylong, __Pyx_PyLong_CompactValueUnsigned(x))
+            } else {
+                const digit* digits = __Pyx_PyLong_Digits(x);
+                assert(__Pyx_PyLong_DigitCount(x) > 1);
+                switch (__Pyx_PyLong_DigitCount(x)) {
+                    {{for _size in (2, 3, 4)}}
+                    case {{_size}}:
+                        if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) {
+                            if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) {
+                                __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}})
+                            } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) {
+                                return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}};
+                            }
                         }
-                    }
-                    break;
-                {{endfor}}
+                        break;
+                    {{endfor}}
+                }
             }
 #endif
-#if CYTHON_COMPILING_IN_CPYTHON
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7
             if (unlikely(Py_SIZE(x) < 0)) {
                 goto raise_neg_overflow;
             }
@@ -1017,24 +1037,26 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) {
         } else {
             // signed
 #if CYTHON_USE_PYLONG_INTERNALS
-            const digit* digits = __Pyx_PyLong_Digits(x);
-            switch (Py_SIZE(x)) {
-                case  0: return ({{TYPE}}) 0;
-                case -1: __PYX_VERIFY_RETURN_INT({{TYPE}}, sdigit, (sdigit) (-(sdigit)digits[0]))
-                case  1: __PYX_VERIFY_RETURN_INT({{TYPE}},  digit, +digits[0])
-                {{for _size in (2, 3, 4)}}
-                {{for _case in (-_size, _size)}}
-                case {{_case}}:
-                    if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) {
-                        if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) {
-                            __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}})
-                        } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) {
-                            return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}});
+            if (__Pyx_PyLong_IsCompact(x)) {
+                __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x))
+            } else {
+                const digit* digits = __Pyx_PyLong_Digits(x);
+                assert(__Pyx_PyLong_DigitCount(x) > 1);
+                switch (__Pyx_PyLong_SignedDigitCount(x)) {
+                    {{for _size in (2, 3, 4)}}
+                    {{for _case in (-_size, _size)}}
+                    case {{_case}}:
+                        if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) {
+                            if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) {
+                                __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}})
+                            } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) {
+                                return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}});
+                            }
                         }
-                    }
-                    break;
-                {{endfor}}
-                {{endfor}}
+                        break;
+                    {{endfor}}
+                    {{endfor}}
+                }
             }
 #endif
             if ((sizeof({{TYPE}}) <= sizeof(long))) {