From 781b087c90817a312e3dc6dd55ef174a798b98d8 Mon Sep 17 00:00:00 2001 From: scoder Date: Wed, 5 Apr 2023 15:09:30 +0200 Subject: Implement support for the new PyLong struct layout in Py3.12a7. (GH-5353) See https://github.com/python/cpython/pull/102464 --- Cython/Utility/Builtins.c | 2 +- Cython/Utility/Optimize.c | 111 ++++++++++++++++------------------ Cython/Utility/StringTools.c | 6 +- Cython/Utility/TypeConversion.c | 130 +++++++++++++++++++++++----------------- 4 files changed, 133 insertions(+), 116 deletions(-) diff --git a/Cython/Utility/Builtins.c b/Cython/Utility/Builtins.c index cf0c69e62..5cc786c4b 100644 --- a/Cython/Utility/Builtins.c +++ b/Cython/Utility/Builtins.c @@ -234,7 +234,7 @@ static PyObject *__Pyx_PyLong_AbsNeg(PyObject *num);/*proto*/ #define __Pyx_PyNumber_Absolute(x) \ ((likely(PyLong_CheckExact(x))) ? \ - (likely(Py_SIZE(x) >= 0) ? (Py_INCREF(x), (x)) : __Pyx_PyLong_AbsNeg(x)) : \ + (likely(__Pyx_PyLong_IsNonNeg(x)) ? (Py_INCREF(x), (x)) : __Pyx_PyLong_AbsNeg(x)) : \ PyNumber_Absolute(x)) #else diff --git a/Cython/Utility/Optimize.c b/Cython/Utility/Optimize.c index 9ec7bda37..a16fb6ac9 100644 --- a/Cython/Utility/Optimize.c +++ b/Cython/Utility/Optimize.c @@ -979,14 +979,12 @@ static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject #endif if (likely(PyLong_CheckExact(exp))) { #if CYTHON_USE_PYLONG_INTERNALS - const Py_ssize_t size = Py_SIZE(exp); - // tuned to optimise branch prediction - if (likely(size == 1)) { - shiftby = __Pyx_PyLong_Digits(exp)[0]; - } else if (size == 0) { + if (__Pyx_PyLong_IsZero(exp)) { return PyInt_FromLong(1L); - } else if (unlikely(size < 0)) { + } else if (__Pyx_PyLong_IsNeg(exp)) { goto fallback; + } else if (__Pyx_PyLong_IsCompact(exp)) { + shiftby = __Pyx_PyLong_CompactValueUnsigned(exp); } else { shiftby = PyLong_AsSsize_t(exp); } @@ -1062,21 +1060,18 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els if (likely(PyLong_CheckExact({{pyval}}))) { int unequal; unsigned long uintval; - Py_ssize_t size = Py_SIZE({{pyval}}); + Py_ssize_t size = __Pyx_PyLong_DigitCount({{pyval}}); const digit* digits = __Pyx_PyLong_Digits({{pyval}}); if (intval == 0) { - // == 0 => Py_SIZE(pyval) == 0 - {{return_compare('size', '0', c_op)}} + {{return_compare('__Pyx_PyLong_IsZero(%s)' % pyval, '1', c_op)}} } else if (intval < 0) { - // < 0 => Py_SIZE(pyval) < 0 - if (size >= 0) + if (__Pyx_PyLong_IsNonNeg({{pyval}})) {{return_false if op == 'Eq' else return_true}}; // both are negative => can use absolute values now. intval = -intval; - size = -size; } else { // > 0 => Py_SIZE(pyval) > 0 - if (size <= 0) + if (__Pyx_PyLong_IsNeg({{pyval}})) {{return_false if op == 'Eq' else return_true}}; } // After checking that the sign is the same (and excluding 0), now compare the absolute values. @@ -1242,20 +1237,15 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, PY_LONG_LONG ll{{ival}}, llx; #endif {{endif}} - const digit* digits = __Pyx_PyLong_Digits({{pyval}}); - const Py_ssize_t size = Py_SIZE({{pyval}}); {{if c_op == '&'}} // special case for &-ing arbitrarily large numbers with known single digit operands if ((intval & PyLong_MASK) == intval) { - long result = 0; - if(likely(size)) { - result = intval & (likely(size>0) ? digits[0] : (PyLong_MASK - digits[0] + 1)); - } + long result = intval & (long) __Pyx_PyLong_CompactValue({{pyval}}); return PyLong_FromLong(result); } {{endif}} // special cases for 0: + - * % / // | ^ & >> << - if (unlikely(size == 0)) { + if (unlikely(__Pyx_PyLong_IsZero({{pyval}}))) { {{if order == 'CObj' and c_op in '%/'}} // division by zero! {{zerodiv_check('0')}} @@ -1277,10 +1267,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, {{endif}} } // handle most common case first to avoid indirect branch and optimise branch prediction - if (likely(__Pyx_sst_abs(size) <= 1)) { - {{ival}} = likely(size) ? digits[0] : 0; - if (size == -1) {{ival}} = -{{ival}}; + if (likely(__Pyx_PyLong_IsCompact({{pyval}}))) { + {{ival}} = __Pyx_PyLong_CompactValue({{pyval}}); } else { + const digit* digits = __Pyx_PyLong_Digits({{pyval}}); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount({{pyval}}); switch (size) { {{for _size in range(2, 5)}} {{for _case in (-_size, _size)}} @@ -1337,7 +1328,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, x += ((x != 0) & ((x ^ b) < 0)) * b; {{elif op == 'TrueDivide'}} if ((8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) - || __Pyx_sst_abs(size) <= 52 / PyLong_SHIFT) { + || __Pyx_PyLong_DigitCount({{pyval}}) <= 52 / PyLong_SHIFT) { return PyFloat_FromDouble((double)a / (double)b); } return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); @@ -1497,46 +1488,50 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv if (likely(PyLong_CheckExact({{pyval}}))) { #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = __Pyx_PyLong_Digits({{pyval}}); - const Py_ssize_t size = Py_SIZE({{pyval}}); - switch (size) { - case 0: {{fval}} = 0.0; {{zerodiv_check(fval)}} break; - case -1: {{fval}} = -(double) digits[0]; break; - case 1: {{fval}} = (double) digits[0]; break; - {{for _size in (2, 3, 4)}} - case -{{_size}}: - case {{_size}}: - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT && ((8 * sizeof(unsigned long) < 53) || ({{_size-1}} * PyLong_SHIFT < 53))) { - {{fval}} = (double) {{pylong_join(_size, 'digits')}}; - // let CPython do its own float rounding from 2**53 on (max. consecutive integer in double float) - if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) ((PY_LONG_LONG)1 << 53))) { - if (size == {{-_size}}) - {{fval}} = -{{fval}}; - break; + if (__Pyx_PyLong_IsZero({{pyval}})) { + {{fval}} = 0.0; + {{zerodiv_check(fval)}} + } else if (__Pyx_PyLong_IsCompact({{pyval}})) { + {{fval}} = (double) __Pyx_PyLong_CompactValue({{pyval}}); + } else { + const digit* digits = __Pyx_PyLong_Digits({{pyval}}); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount({{pyval}}); + switch (size) { + {{for _size in (2, 3, 4)}} + case -{{_size}}: + case {{_size}}: + if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT && ((8 * sizeof(unsigned long) < 53) || ({{_size-1}} * PyLong_SHIFT < 53))) { + {{fval}} = (double) {{pylong_join(_size, 'digits')}}; + // let CPython do its own float rounding from 2**53 on (max. consecutive integer in double float) + if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) ((PY_LONG_LONG)1 << 53))) { + if (size == {{-_size}}) + {{fval}} = -{{fval}}; + break; + } } - } - // Fall through if size doesn't fit safely into a double anymore. - // It may not be obvious that this is a safe fall-through given the "fval < 2**53" - // check above. However, the number of digits that CPython uses for a given PyLong - // value is minimal, and together with the "(size-1) * SHIFT < 53" check above, - // this should make it safe. - CYTHON_FALLTHROUGH; - {{endfor}} - default: + // Fall through if size doesn't fit safely into a double anymore. + // It may not be obvious that this is a safe fall-through given the "fval < 2**53" + // check above. However, the number of digits that CPython uses for a given PyLong + // value is minimal, and together with the "(size-1) * SHIFT < 53" check above, + // this should make it safe. + CYTHON_FALLTHROUGH; + {{endfor}} + default: #endif {{if op in ('Eq', 'Ne')}} - return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( - PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}})); + return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( + PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}})); {{else}} - {{fval}} = PyLong_AsDouble({{pyval}}); - if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL; - {{if zerodiv_check(fval)}} - #if !CYTHON_USE_PYLONG_INTERNALS - {{zerodiv_check(fval)}} - #endif - {{endif}} + {{fval}} = PyLong_AsDouble({{pyval}}); + if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL; + {{if zerodiv_check(fval)}} + #if !CYTHON_USE_PYLONG_INTERNALS + {{zerodiv_check(fval)}} + #endif + {{endif}} {{endif}} #if CYTHON_USE_PYLONG_INTERNALS + } } #endif } else { diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c index 5d757fd35..553585987 100644 --- a/Cython/Utility/StringTools.c +++ b/Cython/Utility/StringTools.c @@ -1061,11 +1061,11 @@ static CYTHON_INLINE int __Pyx_PyByteArray_AppendObject(PyObject* bytearray, PyO } else #endif #if CYTHON_USE_PYLONG_INTERNALS - if (likely(PyLong_CheckExact(value)) && likely(Py_SIZE(value) == 1 || Py_SIZE(value) == 0)) { - if (Py_SIZE(value) == 0) { + if (likely(PyLong_CheckExact(value)) && likely(__Pyx_PyLong_IsCompact(value))) { + if (__Pyx_PyLong_IsZero(value)) { ival = 0; } else { - ival = __Pyx_PyLong_Digits(value)[0]; + ival = __Pyx_PyLong_CompactValue(value); if (unlikely(ival > 255)) goto bad_range; } } else diff --git a/Cython/Utility/TypeConversion.c b/Cython/Utility/TypeConversion.c index 750fc0cfa..52b8c26d2 100644 --- a/Cython/Utility/TypeConversion.c +++ b/Cython/Utility/TypeConversion.c @@ -130,6 +130,33 @@ static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); // string conversion work the same in all circumstances). #if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & 3) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << 3)) // (2 << NON_SIZE_BITS) + #define __Pyx_PyLong_CompactValue(x) ((1 - __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) (((PyLongObject*)x)->long_value.lv_tag >> 3) // (>> NON_SIZE_BITS) + #define __Pyx_PyLong_SignedDigitCount(x) \ + ((1 - ((PyLongObject*)x)->long_value.lv_tag & 3) * (((PyLongObject*)x)->long_value.lv_tag >> 3)) // (>> NON_SIZE_BITS) + #else // Py < 3.12 + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x) \ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #endif + + typedef sdigit __Pyx_compact_pylong; + #if PY_VERSION_HEX >= 0x030C00A5 #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) #else @@ -413,14 +440,12 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { #endif if (likely(PyLong_CheckExact(b))) { #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = __Pyx_PyLong_Digits(b); - const Py_ssize_t size = Py_SIZE(b); // handle most common case first to avoid indirect branch and optimise branch prediction - if (likely(__Pyx_sst_abs(size) <= 1)) { - ival = likely(size) ? digits[0] : 0; - if (size == -1) ival = -ival; - return ival; + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); switch (size) { {{for _size in (2, 3, 4)}} {{for _case in (_size, -_size)}} @@ -486,24 +511,12 @@ static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj) { double val; if (PyLong_CheckExact(obj)) { #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = __Pyx_PyLong_Digits(obj); - switch (Py_SIZE(obj)) { - case 0: - val = 0.0; - goto no_error; - // single digit PyLong values always cast safely to double - case 1: - val = (double) digits[0]; - goto no_error; - case -1: - val = (double) - (sdigit) digits[0]; - goto no_error; - default: - val = PyLong_AsDouble(obj); + if (likely(__Pyx_PyLong_IsCompact(obj))) { + val = (double) __Pyx_PyLong_CompactValue(obj); + goto no_error; } -#else - val = PyLong_AsDouble(obj); #endif + val = PyLong_AsDouble(obj); } else if (PyUnicode_CheckExact(obj)) { val = __Pyx_PyUnicode_AsDouble(obj); } else if (PyBytes_CheckExact(obj)) { @@ -976,24 +989,31 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = __Pyx_PyLong_Digits(x); - switch (Py_SIZE(x)) { - case 0: return ({{TYPE}}) 0; - case 1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, digits[0]) - {{for _size in (2, 3, 4)}} - case {{_size}}: - if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { - if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) - } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { - return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + //} else if (__Pyx_PyLong_IsZero(x)) { + // return ({{TYPE}}) 0; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_pylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + {{for _size in (2, 3, 4)}} + case {{_size}}: + if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) + } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { + return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; + } } - } - break; - {{endfor}} + break; + {{endfor}} + } } #endif -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 if (unlikely(Py_SIZE(x) < 0)) { goto raise_neg_overflow; } @@ -1017,24 +1037,26 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { } else { // signed #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = __Pyx_PyLong_Digits(x); - switch (Py_SIZE(x)) { - case 0: return ({{TYPE}}) 0; - case -1: __PYX_VERIFY_RETURN_INT({{TYPE}}, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, +digits[0]) - {{for _size in (2, 3, 4)}} - {{for _case in (-_size, _size)}} - case {{_case}}: - if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { - if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) - } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { - return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + {{for _size in (2, 3, 4)}} + {{for _case in (-_size, _size)}} + case {{_case}}: + if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) + } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { + return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); + } } - } - break; - {{endfor}} - {{endfor}} + break; + {{endfor}} + {{endfor}} + } } #endif if ((sizeof({{TYPE}}) <= sizeof(long))) { -- cgit v1.2.1